From 5c36b1a05230690047a5dea5c41bbb11ea7e6a96 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 16 Sep 2019 15:46:58 +0800 Subject: [PATCH 001/217] refine cache code Former-commit-id: 2a982ed5663a5e34c46d8a3be2bb5c875ee6b63a --- cpp/src/cache/Cache.cpp | 212 ------------------ cpp/src/cache/Cache.h | 36 +-- cpp/src/cache/Cache.inl | 173 ++++++++++++++ cpp/src/cache/CacheMgr.h | 12 +- cpp/src/cache/{CacheMgr.cpp => CacheMgr.inl} | 60 ++--- cpp/src/cache/CpuCacheMgr.cpp | 16 +- cpp/src/cache/CpuCacheMgr.h | 9 +- cpp/src/cache/GpuCacheMgr.cpp | 15 +- cpp/src/cache/GpuCacheMgr.h | 8 +- cpp/src/db/engine/ExecutionEngineImpl.cpp | 8 +- .../server/appendix/server_config.yaml | 42 +--- cpp/unittest/server/cache_test.cpp | 86 ++++--- 12 files changed, 292 insertions(+), 385 deletions(-) delete mode 100644 cpp/src/cache/Cache.cpp create mode 100644 cpp/src/cache/Cache.inl rename cpp/src/cache/{CacheMgr.cpp => CacheMgr.inl} (64%) diff --git a/cpp/src/cache/Cache.cpp b/cpp/src/cache/Cache.cpp deleted file mode 100644 index 336bf4e82c..0000000000 --- a/cpp/src/cache/Cache.cpp +++ /dev/null @@ -1,212 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// - -#include "Cache.h" -#include "utils/Log.h" - -#include - -namespace zilliz { -namespace milvus { -namespace cache { - -constexpr double DEFAULT_THRESHHOLD_PERCENT = 0.85; - -Cache::Cache(int64_t capacity, uint64_t cache_max_count) - : usage_(0), - capacity_(capacity), - freemem_percent_(DEFAULT_THRESHHOLD_PERCENT), - lru_(cache_max_count) { -// AGENT_LOG_DEBUG << "Construct Cache with capacity " << std::to_string(mem_capacity) -} - -void Cache::set_capacity(int64_t capacity) { - if(capacity > 0) { - capacity_ = capacity; - free_memory(); - } -} - -size_t Cache::size() const { - std::lock_guard lock(mutex_); - return lru_.size(); -} - -bool Cache::exists(const std::string& key) { - std::lock_guard lock(mutex_); - return lru_.exists(key); -} - -DataObjPtr Cache::get(const std::string& key) { - std::lock_guard lock(mutex_); - if(!lru_.exists(key)){ - return nullptr; - } - - const CacheObjPtr& cache_obj = lru_.get(key); - return cache_obj->data_; -} - -void Cache::insert(const std::string& key, const DataObjPtr& data_ptr) { - { - std::lock_guard lock(mutex_); - - /* if key already exist, over-write old data */ - if (lru_.exists(key)) { - CacheObjPtr obj_ptr = lru_.get(key); - - usage_ -= obj_ptr->data_->size(); - obj_ptr->data_ = data_ptr; - usage_ += data_ptr->size(); - } else { - CacheObjPtr obj_ptr(new CacheObj(data_ptr)); - lru_.put(key, obj_ptr); - usage_ += data_ptr->size(); - } - - SERVER_LOG_DEBUG << "Insert " << key << " size:" << data_ptr->size() - << " bytes into cache, usage: " << usage_ << " bytes"; - } - - if (usage_ > capacity_) { - SERVER_LOG_DEBUG << "Current usage " << usage_ - << " exceeds cache capacity " << capacity_ - << ", start free memory"; - free_memory(); - } -} - -void Cache::erase(const std::string& key) { - std::lock_guard lock(mutex_); - if(!lru_.exists(key)){ - return; - } - - const CacheObjPtr& obj_ptr = lru_.get(key); - const DataObjPtr& data_ptr = obj_ptr->data_; - usage_ -= data_ptr->size(); - - SERVER_LOG_DEBUG << "Erase " << key << " size: " << data_ptr->size(); - - lru_.erase(key); -} - -void Cache::clear() { - std::lock_guard lock(mutex_); - lru_.clear(); - usage_ = 0; - SERVER_LOG_DEBUG << "Clear cache !"; -} - -#if 0 /* caiyd 20190221, need more testing before enable */ -void Cache::flush_to_file(const std::string& key, const CacheObjPtr& obj_ptr) { - if (!this->swap_enabled_) return; - - const DataObjPtr data_ptr = obj_ptr->data(); - - if (data_ptr == nullptr || data_ptr->size() == 0) return; - if (data_ptr->ptr() == nullptr) return; - - std::string name = std::to_string(reinterpret_cast(data_ptr.get())); - filesys::CreateDirectory(this->swap_path_); - - /* write cache data to file */ - obj_ptr->set_file_path(this->swap_path_ + "/" + name); - std::shared_ptr outfile = nullptr; - filesys::OpenWritableFile(obj_ptr->file_path(), false, &outfile); - filesys::WriteFile(outfile, data_ptr->ptr().get(), data_ptr->size()); - (void)outfile->Close(); - - AGENT_LOG_DEBUG << "Flush cache data: " << key << ", to file: " << obj_ptr->file_path(); - - /* free cache memory */ - data_ptr->ptr().reset(); - usage_ -= data_ptr->size(); -} - -void Cache::restore_from_file(const std::string& key, const CacheObjPtr& obj_ptr) { - if (!this->swap_enabled_) return; - - const DataObjPtr data_ptr = obj_ptr->data(); - if (data_ptr == nullptr || data_ptr->size() == 0) return; - - std::shared_ptr infile = nullptr; - int64_t file_size, bytes_read; - - /* load cache data from file */ - if (!filesys::FileExist(obj_ptr->file_path())) { - THROW_AGENT_UNEXPECTED_ERROR("File not exist: " + obj_ptr->file_path()); - } - filesys::OpenReadableFile(obj_ptr->file_path(), &infile); - infile->GetSize(&file_size); - if (data_ptr->size() != file_size) { - THROW_AGENT_UNEXPECTED_ERROR("File size not match: " + obj_ptr->file_path()); - } - data_ptr->set_ptr(lib::gpu::MakeShared(data_ptr->size(), lib::gpu::MallocHint::kUnifiedGlobal)); - infile->Read(file_size, &bytes_read, data_ptr->ptr().get()); - infile->Close(); - - AGENT_LOG_DEBUG << "Restore cache data: " << key << ", from file: " << obj_ptr->file_path(); - - /* clear file path */ - obj_ptr->set_file_path(""); - usage_ += data_ptr->size(); -} -#endif - -/* free memory space when CACHE occupation exceed its capacity */ -void Cache::free_memory() { - if (usage_ <= capacity_) return; - - int64_t threshhold = capacity_ * freemem_percent_; - int64_t delta_size = usage_ - threshhold; - if(delta_size <= 0) { - delta_size = 1;//ensure at least one item erased - } - - std::set key_array; - int64_t released_size = 0; - - { - std::lock_guard lock(mutex_); - - auto it = lru_.rbegin(); - while (it != lru_.rend() && released_size < delta_size) { - auto& key = it->first; - auto& obj_ptr = it->second; - const auto& data_ptr = obj_ptr->data_; - - key_array.emplace(key); - released_size += data_ptr->size(); - ++it; - } - } - - SERVER_LOG_DEBUG << "to be released memory size: " << released_size; - - for (auto& key : key_array) { - erase(key); - } - - print(); -} - -void Cache::print() { - size_t cache_count = 0; - { - std::lock_guard lock(mutex_); - cache_count = lru_.size(); - } - - SERVER_LOG_DEBUG << "[Cache item count]: " << cache_count; - SERVER_LOG_DEBUG << "[Cache usage]: " << usage_ << " bytes"; - SERVER_LOG_DEBUG << "[Cache capacity]: " << capacity_ << " bytes"; -} - -} // cache -} // milvus -} // zilliz - diff --git a/cpp/src/cache/Cache.h b/cpp/src/cache/Cache.h index 4d6f32b9eb..d0df4bdcab 100644 --- a/cpp/src/cache/Cache.h +++ b/cpp/src/cache/Cache.h @@ -6,35 +6,20 @@ #pragma once +#include "LRU.h" +#include "utils/Log.h" + #include #include #include - -#include "LRU.h" -#include "DataObj.h" +#include namespace zilliz { namespace milvus { namespace cache { -const std::string SWAP_DIR = ".CACHE"; - +template class Cache { -private: - class CacheObj { - public: - CacheObj() = delete; - - CacheObj(const DataObjPtr& data) - : data_(data) { - } - - public: - DataObjPtr data_ = nullptr; - }; - - using CacheObjPtr = std::shared_ptr; - public: //mem_capacity, units:GB Cache(int64_t capacity_gb, uint64_t cache_max_count); @@ -49,11 +34,13 @@ public: size_t size() const; bool exists(const std::string& key); - DataObjPtr get(const std::string& key); - void insert(const std::string& key, const DataObjPtr& data); + ItemObj get(const std::string& key); + void insert(const std::string& key, const ItemObj& item); void erase(const std::string& key); void print(); void clear(); + +private: void free_memory(); private: @@ -61,13 +48,12 @@ private: int64_t capacity_; double freemem_percent_; - LRU lru_; + LRU lru_; mutable std::mutex mutex_; }; -using CachePtr = std::shared_ptr; - } // cache } // milvus } // zilliz +#include "cache/Cache.inl" \ No newline at end of file diff --git a/cpp/src/cache/Cache.inl b/cpp/src/cache/Cache.inl new file mode 100644 index 0000000000..be01c11c77 --- /dev/null +++ b/cpp/src/cache/Cache.inl @@ -0,0 +1,173 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// + + +namespace zilliz { +namespace milvus { +namespace cache { + +constexpr double DEFAULT_THRESHHOLD_PERCENT = 0.85; + +template +Cache::Cache(int64_t capacity, uint64_t cache_max_count) + : usage_(0), + capacity_(capacity), + freemem_percent_(DEFAULT_THRESHHOLD_PERCENT), + lru_(cache_max_count) { +// AGENT_LOG_DEBUG << "Construct Cache with capacity " << std::to_string(mem_capacity) +} + +template +void Cache::set_capacity(int64_t capacity) { + if(capacity > 0) { + capacity_ = capacity; + free_memory(); + } +} + +template +size_t Cache::size() const { + std::lock_guard lock(mutex_); + return lru_.size(); +} + +template +bool Cache::exists(const std::string& key) { + std::lock_guard lock(mutex_); + return lru_.exists(key); +} + +template +ItemObj Cache::get(const std::string& key) { + std::lock_guard lock(mutex_); + if(!lru_.exists(key)){ + return nullptr; + } + + return lru_.get(key); +} + +template +void Cache::insert(const std::string& key, const ItemObj& item) { + if(item == nullptr) { + return; + } + + if(item->size() > capacity_) { + SERVER_LOG_ERROR << "Item size " << item->size() + << " is too large to insert into cache, capacity " << capacity_; + return; + } + + //calculate usage + { + std::lock_guard lock(mutex_); + + //if key already exist, subtract old item size + if (lru_.exists(key)) { + const ItemObj& old_item = lru_.get(key); + usage_ -= old_item->size(); + } + + //plus new item size + usage_ += item->size(); + } + + //if usage exceed capacity, free some items + if (usage_ > capacity_) { + SERVER_LOG_DEBUG << "Current usage " << usage_ + << " exceeds cache capacity " << capacity_ + << ", start free memory"; + free_memory(); + } + + //insert new item + { + std::lock_guard lock(mutex_); + + lru_.put(key, item); + SERVER_LOG_DEBUG << "Insert " << key << " size:" << item->size() + << " bytes into cache, usage: " << usage_ << " bytes"; + } +} + +template +void Cache::erase(const std::string& key) { + std::lock_guard lock(mutex_); + if(!lru_.exists(key)){ + return; + } + + const ItemObj& old_item = lru_.get(key); + usage_ -= old_item->size(); + + SERVER_LOG_DEBUG << "Erase " << key << " size: " << old_item->size(); + + lru_.erase(key); +} + +template +void Cache::clear() { + std::lock_guard lock(mutex_); + lru_.clear(); + usage_ = 0; + SERVER_LOG_DEBUG << "Clear cache !"; +} + +/* free memory space when CACHE occupation exceed its capacity */ +template +void Cache::free_memory() { + if (usage_ <= capacity_) return; + + int64_t threshhold = capacity_ * freemem_percent_; + int64_t delta_size = usage_ - threshhold; + if(delta_size <= 0) { + delta_size = 1;//ensure at least one item erased + } + + std::set key_array; + int64_t released_size = 0; + + { + std::lock_guard lock(mutex_); + + auto it = lru_.rbegin(); + while (it != lru_.rend() && released_size < delta_size) { + auto& key = it->first; + auto& obj_ptr = it->second; + + key_array.emplace(key); + released_size += obj_ptr->size(); + ++it; + } + } + + SERVER_LOG_DEBUG << "to be released memory size: " << released_size; + + for (auto& key : key_array) { + erase(key); + } + + print(); +} + +template +void Cache::print() { + size_t cache_count = 0; + { + std::lock_guard lock(mutex_); + cache_count = lru_.size(); + } + + SERVER_LOG_DEBUG << "[Cache item count]: " << cache_count; + SERVER_LOG_DEBUG << "[Cache usage]: " << usage_ << " bytes"; + SERVER_LOG_DEBUG << "[Cache capacity]: " << capacity_ << " bytes"; +} + +} // cache +} // milvus +} // zilliz + diff --git a/cpp/src/cache/CacheMgr.h b/cpp/src/cache/CacheMgr.h index b6f1ec8ef1..8c5647a83a 100644 --- a/cpp/src/cache/CacheMgr.h +++ b/cpp/src/cache/CacheMgr.h @@ -7,22 +7,23 @@ #pragma once #include "Cache.h" +#include "utils/Log.h" +#include "metrics/Metrics.h" namespace zilliz { namespace milvus { namespace cache { +template class CacheMgr { public: virtual uint64_t ItemCount() const; virtual bool ItemExists(const std::string& key); - virtual DataObjPtr GetItem(const std::string& key); - virtual engine::VecIndexPtr GetIndex(const std::string& key); + virtual ItemObj GetItem(const std::string& key); - virtual void InsertItem(const std::string& key, const DataObjPtr& data); - virtual void InsertItem(const std::string& key, const engine::VecIndexPtr& index); + virtual void InsertItem(const std::string& key, const ItemObj& data); virtual void EraseItem(const std::string& key); @@ -39,6 +40,7 @@ protected: virtual ~CacheMgr(); protected: + using CachePtr = std::shared_ptr>; CachePtr cache_; }; @@ -46,3 +48,5 @@ protected: } } } + +#include "cache/CacheMgr.inl" \ No newline at end of file diff --git a/cpp/src/cache/CacheMgr.cpp b/cpp/src/cache/CacheMgr.inl similarity index 64% rename from cpp/src/cache/CacheMgr.cpp rename to cpp/src/cache/CacheMgr.inl index 977c7e1c42..05bb1eb08a 100644 --- a/cpp/src/cache/CacheMgr.cpp +++ b/cpp/src/cache/CacheMgr.inl @@ -4,22 +4,21 @@ // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// -#include "utils/Log.h" -#include "CacheMgr.h" -#include "metrics/Metrics.h" - namespace zilliz { namespace milvus { namespace cache { -CacheMgr::CacheMgr() { +template +CacheMgr::CacheMgr() { } -CacheMgr::~CacheMgr() { +template +CacheMgr::~CacheMgr() { } -uint64_t CacheMgr::ItemCount() const { +template +uint64_t CacheMgr::ItemCount() const { if(cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return 0; @@ -28,7 +27,8 @@ uint64_t CacheMgr::ItemCount() const { return (uint64_t)(cache_->size()); } -bool CacheMgr::ItemExists(const std::string& key) { +template +bool CacheMgr::ItemExists(const std::string& key) { if(cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return false; @@ -37,7 +37,8 @@ bool CacheMgr::ItemExists(const std::string& key) { return cache_->exists(key); } -DataObjPtr CacheMgr::GetItem(const std::string& key) { +template +ItemObj CacheMgr::GetItem(const std::string& key) { if(cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return nullptr; @@ -46,16 +47,8 @@ DataObjPtr CacheMgr::GetItem(const std::string& key) { return cache_->get(key); } -engine::VecIndexPtr CacheMgr::GetIndex(const std::string& key) { - DataObjPtr obj = GetItem(key); - if(obj != nullptr) { - return obj->data(); - } - - return nullptr; -} - -void CacheMgr::InsertItem(const std::string& key, const DataObjPtr& data) { +template +void CacheMgr::InsertItem(const std::string& key, const ItemObj& data) { if(cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return; @@ -65,18 +58,8 @@ void CacheMgr::InsertItem(const std::string& key, const DataObjPtr& data) { server::Metrics::GetInstance().CacheAccessTotalIncrement(); } -void CacheMgr::InsertItem(const std::string& key, const engine::VecIndexPtr& index) { - if(cache_ == nullptr) { - SERVER_LOG_ERROR << "Cache doesn't exist"; - return; - } - - DataObjPtr obj = std::make_shared(index); - cache_->insert(key, obj); - server::Metrics::GetInstance().CacheAccessTotalIncrement(); -} - -void CacheMgr::EraseItem(const std::string& key) { +template +void CacheMgr::EraseItem(const std::string& key) { if(cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return; @@ -86,7 +69,8 @@ void CacheMgr::EraseItem(const std::string& key) { server::Metrics::GetInstance().CacheAccessTotalIncrement(); } -void CacheMgr::PrintInfo() { +template +void CacheMgr::PrintInfo() { if(cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return; @@ -95,7 +79,8 @@ void CacheMgr::PrintInfo() { cache_->print(); } -void CacheMgr::ClearCache() { +template +void CacheMgr::ClearCache() { if(cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return; @@ -104,7 +89,8 @@ void CacheMgr::ClearCache() { cache_->clear(); } -int64_t CacheMgr::CacheUsage() const { +template +int64_t CacheMgr::CacheUsage() const { if(cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return 0; @@ -113,7 +99,8 @@ int64_t CacheMgr::CacheUsage() const { return cache_->usage(); } -int64_t CacheMgr::CacheCapacity() const { +template +int64_t CacheMgr::CacheCapacity() const { if(cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return 0; @@ -122,7 +109,8 @@ int64_t CacheMgr::CacheCapacity() const { return cache_->capacity(); } -void CacheMgr::SetCapacity(int64_t capacity) { +template +void CacheMgr::SetCapacity(int64_t capacity) { if(cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return; diff --git a/cpp/src/cache/CpuCacheMgr.cpp b/cpp/src/cache/CpuCacheMgr.cpp index 167f91f5e5..69962919fa 100644 --- a/cpp/src/cache/CpuCacheMgr.cpp +++ b/cpp/src/cache/CpuCacheMgr.cpp @@ -20,7 +20,7 @@ CpuCacheMgr::CpuCacheMgr() { server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); int64_t cap = config.GetInt64Value(server::CONFIG_CPU_CACHE_CAPACITY, 16); cap *= unit; - cache_ = std::make_shared(cap, 1UL<<32); + cache_ = std::make_shared>(cap, 1UL<<32); double free_percent = config.GetDoubleValue(server::CACHE_FREE_PERCENT, 0.85); if(free_percent > 0.0 && free_percent <= 1.0) { @@ -31,6 +31,20 @@ CpuCacheMgr::CpuCacheMgr() { } } +CpuCacheMgr* CpuCacheMgr::GetInstance() { + static CpuCacheMgr s_mgr; + return &s_mgr; +} + +engine::VecIndexPtr CpuCacheMgr::GetIndex(const std::string& key) { + DataObjPtr obj = GetItem(key); + if(obj != nullptr) { + return obj->data(); + } + + return nullptr; +} + } } } \ No newline at end of file diff --git a/cpp/src/cache/CpuCacheMgr.h b/cpp/src/cache/CpuCacheMgr.h index 39e33aef89..2aee7501d0 100644 --- a/cpp/src/cache/CpuCacheMgr.h +++ b/cpp/src/cache/CpuCacheMgr.h @@ -6,22 +6,21 @@ #pragma once #include "CacheMgr.h" +#include "DataObj.h" namespace zilliz { namespace milvus { namespace cache { -class CpuCacheMgr : public CacheMgr { +class CpuCacheMgr : public CacheMgr { private: CpuCacheMgr(); public: //TODO: use smart pointer instead - static CacheMgr* GetInstance() { - static CpuCacheMgr s_mgr; - return &s_mgr; - } + static CpuCacheMgr* GetInstance(); + engine::VecIndexPtr GetIndex(const std::string& key); }; } diff --git a/cpp/src/cache/GpuCacheMgr.cpp b/cpp/src/cache/GpuCacheMgr.cpp index 5c1afa3f5c..6567f997ad 100644 --- a/cpp/src/cache/GpuCacheMgr.cpp +++ b/cpp/src/cache/GpuCacheMgr.cpp @@ -25,7 +25,7 @@ GpuCacheMgr::GpuCacheMgr() { int64_t cap = config.GetInt64Value(server::CONFIG_GPU_CACHE_CAPACITY, 0); cap *= G_BYTE; - cache_ = std::make_shared(cap, 1UL<<32); + cache_ = std::make_shared>(cap, 1UL<<32); double free_percent = config.GetDoubleValue(server::GPU_CACHE_FREE_PERCENT, 0.85); if (free_percent > 0.0 && free_percent <= 1.0) { @@ -36,7 +36,7 @@ GpuCacheMgr::GpuCacheMgr() { } } -CacheMgr* GpuCacheMgr::GetInstance(uint64_t gpu_id) { +GpuCacheMgr* GpuCacheMgr::GetInstance(uint64_t gpu_id) { if (instance_.find(gpu_id) == instance_.end()) { std::lock_guard lock(mutex_); if (instance_.find(gpu_id) == instance_.end()) { @@ -49,14 +49,13 @@ CacheMgr* GpuCacheMgr::GetInstance(uint64_t gpu_id) { } } -void GpuCacheMgr::InsertItem(const std::string& key, const DataObjPtr& data) { - //TODO: copy data to gpu - if (cache_ == nullptr) { - SERVER_LOG_ERROR << "Cache doesn't exist"; - return; +engine::VecIndexPtr GpuCacheMgr::GetIndex(const std::string& key) { + DataObjPtr obj = GetItem(key); + if(obj != nullptr) { + return obj->data(); } - cache_->insert(key, data); + return nullptr; } } diff --git a/cpp/src/cache/GpuCacheMgr.h b/cpp/src/cache/GpuCacheMgr.h index 35756733f0..ee0ac68660 100644 --- a/cpp/src/cache/GpuCacheMgr.h +++ b/cpp/src/cache/GpuCacheMgr.h @@ -5,6 +5,8 @@ //////////////////////////////////////////////////////////////////////////////// #include "CacheMgr.h" +#include "DataObj.h" + #include #include @@ -15,13 +17,13 @@ namespace cache { class GpuCacheMgr; using GpuCacheMgrPtr = std::shared_ptr; -class GpuCacheMgr : public CacheMgr { +class GpuCacheMgr : public CacheMgr { public: GpuCacheMgr(); - static CacheMgr* GetInstance(uint64_t gpu_id); + static GpuCacheMgr* GetInstance(uint64_t gpu_id); - void InsertItem(const std::string& key, const DataObjPtr& data) override; + engine::VecIndexPtr GetIndex(const std::string& key); private: static std::mutex mutex_; diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 9ddb864e0d..8a738347d2 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -124,7 +124,7 @@ Status ExecutionEngineImpl::Serialize() { } Status ExecutionEngineImpl::Load(bool to_cache) { - index_ = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location_); + index_ = cache::CpuCacheMgr::GetInstance()->GetIndex(location_); bool already_in_cache = (index_ != nullptr); if (!already_in_cache) { try { @@ -151,7 +151,7 @@ Status ExecutionEngineImpl::Load(bool to_cache) { } Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) { - auto index = zilliz::milvus::cache::GpuCacheMgr::GetInstance(device_id)->GetIndex(location_); + auto index = cache::GpuCacheMgr::GetInstance(device_id)->GetIndex(location_); bool already_in_cache = (index != nullptr); if (already_in_cache) { index_ = index; @@ -178,7 +178,7 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) { } Status ExecutionEngineImpl::CopyToCpu() { - auto index = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location_); + auto index = cache::CpuCacheMgr::GetInstance()->GetIndex(location_); bool already_in_cache = (index != nullptr); if (already_in_cache) { index_ = index; @@ -221,7 +221,7 @@ Status ExecutionEngineImpl::Merge(const std::string &location) { } ENGINE_LOG_DEBUG << "Merge index file: " << location << " to: " << location_; - auto to_merge = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location); + auto to_merge = cache::CpuCacheMgr::GetInstance()->GetIndex(location); if (!to_merge) { try { double physical_size = server::CommonUtil::GetFileSize(location); diff --git a/cpp/unittest/server/appendix/server_config.yaml b/cpp/unittest/server/appendix/server_config.yaml index 1a3dca9d97..e4b3de31b0 100644 --- a/cpp/unittest/server/appendix/server_config.yaml +++ b/cpp/unittest/server/appendix/server_config.yaml @@ -38,44 +38,8 @@ engine_config: use_blas_threshold: 20 resource_config: - # resource list, length: 0~N - # please set a DISK resource and a CPU resource least, or system will not return query result. - # - # example: - # resource_name: # resource name, just using in connections below - # type: DISK # resource type, optional: DISK/CPU/GPU - # device_id: 0 - # enable_executor: false # if is enable executor, optional: true, false - + mode: simple resources: - ssda: - type: DISK - device_id: 0 - enable_executor: false - - cpu: - type: CPU - device_id: 0 - enable_executor: false - - gpu0: - type: GPU - device_id: 0 - enable_executor: true - gpu_resource_num: 2 - pinned_memory: 300 - temp_memory: 300 - - # connection list, length: 0~N - # example: - # connection_name: - # speed: 100 # unit: MS/s - # endpoint: === - connections: - io: - speed: 500 - endpoint: ssda===cpu - pcie0: - speed: 11000 - endpoint: cpu===gpu0 + # - cpu + - gpu0 diff --git a/cpp/unittest/server/cache_test.cpp b/cpp/unittest/server/cache_test.cpp index 501d3f486f..8599098b55 100644 --- a/cpp/unittest/server/cache_test.cpp +++ b/cpp/unittest/server/cache_test.cpp @@ -15,21 +15,27 @@ using namespace zilliz::milvus; namespace { -class InvalidCacheMgr : public cache::CacheMgr { +class InvalidCacheMgr : public cache::CacheMgr { public: InvalidCacheMgr() { } }; -class LessItemCacheMgr : public cache::CacheMgr { +class LessItemCacheMgr : public cache::CacheMgr { public: LessItemCacheMgr() { - cache_ = std::make_shared(1UL << 12, 10); + cache_ = std::make_shared>(1UL << 12, 10); } }; class MockVecIndex : public engine::VecIndex { public: + MockVecIndex(int64_t dim, int64_t total) + : dimension_(dim), + ntotal_(total){ + + } + virtual ErrorCode BuildAll(const long &nb, const float *xb, const long *ids, @@ -93,7 +99,7 @@ public: } public: - int64_t dimension_ = 512; + int64_t dimension_ = 256; int64_t ntotal_ = 0; }; @@ -101,7 +107,7 @@ public: TEST(CacheTest, DUMMY_TEST) { engine::Config cfg; - MockVecIndex mock_index; + MockVecIndex mock_index(256, 1000); mock_index.Dimension(); mock_index.Count(); mock_index.Add(1, nullptr, nullptr); @@ -118,29 +124,27 @@ TEST(CacheTest, DUMMY_TEST) { } TEST(CacheTest, CPU_CACHE_TEST) { - cache::CacheMgr *cpu_mgr = cache::CpuCacheMgr::GetInstance(); + auto cpu_mgr = cache::CpuCacheMgr::GetInstance(); - const int64_t gbyte = 1 << 30; + const int64_t gbyte = 1024*1024*1024; int64_t g_num = 16; int64_t cap = g_num * gbyte; cpu_mgr->SetCapacity(cap); ASSERT_EQ(cpu_mgr->CacheCapacity(), cap); - const int dim = 256; - - for (int i = 0; i < 20; i++) { - MockVecIndex* mock_index = new MockVecIndex(); - mock_index->ntotal_ = 1000000;//less 1G per index - engine::VecIndexPtr index(mock_index); - - cpu_mgr->InsertItem("index_" + std::to_string(i), index); + uint64_t item_count = 20; + for (uint64_t i = 0; i < item_count; i++) { + //each vector is 1k byte, total size less than 1G + engine::VecIndexPtr mock_index = std::make_shared(256, 1000000); + cache::DataObjPtr data_obj = std::make_shared(mock_index); + cpu_mgr->InsertItem("index_" + std::to_string(i), data_obj); } ASSERT_LT(cpu_mgr->ItemCount(), g_num); auto obj = cpu_mgr->GetIndex("index_0"); ASSERT_TRUE(obj == nullptr); - obj = cpu_mgr->GetIndex("index_19"); + obj = cpu_mgr->GetIndex("index_" + std::to_string(item_count - 1)); ASSERT_TRUE(obj != nullptr); { @@ -154,30 +158,24 @@ TEST(CacheTest, CPU_CACHE_TEST) { g_num = 5; cpu_mgr->SetCapacity(g_num * gbyte); - MockVecIndex* mock_index = new MockVecIndex(); - mock_index->ntotal_ = 6000000;//6G less - engine::VecIndexPtr index(mock_index); - - cpu_mgr->InsertItem("index_6g", index); - ASSERT_EQ(cpu_mgr->ItemCount(), 0);//data greater than capacity can not be inserted sucessfully + //each vector is 1k byte, total size less than 6G + engine::VecIndexPtr mock_index = std::make_shared(256, 6000000); + cache::DataObjPtr data_obj = std::make_shared(mock_index); + cpu_mgr->InsertItem("index_6g", data_obj); + ASSERT_FALSE(cpu_mgr->ItemExists("index_6g"));//data greater than capacity can not be inserted sucessfully } cpu_mgr->PrintInfo(); } TEST(CacheTest, GPU_CACHE_TEST) { - cache::CacheMgr* gpu_mgr = cache::GpuCacheMgr::GetInstance(0); - - const int dim = 256; + auto gpu_mgr = cache::GpuCacheMgr::GetInstance(0); for(int i = 0; i < 20; i++) { - MockVecIndex* mock_index = new MockVecIndex(); - mock_index->ntotal_ = 1000; - engine::VecIndexPtr index(mock_index); - - cache::DataObjPtr obj = std::make_shared(index); - - gpu_mgr->InsertItem("index_" + std::to_string(i), obj); + //each vector is 1k byte + engine::VecIndexPtr mock_index = std::make_shared(256, 1000); + cache::DataObjPtr data_obj = std::make_shared(mock_index); + gpu_mgr->InsertItem("index_" + std::to_string(i), data_obj); } auto obj = gpu_mgr->GetItem("index_0"); @@ -187,19 +185,13 @@ TEST(CacheTest, GPU_CACHE_TEST) { for (auto i = 0; i < 3; i++) { // TODO: use gpu index to mock - MockVecIndex *mock_index = new MockVecIndex(); - mock_index->ntotal_ = 1000000; //2G - engine::VecIndexPtr index(mock_index); - cache::DataObjPtr data_obj = std::make_shared(index); + //each vector is 1k byte, total size less than 2G + engine::VecIndexPtr mock_index = std::make_shared(256, 2000000); + cache::DataObjPtr data_obj = std::make_shared(mock_index); std::cout << data_obj->size() <InsertItem("index_" + std::to_string(i), data_obj); } -// ASSERT_EQ(gpu_mgr->ItemCount(), 2); -// auto obj0 = gpu_mgr->GetItem("index_0"); -// ASSERT_EQ(obj0, nullptr); -// auto obj1 = gpu_mgr->GetItem("index_1"); -// auto obj2 = gpu_mgr->GetItem("index_2"); gpu_mgr->ClearCache(); ASSERT_EQ(gpu_mgr->ItemCount(), 0); @@ -213,7 +205,7 @@ TEST(CacheTest, INVALID_TEST) { ASSERT_EQ(mgr.GetItem("test"), nullptr); mgr.InsertItem("test", cache::DataObjPtr()); - mgr.InsertItem("test", engine::VecIndexPtr(nullptr)); + mgr.InsertItem("test", nullptr); mgr.EraseItem("test"); mgr.PrintInfo(); mgr.ClearCache(); @@ -225,12 +217,10 @@ TEST(CacheTest, INVALID_TEST) { { LessItemCacheMgr mgr; for(int i = 0; i < 20; i++) { - MockVecIndex* mock_index = new MockVecIndex(); - mock_index->ntotal_ = 2; - engine::VecIndexPtr index(mock_index); - - cache::DataObjPtr obj = std::make_shared(index); - mgr.InsertItem("index_" + std::to_string(i), obj); + //each vector is 1k byte + engine::VecIndexPtr mock_index = std::make_shared(256, 2); + cache::DataObjPtr data_obj = std::make_shared(mock_index); + mgr.InsertItem("index_" + std::to_string(i), data_obj); } ASSERT_EQ(mgr.GetItem("index_0"), nullptr); } From 785a75a121bea3059086f8491e82de1ea0505864 Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Mon, 16 Sep 2019 20:08:46 +0800 Subject: [PATCH 002/217] MS-552 Add and change the easylogging library Former-commit-id: cc29d15a956a343622f1f17c63b1ada51bb172a5 --- cpp/CHANGELOG.md | 1 + cpp/cmake/ThirdPartyPackages.cmake | 166 +- cpp/src/CMakeLists.txt | 5 +- cpp/src/db/Log.h | 2 +- cpp/src/db/Options.cpp | 2 +- cpp/src/main.cpp | 2 +- cpp/src/utils/Log.h | 2 +- cpp/src/utils/LogUtil.cpp | 25 +- cpp/src/utils/LogUtil.h | 3 +- cpp/src/utils/easylogging++.cc | 3112 +++++++++++++++ cpp/src/utils/easylogging++.h | 4569 +++++++++++++++++++++++ cpp/src/wrapper/knowhere/wrapper_log.h | 2 +- cpp/unittest/CMakeLists.txt | 2 +- cpp/unittest/db/CMakeLists.txt | 4 +- cpp/unittest/db/db_tests.cpp | 2 +- cpp/unittest/db/meta_tests.cpp | 2 +- cpp/unittest/db/misc_test.cpp | 2 +- cpp/unittest/db/mysql_db_test.cpp | 2 +- cpp/unittest/db/mysql_meta_test.cpp | 2 +- cpp/unittest/db/scheduler_test.cpp | 2 +- cpp/unittest/knowhere/CMakeLists.txt | 4 +- cpp/unittest/knowhere/knowhere_test.cpp | 2 +- cpp/unittest/main.cpp | 2 +- cpp/unittest/metrics/CMakeLists.txt | 4 +- cpp/unittest/scheduler/CMakeLists.txt | 5 +- cpp/unittest/server/CMakeLists.txt | 5 + cpp/unittest/server/util_test.cpp | 5 +- 27 files changed, 7821 insertions(+), 115 deletions(-) create mode 100644 cpp/src/utils/easylogging++.cc create mode 100644 cpp/src/utils/easylogging++.h diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 78db90639b..c0b8c1a293 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -7,6 +7,7 @@ Please mark all change in change log and use the ticket from JIRA. ## Bug ## Improvement +- MS-552 - Add and change the easylogging library ## New Feature diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index d1cbc3ddf0..933d470247 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -19,7 +19,7 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES ARROW BOOST BZip2 - Easylogging++ +# Easylogging++ FAISS GTest Knowhere @@ -56,8 +56,8 @@ macro(build_dependency DEPENDENCY_NAME) build_arrow() elseif("${DEPENDENCY_NAME}" STREQUAL "BZip2") build_bzip2() - elseif("${DEPENDENCY_NAME}" STREQUAL "Easylogging++") - build_easyloggingpp() +# elseif("${DEPENDENCY_NAME}" STREQUAL "Easylogging++") +# build_easyloggingpp() elseif("${DEPENDENCY_NAME}" STREQUAL "FAISS") build_faiss() elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest") @@ -295,12 +295,12 @@ else() endif() set(BZIP2_MD5 "00b516f4704d4a7cb50a1d97e6e8e15b") -if(DEFINED ENV{MILVUS_EASYLOGGINGPP_URL}) - set(EASYLOGGINGPP_SOURCE_URL "$ENV{MILVUS_EASYLOGGINGPP_URL}") -else() - set(EASYLOGGINGPP_SOURCE_URL "https://github.com/zuhd-org/easyloggingpp/archive/${EASYLOGGINGPP_VERSION}.tar.gz") -endif() -set(EASYLOGGINGPP_MD5 "b78cd319db4be9b639927657b8aa7732") +#if(DEFINED ENV{MILVUS_EASYLOGGINGPP_URL}) +# set(EASYLOGGINGPP_SOURCE_URL "$ENV{MILVUS_EASYLOGGINGPP_URL}") +#else() +# set(EASYLOGGINGPP_SOURCE_URL "https://github.com/zuhd-org/easyloggingpp/archive/${EASYLOGGINGPP_VERSION}.tar.gz") +#endif() +#set(EASYLOGGINGPP_MD5 "b78cd319db4be9b639927657b8aa7732") if(DEFINED ENV{MILVUS_FAISS_URL}) set(FAISS_SOURCE_URL "$ENV{MILVUS_FAISS_URL}") @@ -817,80 +817,80 @@ endif() # ---------------------------------------------------------------------- # Easylogging++ -macro(build_easyloggingpp) - message(STATUS "Building Easylogging++-${EASYLOGGINGPP_VERSION} from source") - set(EASYLOGGINGPP_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/easyloggingpp_ep-prefix/src/easyloggingpp_ep") - set(EASYLOGGINGPP_INCLUDE_DIR "${EASYLOGGINGPP_PREFIX}/include") - set(EASYLOGGINGPP_STATIC_LIB - "${EASYLOGGINGPP_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}easyloggingpp${CMAKE_STATIC_LIBRARY_SUFFIX}") - - set(EASYLOGGINGPP_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${EASYLOGGINGPP_PREFIX}" - -DCMAKE_INSTALL_LIBDIR=lib - -Dtest=OFF - -Dbuild_static_lib=ON) - - if(USE_JFROG_CACHE STREQUAL "ON") - set(EASYLOGGINGPP_CACHE_PACKAGE_NAME "easyloggingpp_${EASYLOGGINGPP_MD5}.tar.gz") - set(EASYLOGGINGPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${EASYLOGGINGPP_CACHE_PACKAGE_NAME}") - set(EASYLOGGINGPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${EASYLOGGINGPP_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${EASYLOGGINGPP_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote cache file ${EASYLOGGINGPP_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(easyloggingpp_ep - URL - ${EASYLOGGINGPP_SOURCE_URL} - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${EASYLOGGINGPP_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - ${EASYLOGGINGPP_STATIC_LIB}) - - ExternalProject_Create_Cache(easyloggingpp_ep ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/easyloggingpp_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${EASYLOGGINGPP_CACHE_URL}) - else() - file(DOWNLOAD ${EASYLOGGINGPP_CACHE_URL} ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${EASYLOGGINGPP_CACHE_URL} TO ${EASYLOGGINGPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(easyloggingpp_ep ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() - externalproject_add(easyloggingpp_ep - URL - ${EASYLOGGINGPP_SOURCE_URL} - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${EASYLOGGINGPP_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - ${EASYLOGGINGPP_STATIC_LIB}) - endif() - - file(MAKE_DIRECTORY "${EASYLOGGINGPP_INCLUDE_DIR}") - add_library(easyloggingpp STATIC IMPORTED) - set_target_properties( - easyloggingpp - PROPERTIES IMPORTED_LOCATION "${EASYLOGGINGPP_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${EASYLOGGINGPP_INCLUDE_DIR}") - - add_dependencies(easyloggingpp easyloggingpp_ep) -endmacro() - -if(MILVUS_WITH_EASYLOGGINGPP) - resolve_dependency(Easylogging++) - - get_target_property(EASYLOGGINGPP_INCLUDE_DIR easyloggingpp INTERFACE_INCLUDE_DIRECTORIES) - link_directories(SYSTEM "${EASYLOGGINGPP_PREFIX}/lib") - include_directories(SYSTEM "${EASYLOGGINGPP_INCLUDE_DIR}") -endif() +#macro(build_easyloggingpp) +# message(STATUS "Building Easylogging++-${EASYLOGGINGPP_VERSION} from source") +# set(EASYLOGGINGPP_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/easyloggingpp_ep-prefix/src/easyloggingpp_ep") +# set(EASYLOGGINGPP_INCLUDE_DIR "${EASYLOGGINGPP_PREFIX}/include") +# set(EASYLOGGINGPP_STATIC_LIB +# "${EASYLOGGINGPP_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}easyloggingpp${CMAKE_STATIC_LIBRARY_SUFFIX}") +# +# set(EASYLOGGINGPP_CMAKE_ARGS +# ${EP_COMMON_CMAKE_ARGS} +# "-DCMAKE_INSTALL_PREFIX=${EASYLOGGINGPP_PREFIX}" +# -DCMAKE_INSTALL_LIBDIR=lib +# -Dtest=OFF +# -Dbuild_static_lib=ON) +# +# if(USE_JFROG_CACHE STREQUAL "ON") +# set(EASYLOGGINGPP_CACHE_PACKAGE_NAME "easyloggingpp_${EASYLOGGINGPP_MD5}.tar.gz") +# set(EASYLOGGINGPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${EASYLOGGINGPP_CACHE_PACKAGE_NAME}") +# set(EASYLOGGINGPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${EASYLOGGINGPP_CACHE_PACKAGE_NAME}") +# +# execute_process(COMMAND wget -q --method HEAD ${EASYLOGGINGPP_CACHE_URL} RESULT_VARIABLE return_code) +# message(STATUS "Check the remote cache file ${EASYLOGGINGPP_CACHE_URL}. return code = ${return_code}") +# if (NOT return_code EQUAL 0) +# externalproject_add(easyloggingpp_ep +# URL +# ${EASYLOGGINGPP_SOURCE_URL} +# ${EP_LOG_OPTIONS} +# CMAKE_ARGS +# ${EASYLOGGINGPP_CMAKE_ARGS} +# BUILD_COMMAND +# ${MAKE} +# ${MAKE_BUILD_ARGS} +# BUILD_BYPRODUCTS +# ${EASYLOGGINGPP_STATIC_LIB}) +# +# ExternalProject_Create_Cache(easyloggingpp_ep ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/easyloggingpp_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${EASYLOGGINGPP_CACHE_URL}) +# else() +# file(DOWNLOAD ${EASYLOGGINGPP_CACHE_URL} ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} STATUS status) +# list(GET status 0 status_code) +# message(STATUS "DOWNLOADING FROM ${EASYLOGGINGPP_CACHE_URL} TO ${EASYLOGGINGPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") +# if (status_code EQUAL 0) +# ExternalProject_Use_Cache(easyloggingpp_ep ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) +# endif() +# endif() +# else() +# externalproject_add(easyloggingpp_ep +# URL +# ${EASYLOGGINGPP_SOURCE_URL} +# ${EP_LOG_OPTIONS} +# CMAKE_ARGS +# ${EASYLOGGINGPP_CMAKE_ARGS} +# BUILD_COMMAND +# ${MAKE} +# ${MAKE_BUILD_ARGS} +# BUILD_BYPRODUCTS +# ${EASYLOGGINGPP_STATIC_LIB}) +# endif() +# +# file(MAKE_DIRECTORY "${EASYLOGGINGPP_INCLUDE_DIR}") +# add_library(easyloggingpp STATIC IMPORTED) +# set_target_properties( +# easyloggingpp +# PROPERTIES IMPORTED_LOCATION "${EASYLOGGINGPP_STATIC_LIB}" +# INTERFACE_INCLUDE_DIRECTORIES "${EASYLOGGINGPP_INCLUDE_DIR}") +# +# add_dependencies(easyloggingpp easyloggingpp_ep) +#endmacro() +# +#if(MILVUS_WITH_EASYLOGGINGPP) +# resolve_dependency(Easylogging++) +# +# get_target_property(EASYLOGGINGPP_INCLUDE_DIR easyloggingpp INTERFACE_INCLUDE_DIRECTORIES) +# link_directories(SYSTEM "${EASYLOGGINGPP_PREFIX}/lib") +# include_directories(SYSTEM "${EASYLOGGINGPP_INCLUDE_DIR}") +#endif() # ---------------------------------------------------------------------- # OpenBLAS diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 387693a30c..733c493b59 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -73,6 +73,7 @@ set(db_files ${db_scheduler_files} ${metrics_files} ${knowhere_files} + ${utils_files} ) set(s3_client_files @@ -94,7 +95,7 @@ set(client_grpc_lib set(third_party_libs knowhere - easyloggingpp +# easyloggingpp sqlite ${client_grpc_lib} yaml-cpp @@ -145,7 +146,7 @@ target_link_libraries(milvus_engine ${engine_libs} ${third_party_libs}) add_library(metrics STATIC ${metrics_files}) set(metrics_lib - easyloggingpp +# easyloggingpp yaml-cpp prometheus-cpp-push prometheus-cpp-pull diff --git a/cpp/src/db/Log.h b/cpp/src/db/Log.h index ac46a043aa..f36d861c3e 100644 --- a/cpp/src/db/Log.h +++ b/cpp/src/db/Log.h @@ -5,7 +5,7 @@ ******************************************************************************/ #pragma once -#include +#include "utils/easylogging++.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index 287f9f074d..ecebd30bdb 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -5,7 +5,7 @@ ******************************************************************************/ #include #include -#include +#include "utils/easylogging++.h" #include #include "Options.h" diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index 8590da27be..e31e286652 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -11,7 +11,7 @@ #include #include #include -#include +#include "utils/easylogging++.h" #include "metrics/Metrics.h" #include "utils/SignalUtil.h" diff --git a/cpp/src/utils/Log.h b/cpp/src/utils/Log.h index 62c58d5fa1..204d5812b6 100644 --- a/cpp/src/utils/Log.h +++ b/cpp/src/utils/Log.h @@ -6,7 +6,7 @@ #pragma once #include "Error.h" -#include +#include "easylogging++.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp index 98a9596da8..5f0573cce8 100644 --- a/cpp/src/utils/LogUtil.cpp +++ b/cpp/src/utils/LogUtil.cpp @@ -5,8 +5,8 @@ //////////////////////////////////////////////////////////////////////////////// #include "LogUtil.h" #include "server/ServerConfig.h" +#include "easylogging++.h" -#include #include #include @@ -27,7 +27,7 @@ static int fatal_idx = 0; } // TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename -void RolloutHandler(const char *filename, std::size_t size) { +void RolloutHandler(const char *filename, std::size_t size, el::Level level) { char *dirc = strdup(filename); char *basec = strdup(filename); char *dir = dirname(dirc); @@ -48,25 +48,31 @@ void RolloutHandler(const char *filename, std::size_t size) { int ret; std::string m(std::string(dir) + "/" + s); s = m; - if ((position = s.find("global")) != std::string::npos) { + if(level == el::Level::Global){ s.append("." + std::to_string(++global_idx)); ret = rename(m.c_str(), s.c_str()); - } else if ((position = s.find("debug")) != std::string::npos) { + } + else if(level == el::Level::Debug){ s.append("." + std::to_string(++debug_idx)); ret = rename(m.c_str(), s.c_str()); - } else if ((position = s.find("warning")) != std::string::npos) { + } + else if(level == el::Level::Warning){ s.append("." + std::to_string(++warning_idx)); ret = rename(m.c_str(), s.c_str()); - } else if ((position = s.find("trace")) != std::string::npos) { + } + else if(level == el::Level::Trace){ s.append("." + std::to_string(++trace_idx)); ret = rename(m.c_str(), s.c_str()); - } else if ((position = s.find("error")) != std::string::npos) { + } + else if(level == el::Level::Error){ s.append("." + std::to_string(++error_idx)); ret = rename(m.c_str(), s.c_str()); - } else if ((position = s.find("fatal")) != std::string::npos) { + } + else if(level == el::Level::Fatal){ s.append("." + std::to_string(++fatal_idx)); ret = rename(m.c_str(), s.c_str()); - } else { + } + else{ s.append("." + std::to_string(++global_idx)); ret = rename(m.c_str(), s.c_str()); } @@ -113,6 +119,7 @@ int32_t InitLog(const std::string &log_config_file) { el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck); el::Helpers::installPreRollOutCallback(RolloutHandler); + el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog); return 0; } diff --git a/cpp/src/utils/LogUtil.h b/cpp/src/utils/LogUtil.h index 7c8d936a6d..e2cb011ce7 100644 --- a/cpp/src/utils/LogUtil.h +++ b/cpp/src/utils/LogUtil.h @@ -7,6 +7,7 @@ #include #include +#include "easylogging++.h" namespace zilliz { namespace milvus { @@ -16,7 +17,7 @@ inline std::string GetFileName(std::string filename) { int pos = filename.find_last_of('/'); return filename.substr(pos + 1); } -void RolloutHandler(const char *filename, std::size_t size); +void RolloutHandler(const char *filename, std::size_t size, el::Level level); #define SHOW_LOCATION #ifdef SHOW_LOCATION diff --git a/cpp/src/utils/easylogging++.cc b/cpp/src/utils/easylogging++.cc new file mode 100644 index 0000000000..0c3bb0d375 --- /dev/null +++ b/cpp/src/utils/easylogging++.cc @@ -0,0 +1,3112 @@ +// +// Bismillah ar-Rahmaan ar-Raheem +// +// Easylogging++ v9.96.7 +// Cross-platform logging library for C++ applications +// +// Copyright (c) 2012-2018 Zuhd Web Services +// Copyright (c) 2012-2018 @abumusamq +// +// This library is released under the MIT Licence. +// https://github.com/zuhd-org/easyloggingpp/blob/master/LICENSE +// +// https://zuhd.org +// http://muflihun.com +// + +#include "easylogging++.h" + +#if defined(AUTO_INITIALIZE_EASYLOGGINGPP) +INITIALIZE_EASYLOGGINGPP +#endif + +namespace el { + +// el::base +namespace base { +// el::base::consts +namespace consts { + +// Level log values - These are values that are replaced in place of %level format specifier +// Extra spaces after format specifiers are only for readability purposes in log files +static const base::type::char_t* kInfoLevelLogValue = ELPP_LITERAL("INFO"); +static const base::type::char_t* kDebugLevelLogValue = ELPP_LITERAL("DEBUG"); +static const base::type::char_t* kWarningLevelLogValue = ELPP_LITERAL("WARNING"); +static const base::type::char_t* kErrorLevelLogValue = ELPP_LITERAL("ERROR"); +static const base::type::char_t* kFatalLevelLogValue = ELPP_LITERAL("FATAL"); +static const base::type::char_t* kVerboseLevelLogValue = + ELPP_LITERAL("VERBOSE"); // will become VERBOSE-x where x = verbose level +static const base::type::char_t* kTraceLevelLogValue = ELPP_LITERAL("TRACE"); +static const base::type::char_t* kInfoLevelShortLogValue = ELPP_LITERAL("I"); +static const base::type::char_t* kDebugLevelShortLogValue = ELPP_LITERAL("D"); +static const base::type::char_t* kWarningLevelShortLogValue = ELPP_LITERAL("W"); +static const base::type::char_t* kErrorLevelShortLogValue = ELPP_LITERAL("E"); +static const base::type::char_t* kFatalLevelShortLogValue = ELPP_LITERAL("F"); +static const base::type::char_t* kVerboseLevelShortLogValue = ELPP_LITERAL("V"); +static const base::type::char_t* kTraceLevelShortLogValue = ELPP_LITERAL("T"); +// Format specifiers - These are used to define log format +static const base::type::char_t* kAppNameFormatSpecifier = ELPP_LITERAL("%app"); +static const base::type::char_t* kLoggerIdFormatSpecifier = ELPP_LITERAL("%logger"); +static const base::type::char_t* kThreadIdFormatSpecifier = ELPP_LITERAL("%thread"); +static const base::type::char_t* kSeverityLevelFormatSpecifier = ELPP_LITERAL("%level"); +static const base::type::char_t* kSeverityLevelShortFormatSpecifier = ELPP_LITERAL("%levshort"); +static const base::type::char_t* kDateTimeFormatSpecifier = ELPP_LITERAL("%datetime"); +static const base::type::char_t* kLogFileFormatSpecifier = ELPP_LITERAL("%file"); +static const base::type::char_t* kLogFileBaseFormatSpecifier = ELPP_LITERAL("%fbase"); +static const base::type::char_t* kLogLineFormatSpecifier = ELPP_LITERAL("%line"); +static const base::type::char_t* kLogLocationFormatSpecifier = ELPP_LITERAL("%loc"); +static const base::type::char_t* kLogFunctionFormatSpecifier = ELPP_LITERAL("%func"); +static const base::type::char_t* kCurrentUserFormatSpecifier = ELPP_LITERAL("%user"); +static const base::type::char_t* kCurrentHostFormatSpecifier = ELPP_LITERAL("%host"); +static const base::type::char_t* kMessageFormatSpecifier = ELPP_LITERAL("%msg"); +static const base::type::char_t* kVerboseLevelFormatSpecifier = ELPP_LITERAL("%vlevel"); +static const char* kDateTimeFormatSpecifierForFilename = "%datetime"; +// Date/time +static const char* kDays[7] = { "Sundayaaa", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday" }; +static const char* kDaysAbbrev[7] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" }; +static const char* kMonths[12] = { "January", "February", "March", "Apri", "May", "June", "July", "August", + "September", "October", "November", "December" + }; +static const char* kMonthsAbbrev[12] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; +static const char* kDefaultDateTimeFormat = "%Y-%M-%d %H:%m:%s,%g"; +static const char* kDefaultDateTimeFormatInFilename = "%Y-%M-%d_%H-%m"; +static const int kYearBase = 1900; +static const char* kAm = "AM"; +static const char* kPm = "PM"; +// Miscellaneous constants + +static const char* kNullPointer = "nullptr"; +#if ELPP_VARIADIC_TEMPLATES_SUPPORTED +#endif // ELPP_VARIADIC_TEMPLATES_SUPPORTED +static const base::type::VerboseLevel kMaxVerboseLevel = 9; +static const char* kUnknownUser = "user"; +static const char* kUnknownHost = "unknown-host"; + + +//---------------- DEFAULT LOG FILE ----------------------- + +#if defined(ELPP_NO_DEFAULT_LOG_FILE) +# if ELPP_OS_UNIX +static const char* kDefaultLogFile = "/dev/null"; +# elif ELPP_OS_WINDOWS +static const char* kDefaultLogFile = "nul"; +# endif // ELPP_OS_UNIX +#elif defined(ELPP_DEFAULT_LOG_FILE) +static const char* kDefaultLogFile = ELPP_DEFAULT_LOG_FILE; +#else +static const char* kDefaultLogFile = "myeasylog.log"; +#endif // defined(ELPP_NO_DEFAULT_LOG_FILE) + + +#if !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) +static const char* kDefaultLogFileParam = "--default-log-file"; +#endif // !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) +#if defined(ELPP_LOGGING_FLAGS_FROM_ARG) +static const char* kLoggingFlagsParam = "--logging-flags"; +#endif // defined(ELPP_LOGGING_FLAGS_FROM_ARG) +static const char* kValidLoggerIdSymbols = + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._"; +static const char* kConfigurationComment = "##"; +static const char* kConfigurationLevel = "*"; +static const char* kConfigurationLoggerId = "--"; +} +// el::base::utils +namespace utils { + +/// @brief Aborts application due with user-defined status +static void abort(int status, const std::string& reason) { + // Both status and reason params are there for debugging with tools like gdb etc + ELPP_UNUSED(status); + ELPP_UNUSED(reason); +#if defined(ELPP_COMPILER_MSVC) && defined(_M_IX86) && defined(_DEBUG) + // Ignore msvc critical error dialog - break instead (on debug mode) + _asm int 3 +#else + ::abort(); +#endif // defined(ELPP_COMPILER_MSVC) && defined(_M_IX86) && defined(_DEBUG) +} + +} // namespace utils +} // namespace base + +// el + +// LevelHelper + +const char* LevelHelper::convertToString(Level level) { + // Do not use switch over strongly typed enums because Intel C++ compilers dont support them yet. + if (level == Level::Global) return "GLOBAL"; + if (level == Level::Debug) return "DEBUG"; + if (level == Level::Info) return "INFO"; + if (level == Level::Warning) return "WARNING"; + if (level == Level::Error) return "ERROR"; + if (level == Level::Fatal) return "FATAL"; + if (level == Level::Verbose) return "VERBOSE"; + if (level == Level::Trace) return "TRACE"; + return "UNKNOWN"; +} + +struct StringToLevelItem { + const char* levelString; + Level level; +}; + +static struct StringToLevelItem stringToLevelMap[] = { + { "global", Level::Global }, + { "debug", Level::Debug }, + { "info", Level::Info }, + { "warning", Level::Warning }, + { "error", Level::Error }, + { "fatal", Level::Fatal }, + { "verbose", Level::Verbose }, + { "trace", Level::Trace } +}; + +Level LevelHelper::convertFromString(const char* levelStr) { + for (auto& item : stringToLevelMap) { + if (base::utils::Str::cStringCaseEq(levelStr, item.levelString)) { + return item.level; + } + } + return Level::Unknown; +} + +void LevelHelper::forEachLevel(base::type::EnumType* startIndex, const std::function& fn) { + base::type::EnumType lIndexMax = LevelHelper::kMaxValid; + do { + if (fn()) { + break; + } + *startIndex = static_cast(*startIndex << 1); + } while (*startIndex <= lIndexMax); +} + +// ConfigurationTypeHelper + +const char* ConfigurationTypeHelper::convertToString(ConfigurationType configurationType) { + // Do not use switch over strongly typed enums because Intel C++ compilers dont support them yet. + if (configurationType == ConfigurationType::Enabled) return "ENABLED"; + if (configurationType == ConfigurationType::Filename) return "FILENAME"; + if (configurationType == ConfigurationType::Format) return "FORMAT"; + if (configurationType == ConfigurationType::ToFile) return "TO_FILE"; + if (configurationType == ConfigurationType::ToStandardOutput) return "TO_STANDARD_OUTPUT"; + if (configurationType == ConfigurationType::SubsecondPrecision) return "SUBSECOND_PRECISION"; + if (configurationType == ConfigurationType::PerformanceTracking) return "PERFORMANCE_TRACKING"; + if (configurationType == ConfigurationType::MaxLogFileSize) return "MAX_LOG_FILE_SIZE"; + if (configurationType == ConfigurationType::LogFlushThreshold) return "LOG_FLUSH_THRESHOLD"; + return "UNKNOWN"; +} + +struct ConfigurationStringToTypeItem { + const char* configString; + ConfigurationType configType; +}; + +static struct ConfigurationStringToTypeItem configStringToTypeMap[] = { + { "enabled", ConfigurationType::Enabled }, + { "to_file", ConfigurationType::ToFile }, + { "to_standard_output", ConfigurationType::ToStandardOutput }, + { "format", ConfigurationType::Format }, + { "filename", ConfigurationType::Filename }, + { "subsecond_precision", ConfigurationType::SubsecondPrecision }, + { "milliseconds_width", ConfigurationType::MillisecondsWidth }, + { "performance_tracking", ConfigurationType::PerformanceTracking }, + { "max_log_file_size", ConfigurationType::MaxLogFileSize }, + { "log_flush_threshold", ConfigurationType::LogFlushThreshold }, +}; + +ConfigurationType ConfigurationTypeHelper::convertFromString(const char* configStr) { + for (auto& item : configStringToTypeMap) { + if (base::utils::Str::cStringCaseEq(configStr, item.configString)) { + return item.configType; + } + } + return ConfigurationType::Unknown; +} + +void ConfigurationTypeHelper::forEachConfigType(base::type::EnumType* startIndex, const std::function& fn) { + base::type::EnumType cIndexMax = ConfigurationTypeHelper::kMaxValid; + do { + if (fn()) { + break; + } + *startIndex = static_cast(*startIndex << 1); + } while (*startIndex <= cIndexMax); +} + +// Configuration + +Configuration::Configuration(const Configuration& c) : + m_level(c.m_level), + m_configurationType(c.m_configurationType), + m_value(c.m_value) { +} + +Configuration& Configuration::operator=(const Configuration& c) { + if (&c != this) { + m_level = c.m_level; + m_configurationType = c.m_configurationType; + m_value = c.m_value; + } + return *this; +} + +/// @brief Full constructor used to sets value of configuration +Configuration::Configuration(Level level, ConfigurationType configurationType, const std::string& value) : + m_level(level), + m_configurationType(configurationType), + m_value(value) { +} + +void Configuration::log(el::base::type::ostream_t& os) const { + os << LevelHelper::convertToString(m_level) + << ELPP_LITERAL(" ") << ConfigurationTypeHelper::convertToString(m_configurationType) + << ELPP_LITERAL(" = ") << m_value.c_str(); +} + +/// @brief Used to find configuration from configuration (pointers) repository. Avoid using it. +Configuration::Predicate::Predicate(Level level, ConfigurationType configurationType) : + m_level(level), + m_configurationType(configurationType) { +} + +bool Configuration::Predicate::operator()(const Configuration* conf) const { + return ((conf != nullptr) && (conf->level() == m_level) && (conf->configurationType() == m_configurationType)); +} + +// Configurations + +Configurations::Configurations(void) : + m_configurationFile(std::string()), + m_isFromFile(false) { +} + +Configurations::Configurations(const std::string& configurationFile, bool useDefaultsForRemaining, + Configurations* base) : + m_configurationFile(configurationFile), + m_isFromFile(false) { + parseFromFile(configurationFile, base); + if (useDefaultsForRemaining) { + setRemainingToDefault(); + } +} + +bool Configurations::parseFromFile(const std::string& configurationFile, Configurations* base) { + // We initial assertion with true because if we have assertion diabled, we want to pass this + // check and if assertion is enabled we will have values re-assigned any way. + bool assertionPassed = true; + ELPP_ASSERT((assertionPassed = base::utils::File::pathExists(configurationFile.c_str(), true)) == true, + "Configuration file [" << configurationFile << "] does not exist!"); + if (!assertionPassed) { + return false; + } + bool success = Parser::parseFromFile(configurationFile, this, base); + m_isFromFile = success; + return success; +} + +bool Configurations::parseFromText(const std::string& configurationsString, Configurations* base) { + bool success = Parser::parseFromText(configurationsString, this, base); + if (success) { + m_isFromFile = false; + } + return success; +} + +void Configurations::setFromBase(Configurations* base) { + if (base == nullptr || base == this) { + return; + } + base::threading::ScopedLock scopedLock(base->lock()); + for (Configuration*& conf : base->list()) { + set(conf); + } +} + +bool Configurations::hasConfiguration(ConfigurationType configurationType) { + base::type::EnumType lIndex = LevelHelper::kMinValid; + bool result = false; + LevelHelper::forEachLevel(&lIndex, [&](void) -> bool { + if (hasConfiguration(LevelHelper::castFromInt(lIndex), configurationType)) { + result = true; + } + return result; + }); + return result; +} + +bool Configurations::hasConfiguration(Level level, ConfigurationType configurationType) { + base::threading::ScopedLock scopedLock(lock()); +#if ELPP_COMPILER_INTEL + // We cant specify template types here, Intel C++ throws compilation error + // "error: type name is not allowed" + return RegistryWithPred::get(level, configurationType) != nullptr; +#else + return RegistryWithPred::get(level, configurationType) != nullptr; +#endif // ELPP_COMPILER_INTEL +} + +void Configurations::set(Level level, ConfigurationType configurationType, const std::string& value) { + base::threading::ScopedLock scopedLock(lock()); + unsafeSet(level, configurationType, value); // This is not unsafe anymore as we have locked mutex + if (level == Level::Global) { + unsafeSetGlobally(configurationType, value, false); // Again this is not unsafe either + } +} + +void Configurations::set(Configuration* conf) { + if (conf == nullptr) { + return; + } + set(conf->level(), conf->configurationType(), conf->value()); +} + +void Configurations::setToDefault(void) { + setGlobally(ConfigurationType::Enabled, std::string("true"), true); + setGlobally(ConfigurationType::Filename, std::string(base::consts::kDefaultLogFile), true); +#if defined(ELPP_NO_LOG_TO_FILE) + setGlobally(ConfigurationType::ToFile, std::string("false"), true); +#else + setGlobally(ConfigurationType::ToFile, std::string("true"), true); +#endif // defined(ELPP_NO_LOG_TO_FILE) + setGlobally(ConfigurationType::ToStandardOutput, std::string("true"), true); + setGlobally(ConfigurationType::SubsecondPrecision, std::string("3"), true); + setGlobally(ConfigurationType::PerformanceTracking, std::string("true"), true); + setGlobally(ConfigurationType::MaxLogFileSize, std::string("0"), true); + setGlobally(ConfigurationType::LogFlushThreshold, std::string("0"), true); + + setGlobally(ConfigurationType::Format, std::string("%datetime %level [%logger] %msg"), true); + set(Level::Debug, ConfigurationType::Format, + std::string("%datetime %level [%logger] [%user@%host] [%func] [%loc] %msg")); + // INFO and WARNING are set to default by Level::Global + set(Level::Error, ConfigurationType::Format, std::string("%datetime %level [%logger] %msg")); + set(Level::Fatal, ConfigurationType::Format, std::string("%datetime %level [%logger] %msg")); + set(Level::Verbose, ConfigurationType::Format, std::string("%datetime %level-%vlevel [%logger] %msg")); + set(Level::Trace, ConfigurationType::Format, std::string("%datetime %level [%logger] [%func] [%loc] %msg")); +} + +void Configurations::setRemainingToDefault(void) { + base::threading::ScopedLock scopedLock(lock()); +#if defined(ELPP_NO_LOG_TO_FILE) + unsafeSetIfNotExist(Level::Global, ConfigurationType::Enabled, std::string("false")); +#else + unsafeSetIfNotExist(Level::Global, ConfigurationType::Enabled, std::string("true")); +#endif // defined(ELPP_NO_LOG_TO_FILE) + unsafeSetIfNotExist(Level::Global, ConfigurationType::Filename, std::string(base::consts::kDefaultLogFile)); + unsafeSetIfNotExist(Level::Global, ConfigurationType::ToStandardOutput, std::string("true")); + unsafeSetIfNotExist(Level::Global, ConfigurationType::SubsecondPrecision, std::string("3")); + unsafeSetIfNotExist(Level::Global, ConfigurationType::PerformanceTracking, std::string("true")); + unsafeSetIfNotExist(Level::Global, ConfigurationType::MaxLogFileSize, std::string("0")); + unsafeSetIfNotExist(Level::Global, ConfigurationType::Format, std::string("%datetime %level [%logger] %msg")); + unsafeSetIfNotExist(Level::Debug, ConfigurationType::Format, + std::string("%datetime %level [%logger] [%user@%host] [%func] [%loc] %msg")); + // INFO and WARNING are set to default by Level::Global + unsafeSetIfNotExist(Level::Error, ConfigurationType::Format, std::string("%datetime %level [%logger] %msg")); + unsafeSetIfNotExist(Level::Fatal, ConfigurationType::Format, std::string("%datetime %level [%logger] %msg")); + unsafeSetIfNotExist(Level::Verbose, ConfigurationType::Format, std::string("%datetime %level-%vlevel [%logger] %msg")); + unsafeSetIfNotExist(Level::Trace, ConfigurationType::Format, + std::string("%datetime %level [%logger] [%func] [%loc] %msg")); +} + +bool Configurations::Parser::parseFromFile(const std::string& configurationFile, Configurations* sender, + Configurations* base) { + sender->setFromBase(base); + std::ifstream fileStream_(configurationFile.c_str(), std::ifstream::in); + ELPP_ASSERT(fileStream_.is_open(), "Unable to open configuration file [" << configurationFile << "] for parsing."); + bool parsedSuccessfully = false; + std::string line = std::string(); + Level currLevel = Level::Unknown; + std::string currConfigStr = std::string(); + std::string currLevelStr = std::string(); + while (fileStream_.good()) { + std::getline(fileStream_, line); + parsedSuccessfully = parseLine(&line, &currConfigStr, &currLevelStr, &currLevel, sender); + ELPP_ASSERT(parsedSuccessfully, "Unable to parse configuration line: " << line); + } + return parsedSuccessfully; +} + +bool Configurations::Parser::parseFromText(const std::string& configurationsString, Configurations* sender, + Configurations* base) { + sender->setFromBase(base); + bool parsedSuccessfully = false; + std::stringstream ss(configurationsString); + std::string line = std::string(); + Level currLevel = Level::Unknown; + std::string currConfigStr = std::string(); + std::string currLevelStr = std::string(); + while (std::getline(ss, line)) { + parsedSuccessfully = parseLine(&line, &currConfigStr, &currLevelStr, &currLevel, sender); + ELPP_ASSERT(parsedSuccessfully, "Unable to parse configuration line: " << line); + } + return parsedSuccessfully; +} + +void Configurations::Parser::ignoreComments(std::string* line) { + std::size_t foundAt = 0; + std::size_t quotesStart = line->find("\""); + std::size_t quotesEnd = std::string::npos; + if (quotesStart != std::string::npos) { + quotesEnd = line->find("\"", quotesStart + 1); + while (quotesEnd != std::string::npos && line->at(quotesEnd - 1) == '\\') { + // Do not erase slash yet - we will erase it in parseLine(..) while loop + quotesEnd = line->find("\"", quotesEnd + 2); + } + } + if ((foundAt = line->find(base::consts::kConfigurationComment)) != std::string::npos) { + if (foundAt < quotesEnd) { + foundAt = line->find(base::consts::kConfigurationComment, quotesEnd + 1); + } + *line = line->substr(0, foundAt); + } +} + +bool Configurations::Parser::isLevel(const std::string& line) { + return base::utils::Str::startsWith(line, std::string(base::consts::kConfigurationLevel)); +} + +bool Configurations::Parser::isComment(const std::string& line) { + return base::utils::Str::startsWith(line, std::string(base::consts::kConfigurationComment)); +} + +bool Configurations::Parser::isConfig(const std::string& line) { + std::size_t assignment = line.find('='); + return line != "" && + ((line[0] >= 'A' && line[0] <= 'Z') || (line[0] >= 'a' && line[0] <= 'z')) && + (assignment != std::string::npos) && + (line.size() > assignment); +} + +bool Configurations::Parser::parseLine(std::string* line, std::string* currConfigStr, std::string* currLevelStr, + Level* currLevel, + Configurations* conf) { + ConfigurationType currConfig = ConfigurationType::Unknown; + std::string currValue = std::string(); + *line = base::utils::Str::trim(*line); + if (isComment(*line)) return true; + ignoreComments(line); + *line = base::utils::Str::trim(*line); + if (line->empty()) { + // Comment ignored + return true; + } + if (isLevel(*line)) { + if (line->size() <= 2) { + return true; + } + *currLevelStr = line->substr(1, line->size() - 2); + *currLevelStr = base::utils::Str::toUpper(*currLevelStr); + *currLevelStr = base::utils::Str::trim(*currLevelStr); + *currLevel = LevelHelper::convertFromString(currLevelStr->c_str()); + return true; + } + if (isConfig(*line)) { + std::size_t assignment = line->find('='); + *currConfigStr = line->substr(0, assignment); + *currConfigStr = base::utils::Str::toUpper(*currConfigStr); + *currConfigStr = base::utils::Str::trim(*currConfigStr); + currConfig = ConfigurationTypeHelper::convertFromString(currConfigStr->c_str()); + currValue = line->substr(assignment + 1); + currValue = base::utils::Str::trim(currValue); + std::size_t quotesStart = currValue.find("\"", 0); + std::size_t quotesEnd = std::string::npos; + if (quotesStart != std::string::npos) { + quotesEnd = currValue.find("\"", quotesStart + 1); + while (quotesEnd != std::string::npos && currValue.at(quotesEnd - 1) == '\\') { + currValue = currValue.erase(quotesEnd - 1, 1); + quotesEnd = currValue.find("\"", quotesEnd + 2); + } + } + if (quotesStart != std::string::npos && quotesEnd != std::string::npos) { + // Quote provided - check and strip if valid + ELPP_ASSERT((quotesStart < quotesEnd), "Configuration error - No ending quote found in [" + << currConfigStr << "]"); + ELPP_ASSERT((quotesStart + 1 != quotesEnd), "Empty configuration value for [" << currConfigStr << "]"); + if ((quotesStart != quotesEnd) && (quotesStart + 1 != quotesEnd)) { + // Explicit check in case if assertion is disabled + currValue = currValue.substr(quotesStart + 1, quotesEnd - 1); + } + } + } + ELPP_ASSERT(*currLevel != Level::Unknown, "Unrecognized severity level [" << *currLevelStr << "]"); + ELPP_ASSERT(currConfig != ConfigurationType::Unknown, "Unrecognized configuration [" << *currConfigStr << "]"); + if (*currLevel == Level::Unknown || currConfig == ConfigurationType::Unknown) { + return false; // unrecognizable level or config + } + conf->set(*currLevel, currConfig, currValue); + return true; +} + +void Configurations::unsafeSetIfNotExist(Level level, ConfigurationType configurationType, const std::string& value) { + Configuration* conf = RegistryWithPred::get(level, configurationType); + if (conf == nullptr) { + unsafeSet(level, configurationType, value); + } +} + +void Configurations::unsafeSet(Level level, ConfigurationType configurationType, const std::string& value) { + Configuration* conf = RegistryWithPred::get(level, configurationType); + if (conf == nullptr) { + registerNew(new Configuration(level, configurationType, value)); + } else { + conf->setValue(value); + } + if (level == Level::Global) { + unsafeSetGlobally(configurationType, value, false); + } +} + +void Configurations::setGlobally(ConfigurationType configurationType, const std::string& value, + bool includeGlobalLevel) { + if (includeGlobalLevel) { + set(Level::Global, configurationType, value); + } + base::type::EnumType lIndex = LevelHelper::kMinValid; + LevelHelper::forEachLevel(&lIndex, [&](void) -> bool { + set(LevelHelper::castFromInt(lIndex), configurationType, value); + return false; // Do not break lambda function yet as we need to set all levels regardless + }); +} + +void Configurations::unsafeSetGlobally(ConfigurationType configurationType, const std::string& value, + bool includeGlobalLevel) { + if (includeGlobalLevel) { + unsafeSet(Level::Global, configurationType, value); + } + base::type::EnumType lIndex = LevelHelper::kMinValid; + LevelHelper::forEachLevel(&lIndex, [&](void) -> bool { + unsafeSet(LevelHelper::castFromInt(lIndex), configurationType, value); + return false; // Do not break lambda function yet as we need to set all levels regardless + }); +} + +// LogBuilder + +void LogBuilder::convertToColoredOutput(base::type::string_t* logLine, Level level) { + if (!m_termSupportsColor) return; + const base::type::char_t* resetColor = ELPP_LITERAL("\x1b[0m"); + if (level == Level::Error || level == Level::Fatal) + *logLine = ELPP_LITERAL("\x1b[31m") + *logLine + resetColor; + else if (level == Level::Warning) + *logLine = ELPP_LITERAL("\x1b[33m") + *logLine + resetColor; + else if (level == Level::Debug) + *logLine = ELPP_LITERAL("\x1b[32m") + *logLine + resetColor; + else if (level == Level::Info) + *logLine = ELPP_LITERAL("\x1b[36m") + *logLine + resetColor; + else if (level == Level::Trace) + *logLine = ELPP_LITERAL("\x1b[35m") + *logLine + resetColor; +} + +// Logger + +Logger::Logger(const std::string& id, base::LogStreamsReferenceMap* logStreamsReference) : + m_id(id), + m_typedConfigurations(nullptr), + m_parentApplicationName(std::string()), + m_isConfigured(false), + m_logStreamsReference(logStreamsReference) { + initUnflushedCount(); +} + +Logger::Logger(const std::string& id, const Configurations& configurations, + base::LogStreamsReferenceMap* logStreamsReference) : + m_id(id), + m_typedConfigurations(nullptr), + m_parentApplicationName(std::string()), + m_isConfigured(false), + m_logStreamsReference(logStreamsReference) { + initUnflushedCount(); + configure(configurations); +} + +Logger::Logger(const Logger& logger) { + base::utils::safeDelete(m_typedConfigurations); + m_id = logger.m_id; + m_typedConfigurations = logger.m_typedConfigurations; + m_parentApplicationName = logger.m_parentApplicationName; + m_isConfigured = logger.m_isConfigured; + m_configurations = logger.m_configurations; + m_unflushedCount = logger.m_unflushedCount; + m_logStreamsReference = logger.m_logStreamsReference; +} + +Logger& Logger::operator=(const Logger& logger) { + if (&logger != this) { + base::utils::safeDelete(m_typedConfigurations); + m_id = logger.m_id; + m_typedConfigurations = logger.m_typedConfigurations; + m_parentApplicationName = logger.m_parentApplicationName; + m_isConfigured = logger.m_isConfigured; + m_configurations = logger.m_configurations; + m_unflushedCount = logger.m_unflushedCount; + m_logStreamsReference = logger.m_logStreamsReference; + } + return *this; +} + +void Logger::configure(const Configurations& configurations) { + m_isConfigured = false; // we set it to false in case if we fail + initUnflushedCount(); + if (m_typedConfigurations != nullptr) { + Configurations* c = const_cast(m_typedConfigurations->configurations()); + if (c->hasConfiguration(Level::Global, ConfigurationType::Filename)) { + flush(); + } + } + base::threading::ScopedLock scopedLock(lock()); + if (m_configurations != configurations) { + m_configurations.setFromBase(const_cast(&configurations)); + } + base::utils::safeDelete(m_typedConfigurations); + m_typedConfigurations = new base::TypedConfigurations(&m_configurations, m_logStreamsReference); + resolveLoggerFormatSpec(); + m_isConfigured = true; +} + +void Logger::reconfigure(void) { + ELPP_INTERNAL_INFO(1, "Reconfiguring logger [" << m_id << "]"); + configure(m_configurations); +} + +bool Logger::isValidId(const std::string& id) { + for (std::string::const_iterator it = id.begin(); it != id.end(); ++it) { + if (!base::utils::Str::contains(base::consts::kValidLoggerIdSymbols, *it)) { + return false; + } + } + return true; +} + +void Logger::flush(void) { + ELPP_INTERNAL_INFO(3, "Flushing logger [" << m_id << "] all levels"); + base::threading::ScopedLock scopedLock(lock()); + base::type::EnumType lIndex = LevelHelper::kMinValid; + LevelHelper::forEachLevel(&lIndex, [&](void) -> bool { + flush(LevelHelper::castFromInt(lIndex), nullptr); + return false; + }); +} + +void Logger::flush(Level level, base::type::fstream_t* fs) { + if (fs == nullptr && m_typedConfigurations->toFile(level)) { + fs = m_typedConfigurations->fileStream(level); + } + if (fs != nullptr) { + fs->flush(); + std::unordered_map::iterator iter = m_unflushedCount.find(level); + if (iter != m_unflushedCount.end()) { + iter->second = 0; + } + Helpers::validateFileRolling(this, level); + } +} + +void Logger::initUnflushedCount(void) { + m_unflushedCount.clear(); + base::type::EnumType lIndex = LevelHelper::kMinValid; + LevelHelper::forEachLevel(&lIndex, [&](void) -> bool { + m_unflushedCount.insert(std::make_pair(LevelHelper::castFromInt(lIndex), 0)); + return false; + }); +} + +void Logger::resolveLoggerFormatSpec(void) const { + base::type::EnumType lIndex = LevelHelper::kMinValid; + LevelHelper::forEachLevel(&lIndex, [&](void) -> bool { + base::LogFormat* logFormat = + const_cast(&m_typedConfigurations->logFormat(LevelHelper::castFromInt(lIndex))); + base::utils::Str::replaceFirstWithEscape(logFormat->m_format, base::consts::kLoggerIdFormatSpecifier, m_id); + return false; + }); +} + +// el::base +namespace base { + +// el::base::utils +namespace utils { + +// File + +base::type::fstream_t* File::newFileStream(const std::string& filename) { + base::type::fstream_t *fs = new base::type::fstream_t(filename.c_str(), + base::type::fstream_t::out +#if !defined(ELPP_FRESH_LOG_FILE) + | base::type::fstream_t::app +#endif + ); +#if defined(ELPP_UNICODE) + std::locale elppUnicodeLocale(""); +# if ELPP_OS_WINDOWS + std::locale elppUnicodeLocaleWindows(elppUnicodeLocale, new std::codecvt_utf8_utf16); + elppUnicodeLocale = elppUnicodeLocaleWindows; +# endif // ELPP_OS_WINDOWS + fs->imbue(elppUnicodeLocale); +#endif // defined(ELPP_UNICODE) + if (fs->is_open()) { + fs->flush(); + } else { + base::utils::safeDelete(fs); + ELPP_INTERNAL_ERROR("Bad file [" << filename << "]", true); + } + return fs; +} + +std::size_t File::getSizeOfFile(base::type::fstream_t* fs) { + if (fs == nullptr) { + return 0; + } + // Since the file stream is appended to or truncated, the current + // offset is the file size. + std::size_t size = static_cast(fs->tellg()); + return size; +} + +bool File::pathExists(const char* path, bool considerFile) { + if (path == nullptr) { + return false; + } +#if ELPP_OS_UNIX + ELPP_UNUSED(considerFile); + struct stat st; + return (stat(path, &st) == 0); +#elif ELPP_OS_WINDOWS + DWORD fileType = GetFileAttributesA(path); + if (fileType == INVALID_FILE_ATTRIBUTES) { + return false; + } + return considerFile ? true : ((fileType & FILE_ATTRIBUTE_DIRECTORY) == 0 ? false : true); +#endif // ELPP_OS_UNIX +} + +bool File::createPath(const std::string& path) { + if (path.empty()) { + return false; + } + if (base::utils::File::pathExists(path.c_str())) { + return true; + } + int status = -1; + + char* currPath = const_cast(path.c_str()); + std::string builtPath = std::string(); +#if ELPP_OS_UNIX + if (path[0] == '/') { + builtPath = "/"; + } + currPath = STRTOK(currPath, base::consts::kFilePathSeperator, 0); +#elif ELPP_OS_WINDOWS + // Use secure functions API + char* nextTok_ = nullptr; + currPath = STRTOK(currPath, base::consts::kFilePathSeperator, &nextTok_); + ELPP_UNUSED(nextTok_); +#endif // ELPP_OS_UNIX + while (currPath != nullptr) { + builtPath.append(currPath); + builtPath.append(base::consts::kFilePathSeperator); +#if ELPP_OS_UNIX + status = mkdir(builtPath.c_str(), ELPP_LOG_PERMS); + currPath = STRTOK(nullptr, base::consts::kFilePathSeperator, 0); +#elif ELPP_OS_WINDOWS + status = _mkdir(builtPath.c_str()); + currPath = STRTOK(nullptr, base::consts::kFilePathSeperator, &nextTok_); +#endif // ELPP_OS_UNIX + } + if (status == -1) { + ELPP_INTERNAL_ERROR("Error while creating path [" << path << "]", true); + return false; + } + return true; +} + +std::string File::extractPathFromFilename(const std::string& fullPath, const char* separator) { + if ((fullPath == "") || (fullPath.find(separator) == std::string::npos)) { + return fullPath; + } + std::size_t lastSlashAt = fullPath.find_last_of(separator); + if (lastSlashAt == 0) { + return std::string(separator); + } + return fullPath.substr(0, lastSlashAt + 1); +} + +void File::buildStrippedFilename(const char* filename, char buff[], std::size_t limit) { + std::size_t sizeOfFilename = strlen(filename); + if (sizeOfFilename >= limit) { + filename += (sizeOfFilename - limit); + if (filename[0] != '.' && filename[1] != '.') { // prepend if not already + filename += 3; // 3 = '..' + STRCAT(buff, "..", limit); + } + } + STRCAT(buff, filename, limit); +} + +void File::buildBaseFilename(const std::string& fullPath, char buff[], std::size_t limit, const char* separator) { + const char *filename = fullPath.c_str(); + std::size_t lastSlashAt = fullPath.find_last_of(separator); + filename += lastSlashAt ? lastSlashAt+1 : 0; + std::size_t sizeOfFilename = strlen(filename); + if (sizeOfFilename >= limit) { + filename += (sizeOfFilename - limit); + if (filename[0] != '.' && filename[1] != '.') { // prepend if not already + filename += 3; // 3 = '..' + STRCAT(buff, "..", limit); + } + } + STRCAT(buff, filename, limit); +} + +// Str + +bool Str::wildCardMatch(const char* str, const char* pattern) { + while (*pattern) { + switch (*pattern) { + case '?': + if (!*str) + return false; + ++str; + ++pattern; + break; + case '*': + if (wildCardMatch(str, pattern + 1)) + return true; + if (*str && wildCardMatch(str + 1, pattern)) + return true; + return false; + default: + if (*str++ != *pattern++) + return false; + break; + } + } + return !*str && !*pattern; +} + +std::string& Str::ltrim(std::string& str) { + str.erase(str.begin(), std::find_if(str.begin(), str.end(), [](char c) { + return !std::isspace(c); + } )); + return str; +} + +std::string& Str::rtrim(std::string& str) { + str.erase(std::find_if(str.rbegin(), str.rend(), [](char c) { + return !std::isspace(c); + }).base(), str.end()); + return str; +} + +std::string& Str::trim(std::string& str) { + return ltrim(rtrim(str)); +} + +bool Str::startsWith(const std::string& str, const std::string& start) { + return (str.length() >= start.length()) && (str.compare(0, start.length(), start) == 0); +} + +bool Str::endsWith(const std::string& str, const std::string& end) { + return (str.length() >= end.length()) && (str.compare(str.length() - end.length(), end.length(), end) == 0); +} + +std::string& Str::replaceAll(std::string& str, char replaceWhat, char replaceWith) { + std::replace(str.begin(), str.end(), replaceWhat, replaceWith); + return str; +} + +std::string& Str::replaceAll(std::string& str, const std::string& replaceWhat, + const std::string& replaceWith) { + if (replaceWhat == replaceWith) + return str; + std::size_t foundAt = std::string::npos; + while ((foundAt = str.find(replaceWhat, foundAt + 1)) != std::string::npos) { + str.replace(foundAt, replaceWhat.length(), replaceWith); + } + return str; +} + +void Str::replaceFirstWithEscape(base::type::string_t& str, const base::type::string_t& replaceWhat, + const base::type::string_t& replaceWith) { + std::size_t foundAt = base::type::string_t::npos; + while ((foundAt = str.find(replaceWhat, foundAt + 1)) != base::type::string_t::npos) { + if (foundAt > 0 && str[foundAt - 1] == base::consts::kFormatSpecifierChar) { + str.erase(foundAt - 1, 1); + ++foundAt; + } else { + str.replace(foundAt, replaceWhat.length(), replaceWith); + return; + } + } +} +#if defined(ELPP_UNICODE) +void Str::replaceFirstWithEscape(base::type::string_t& str, const base::type::string_t& replaceWhat, + const std::string& replaceWith) { + replaceFirstWithEscape(str, replaceWhat, base::type::string_t(replaceWith.begin(), replaceWith.end())); +} +#endif // defined(ELPP_UNICODE) + +std::string& Str::toUpper(std::string& str) { + std::transform(str.begin(), str.end(), str.begin(), + [](char c) { + return static_cast(::toupper(c)); + }); + return str; +} + +bool Str::cStringEq(const char* s1, const char* s2) { + if (s1 == nullptr && s2 == nullptr) return true; + if (s1 == nullptr || s2 == nullptr) return false; + return strcmp(s1, s2) == 0; +} + +bool Str::cStringCaseEq(const char* s1, const char* s2) { + if (s1 == nullptr && s2 == nullptr) return true; + if (s1 == nullptr || s2 == nullptr) return false; + + // With thanks to cygwin for this code + int d = 0; + + while (true) { + const int c1 = toupper(*s1++); + const int c2 = toupper(*s2++); + + if (((d = c1 - c2) != 0) || (c2 == '\0')) { + break; + } + } + + return d == 0; +} + +bool Str::contains(const char* str, char c) { + for (; *str; ++str) { + if (*str == c) + return true; + } + return false; +} + +char* Str::convertAndAddToBuff(std::size_t n, int len, char* buf, const char* bufLim, bool zeroPadded) { + char localBuff[10] = ""; + char* p = localBuff + sizeof(localBuff) - 2; + if (n > 0) { + for (; n > 0 && p > localBuff && len > 0; n /= 10, --len) + *--p = static_cast(n % 10 + '0'); + } else { + *--p = '0'; + --len; + } + if (zeroPadded) + while (p > localBuff && len-- > 0) *--p = static_cast('0'); + return addToBuff(p, buf, bufLim); +} + +char* Str::addToBuff(const char* str, char* buf, const char* bufLim) { + while ((buf < bufLim) && ((*buf = *str++) != '\0')) + ++buf; + return buf; +} + +char* Str::clearBuff(char buff[], std::size_t lim) { + STRCPY(buff, "", lim); + ELPP_UNUSED(lim); // For *nix we dont have anything using lim in above STRCPY macro + return buff; +} + +/// @brief Converst wchar* to char* +/// NOTE: Need to free return value after use! +char* Str::wcharPtrToCharPtr(const wchar_t* line) { + std::size_t len_ = wcslen(line) + 1; + char* buff_ = static_cast(malloc(len_ + 1)); +# if ELPP_OS_UNIX || (ELPP_OS_WINDOWS && !ELPP_CRT_DBG_WARNINGS) + std::wcstombs(buff_, line, len_); +# elif ELPP_OS_WINDOWS + std::size_t convCount_ = 0; + mbstate_t mbState_; + ::memset(static_cast(&mbState_), 0, sizeof(mbState_)); + wcsrtombs_s(&convCount_, buff_, len_, &line, len_, &mbState_); +# endif // ELPP_OS_UNIX || (ELPP_OS_WINDOWS && !ELPP_CRT_DBG_WARNINGS) + return buff_; +} + +// OS + +#if ELPP_OS_WINDOWS +/// @brief Gets environment variables for Windows based OS. +/// We are not using getenv(const char*) because of CRT deprecation +/// @param varname Variable name to get environment variable value for +/// @return If variable exist the value of it otherwise nullptr +const char* OS::getWindowsEnvironmentVariable(const char* varname) { + const DWORD bufferLen = 50; + static char buffer[bufferLen]; + if (GetEnvironmentVariableA(varname, buffer, bufferLen)) { + return buffer; + } + return nullptr; +} +#endif // ELPP_OS_WINDOWS +#if ELPP_OS_ANDROID +std::string OS::getProperty(const char* prop) { + char propVal[PROP_VALUE_MAX + 1]; + int ret = __system_property_get(prop, propVal); + return ret == 0 ? std::string() : std::string(propVal); +} + +std::string OS::getDeviceName(void) { + std::stringstream ss; + std::string manufacturer = getProperty("ro.product.manufacturer"); + std::string model = getProperty("ro.product.model"); + if (manufacturer.empty() || model.empty()) { + return std::string(); + } + ss << manufacturer << "-" << model; + return ss.str(); +} +#endif // ELPP_OS_ANDROID + +const std::string OS::getBashOutput(const char* command) { +#if (ELPP_OS_UNIX && !ELPP_OS_ANDROID && !ELPP_CYGWIN) + if (command == nullptr) { + return std::string(); + } + FILE* proc = nullptr; + if ((proc = popen(command, "r")) == nullptr) { + ELPP_INTERNAL_ERROR("\nUnable to run command [" << command << "]", true); + return std::string(); + } + char hBuff[4096]; + if (fgets(hBuff, sizeof(hBuff), proc) != nullptr) { + pclose(proc); + const std::size_t buffLen = strlen(hBuff); + if (buffLen > 0 && hBuff[buffLen - 1] == '\n') { + hBuff[buffLen - 1] = '\0'; + } + return std::string(hBuff); + } else { + pclose(proc); + } + return std::string(); +#else + ELPP_UNUSED(command); + return std::string(); +#endif // (ELPP_OS_UNIX && !ELPP_OS_ANDROID && !ELPP_CYGWIN) +} + +std::string OS::getEnvironmentVariable(const char* variableName, const char* defaultVal, + const char* alternativeBashCommand) { +#if ELPP_OS_UNIX + const char* val = getenv(variableName); +#elif ELPP_OS_WINDOWS + const char* val = getWindowsEnvironmentVariable(variableName); +#endif // ELPP_OS_UNIX + if ((val == nullptr) || ((strcmp(val, "") == 0))) { +#if ELPP_OS_UNIX && defined(ELPP_FORCE_ENV_VAR_FROM_BASH) + // Try harder on unix-based systems + std::string valBash = base::utils::OS::getBashOutput(alternativeBashCommand); + if (valBash.empty()) { + return std::string(defaultVal); + } else { + return valBash; + } +#elif ELPP_OS_WINDOWS || ELPP_OS_UNIX + ELPP_UNUSED(alternativeBashCommand); + return std::string(defaultVal); +#endif // ELPP_OS_UNIX && defined(ELPP_FORCE_ENV_VAR_FROM_BASH) + } + return std::string(val); +} + +std::string OS::currentUser(void) { +#if ELPP_OS_UNIX && !ELPP_OS_ANDROID + return getEnvironmentVariable("USER", base::consts::kUnknownUser, "whoami"); +#elif ELPP_OS_WINDOWS + return getEnvironmentVariable("USERNAME", base::consts::kUnknownUser); +#elif ELPP_OS_ANDROID + ELPP_UNUSED(base::consts::kUnknownUser); + return std::string("android"); +#else + return std::string(); +#endif // ELPP_OS_UNIX && !ELPP_OS_ANDROID +} + +std::string OS::currentHost(void) { +#if ELPP_OS_UNIX && !ELPP_OS_ANDROID + return getEnvironmentVariable("HOSTNAME", base::consts::kUnknownHost, "hostname"); +#elif ELPP_OS_WINDOWS + return getEnvironmentVariable("COMPUTERNAME", base::consts::kUnknownHost); +#elif ELPP_OS_ANDROID + ELPP_UNUSED(base::consts::kUnknownHost); + return getDeviceName(); +#else + return std::string(); +#endif // ELPP_OS_UNIX && !ELPP_OS_ANDROID +} + +bool OS::termSupportsColor(void) { + std::string term = getEnvironmentVariable("TERM", ""); + return term == "xterm" || term == "xterm-color" || term == "xterm-256color" + || term == "screen" || term == "linux" || term == "cygwin" + || term == "screen-256color"; +} + +// DateTime + +void DateTime::gettimeofday(struct timeval* tv) { +#if ELPP_OS_WINDOWS + if (tv != nullptr) { +# if ELPP_COMPILER_MSVC || defined(_MSC_EXTENSIONS) + const unsigned __int64 delta_ = 11644473600000000Ui64; +# else + const unsigned __int64 delta_ = 11644473600000000ULL; +# endif // ELPP_COMPILER_MSVC || defined(_MSC_EXTENSIONS) + const double secOffSet = 0.000001; + const unsigned long usecOffSet = 1000000; + FILETIME fileTime; + GetSystemTimeAsFileTime(&fileTime); + unsigned __int64 present = 0; + present |= fileTime.dwHighDateTime; + present = present << 32; + present |= fileTime.dwLowDateTime; + present /= 10; // mic-sec + // Subtract the difference + present -= delta_; + tv->tv_sec = static_cast(present * secOffSet); + tv->tv_usec = static_cast(present % usecOffSet); + } +#else + ::gettimeofday(tv, nullptr); +#endif // ELPP_OS_WINDOWS +} + +std::string DateTime::getDateTime(const char* format, const base::SubsecondPrecision* ssPrec) { + struct timeval currTime; + gettimeofday(&currTime); + return timevalToString(currTime, format, ssPrec); +} + +std::string DateTime::timevalToString(struct timeval tval, const char* format, + const el::base::SubsecondPrecision* ssPrec) { + struct ::tm timeInfo; + buildTimeInfo(&tval, &timeInfo); + const int kBuffSize = 30; + char buff_[kBuffSize] = ""; + parseFormat(buff_, kBuffSize, format, &timeInfo, static_cast(tval.tv_usec / ssPrec->m_offset), + ssPrec); + return std::string(buff_); +} + +base::type::string_t DateTime::formatTime(unsigned long long time, base::TimestampUnit timestampUnit) { + base::type::EnumType start = static_cast(timestampUnit); + const base::type::char_t* unit = base::consts::kTimeFormats[start].unit; + for (base::type::EnumType i = start; i < base::consts::kTimeFormatsCount - 1; ++i) { + if (time <= base::consts::kTimeFormats[i].value) { + break; + } + if (base::consts::kTimeFormats[i].value == 1000.0f && time / 1000.0f < 1.9f) { + break; + } + time /= static_cast(base::consts::kTimeFormats[i].value); + unit = base::consts::kTimeFormats[i + 1].unit; + } + base::type::stringstream_t ss; + ss << time << " " << unit; + return ss.str(); +} + +unsigned long long DateTime::getTimeDifference(const struct timeval& endTime, const struct timeval& startTime, + base::TimestampUnit timestampUnit) { + if (timestampUnit == base::TimestampUnit::Microsecond) { + return static_cast(static_cast(1000000 * endTime.tv_sec + endTime.tv_usec) - + static_cast(1000000 * startTime.tv_sec + startTime.tv_usec)); + } + // milliseconds + auto conv = [](const struct timeval& tim) { + return static_cast((tim.tv_sec * 1000) + (tim.tv_usec / 1000)); + }; + return static_cast(conv(endTime) - conv(startTime)); +} + +struct ::tm* DateTime::buildTimeInfo(struct timeval* currTime, struct ::tm* timeInfo) { +#if ELPP_OS_UNIX + time_t rawTime = currTime->tv_sec; + ::elpptime_r(&rawTime, timeInfo); + return timeInfo; +#else +# if ELPP_COMPILER_MSVC + ELPP_UNUSED(currTime); + time_t t; +# if defined(_USE_32BIT_TIME_T) + _time32(&t); +# else + _time64(&t); +# endif + elpptime_s(timeInfo, &t); + return timeInfo; +# else + // For any other compilers that don't have CRT warnings issue e.g, MinGW or TDM GCC- we use different method + time_t rawTime = currTime->tv_sec; + struct tm* tmInf = elpptime(&rawTime); + *timeInfo = *tmInf; + return timeInfo; +# endif // ELPP_COMPILER_MSVC +#endif // ELPP_OS_UNIX +} + +char* DateTime::parseFormat(char* buf, std::size_t bufSz, const char* format, const struct tm* tInfo, + std::size_t msec, const base::SubsecondPrecision* ssPrec) { + const char* bufLim = buf + bufSz; + for (; *format; ++format) { + if (*format == base::consts::kFormatSpecifierChar) { + switch (*++format) { + case base::consts::kFormatSpecifierChar: // Escape + break; + case '\0': // End + --format; + break; + case 'd': // Day + buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_mday, 2, buf, bufLim); + continue; + case 'a': // Day of week (short) + buf = base::utils::Str::addToBuff(base::consts::kDaysAbbrev[tInfo->tm_wday], buf, bufLim); + continue; + case 'A': // Day of week (long) + buf = base::utils::Str::addToBuff(base::consts::kDays[tInfo->tm_wday], buf, bufLim); + continue; + case 'M': // month + buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_mon + 1, 2, buf, bufLim); + continue; + case 'b': // month (short) + buf = base::utils::Str::addToBuff(base::consts::kMonthsAbbrev[tInfo->tm_mon], buf, bufLim); + continue; + case 'B': // month (long) + buf = base::utils::Str::addToBuff(base::consts::kMonths[tInfo->tm_mon], buf, bufLim); + continue; + case 'y': // year (two digits) + buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_year + base::consts::kYearBase, 2, buf, bufLim); + continue; + case 'Y': // year (four digits) + buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_year + base::consts::kYearBase, 4, buf, bufLim); + continue; + case 'h': // hour (12-hour) + buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_hour % 12, 2, buf, bufLim); + continue; + case 'H': // hour (24-hour) + buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_hour, 2, buf, bufLim); + continue; + case 'm': // minute + buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_min, 2, buf, bufLim); + continue; + case 's': // second + buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_sec, 2, buf, bufLim); + continue; + case 'z': // subsecond part + case 'g': + buf = base::utils::Str::convertAndAddToBuff(msec, ssPrec->m_width, buf, bufLim); + continue; + case 'F': // AM/PM + buf = base::utils::Str::addToBuff((tInfo->tm_hour >= 12) ? base::consts::kPm : base::consts::kAm, buf, bufLim); + continue; + default: + continue; + } + } + if (buf == bufLim) break; + *buf++ = *format; + } + return buf; +} + +// CommandLineArgs + +void CommandLineArgs::setArgs(int argc, char** argv) { + m_params.clear(); + m_paramsWithValue.clear(); + if (argc == 0 || argv == nullptr) { + return; + } + m_argc = argc; + m_argv = argv; + for (int i = 1; i < m_argc; ++i) { + const char* v = (strstr(m_argv[i], "=")); + if (v != nullptr && strlen(v) > 0) { + std::string key = std::string(m_argv[i]); + key = key.substr(0, key.find_first_of('=')); + if (hasParamWithValue(key.c_str())) { + ELPP_INTERNAL_INFO(1, "Skipping [" << key << "] arg since it already has value [" + << getParamValue(key.c_str()) << "]"); + } else { + m_paramsWithValue.insert(std::make_pair(key, std::string(v + 1))); + } + } + if (v == nullptr) { + if (hasParam(m_argv[i])) { + ELPP_INTERNAL_INFO(1, "Skipping [" << m_argv[i] << "] arg since it already exists"); + } else { + m_params.push_back(std::string(m_argv[i])); + } + } + } +} + +bool CommandLineArgs::hasParamWithValue(const char* paramKey) const { + return m_paramsWithValue.find(std::string(paramKey)) != m_paramsWithValue.end(); +} + +const char* CommandLineArgs::getParamValue(const char* paramKey) const { + std::unordered_map::const_iterator iter = m_paramsWithValue.find(std::string(paramKey)); + return iter != m_paramsWithValue.end() ? iter->second.c_str() : ""; +} + +bool CommandLineArgs::hasParam(const char* paramKey) const { + return std::find(m_params.begin(), m_params.end(), std::string(paramKey)) != m_params.end(); +} + +bool CommandLineArgs::empty(void) const { + return m_params.empty() && m_paramsWithValue.empty(); +} + +std::size_t CommandLineArgs::size(void) const { + return m_params.size() + m_paramsWithValue.size(); +} + +base::type::ostream_t& operator<<(base::type::ostream_t& os, const CommandLineArgs& c) { + for (int i = 1; i < c.m_argc; ++i) { + os << ELPP_LITERAL("[") << c.m_argv[i] << ELPP_LITERAL("]"); + if (i < c.m_argc - 1) { + os << ELPP_LITERAL(" "); + } + } + return os; +} + +} // namespace utils + +// el::base::threading +namespace threading { + +#if ELPP_THREADING_ENABLED +# if ELPP_USE_STD_THREADING +# if ELPP_ASYNC_LOGGING +static void msleep(int ms) { + // Only when async logging enabled - this is because async is strict on compiler +# if defined(ELPP_NO_SLEEP_FOR) + usleep(ms * 1000); +# else + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); +# endif // defined(ELPP_NO_SLEEP_FOR) +} +# endif // ELPP_ASYNC_LOGGING +# endif // !ELPP_USE_STD_THREADING +#endif // ELPP_THREADING_ENABLED + +} // namespace threading + +// el::base + +// SubsecondPrecision + +void SubsecondPrecision::init(int width) { + if (width < 1 || width > 6) { + width = base::consts::kDefaultSubsecondPrecision; + } + m_width = width; + switch (m_width) { + case 3: + m_offset = 1000; + break; + case 4: + m_offset = 100; + break; + case 5: + m_offset = 10; + break; + case 6: + m_offset = 1; + break; + default: + m_offset = 1000; + break; + } +} + +// LogFormat + +LogFormat::LogFormat(void) : + m_level(Level::Unknown), + m_userFormat(base::type::string_t()), + m_format(base::type::string_t()), + m_dateTimeFormat(std::string()), + m_flags(0x0), + m_currentUser(base::utils::OS::currentUser()), + m_currentHost(base::utils::OS::currentHost()) { +} + +LogFormat::LogFormat(Level level, const base::type::string_t& format) + : m_level(level), m_userFormat(format), m_currentUser(base::utils::OS::currentUser()), + m_currentHost(base::utils::OS::currentHost()) { + parseFromFormat(m_userFormat); +} + +LogFormat::LogFormat(const LogFormat& logFormat): + m_level(logFormat.m_level), + m_userFormat(logFormat.m_userFormat), + m_format(logFormat.m_format), + m_dateTimeFormat(logFormat.m_dateTimeFormat), + m_flags(logFormat.m_flags), + m_currentUser(logFormat.m_currentUser), + m_currentHost(logFormat.m_currentHost) { +} + +LogFormat::LogFormat(LogFormat&& logFormat) { + m_level = std::move(logFormat.m_level); + m_userFormat = std::move(logFormat.m_userFormat); + m_format = std::move(logFormat.m_format); + m_dateTimeFormat = std::move(logFormat.m_dateTimeFormat); + m_flags = std::move(logFormat.m_flags); + m_currentUser = std::move(logFormat.m_currentUser); + m_currentHost = std::move(logFormat.m_currentHost); +} + +LogFormat& LogFormat::operator=(const LogFormat& logFormat) { + if (&logFormat != this) { + m_level = logFormat.m_level; + m_userFormat = logFormat.m_userFormat; + m_dateTimeFormat = logFormat.m_dateTimeFormat; + m_flags = logFormat.m_flags; + m_currentUser = logFormat.m_currentUser; + m_currentHost = logFormat.m_currentHost; + } + return *this; +} + +bool LogFormat::operator==(const LogFormat& other) { + return m_level == other.m_level && m_userFormat == other.m_userFormat && m_format == other.m_format && + m_dateTimeFormat == other.m_dateTimeFormat && m_flags == other.m_flags; +} + +/// @brief Updates format to be used while logging. +/// @param userFormat User provided format +void LogFormat::parseFromFormat(const base::type::string_t& userFormat) { + // We make copy because we will be changing the format + // i.e, removing user provided date format from original format + // and then storing it. + base::type::string_t formatCopy = userFormat; + m_flags = 0x0; + auto conditionalAddFlag = [&](const base::type::char_t* specifier, base::FormatFlags flag) { + std::size_t foundAt = base::type::string_t::npos; + while ((foundAt = formatCopy.find(specifier, foundAt + 1)) != base::type::string_t::npos) { + if (foundAt > 0 && formatCopy[foundAt - 1] == base::consts::kFormatSpecifierChar) { + if (hasFlag(flag)) { + // If we already have flag we remove the escape chars so that '%%' is turned to '%' + // even after specifier resolution - this is because we only replaceFirst specifier + formatCopy.erase(foundAt - 1, 1); + ++foundAt; + } + } else { + if (!hasFlag(flag)) addFlag(flag); + } + } + }; + conditionalAddFlag(base::consts::kAppNameFormatSpecifier, base::FormatFlags::AppName); + conditionalAddFlag(base::consts::kSeverityLevelFormatSpecifier, base::FormatFlags::Level); + conditionalAddFlag(base::consts::kSeverityLevelShortFormatSpecifier, base::FormatFlags::LevelShort); + conditionalAddFlag(base::consts::kLoggerIdFormatSpecifier, base::FormatFlags::LoggerId); + conditionalAddFlag(base::consts::kThreadIdFormatSpecifier, base::FormatFlags::ThreadId); + conditionalAddFlag(base::consts::kLogFileFormatSpecifier, base::FormatFlags::File); + conditionalAddFlag(base::consts::kLogFileBaseFormatSpecifier, base::FormatFlags::FileBase); + conditionalAddFlag(base::consts::kLogLineFormatSpecifier, base::FormatFlags::Line); + conditionalAddFlag(base::consts::kLogLocationFormatSpecifier, base::FormatFlags::Location); + conditionalAddFlag(base::consts::kLogFunctionFormatSpecifier, base::FormatFlags::Function); + conditionalAddFlag(base::consts::kCurrentUserFormatSpecifier, base::FormatFlags::User); + conditionalAddFlag(base::consts::kCurrentHostFormatSpecifier, base::FormatFlags::Host); + conditionalAddFlag(base::consts::kMessageFormatSpecifier, base::FormatFlags::LogMessage); + conditionalAddFlag(base::consts::kVerboseLevelFormatSpecifier, base::FormatFlags::VerboseLevel); + // For date/time we need to extract user's date format first + std::size_t dateIndex = std::string::npos; + if ((dateIndex = formatCopy.find(base::consts::kDateTimeFormatSpecifier)) != std::string::npos) { + while (dateIndex > 0 && formatCopy[dateIndex - 1] == base::consts::kFormatSpecifierChar) { + dateIndex = formatCopy.find(base::consts::kDateTimeFormatSpecifier, dateIndex + 1); + } + if (dateIndex != std::string::npos) { + addFlag(base::FormatFlags::DateTime); + updateDateFormat(dateIndex, formatCopy); + } + } + m_format = formatCopy; + updateFormatSpec(); +} + +void LogFormat::updateDateFormat(std::size_t index, base::type::string_t& currFormat) { + if (hasFlag(base::FormatFlags::DateTime)) { + index += ELPP_STRLEN(base::consts::kDateTimeFormatSpecifier); + } + const base::type::char_t* ptr = currFormat.c_str() + index; + if ((currFormat.size() > index) && (ptr[0] == '{')) { + // User has provided format for date/time + ++ptr; + int count = 1; // Start by 1 in order to remove starting brace + std::stringstream ss; + for (; *ptr; ++ptr, ++count) { + if (*ptr == '}') { + ++count; // In order to remove ending brace + break; + } + ss << static_cast(*ptr); + } + currFormat.erase(index, count); + m_dateTimeFormat = ss.str(); + } else { + // No format provided, use default + if (hasFlag(base::FormatFlags::DateTime)) { + m_dateTimeFormat = std::string(base::consts::kDefaultDateTimeFormat); + } + } +} + +void LogFormat::updateFormatSpec(void) { + // Do not use switch over strongly typed enums because Intel C++ compilers dont support them yet. + if (m_level == Level::Debug) { + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, + base::consts::kDebugLevelLogValue); + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, + base::consts::kDebugLevelShortLogValue); + } else if (m_level == Level::Info) { + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, + base::consts::kInfoLevelLogValue); + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, + base::consts::kInfoLevelShortLogValue); + } else if (m_level == Level::Warning) { + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, + base::consts::kWarningLevelLogValue); + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, + base::consts::kWarningLevelShortLogValue); + } else if (m_level == Level::Error) { + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, + base::consts::kErrorLevelLogValue); + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, + base::consts::kErrorLevelShortLogValue); + } else if (m_level == Level::Fatal) { + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, + base::consts::kFatalLevelLogValue); + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, + base::consts::kFatalLevelShortLogValue); + } else if (m_level == Level::Verbose) { + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, + base::consts::kVerboseLevelLogValue); + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, + base::consts::kVerboseLevelShortLogValue); + } else if (m_level == Level::Trace) { + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, + base::consts::kTraceLevelLogValue); + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, + base::consts::kTraceLevelShortLogValue); + } + if (hasFlag(base::FormatFlags::User)) { + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kCurrentUserFormatSpecifier, + m_currentUser); + } + if (hasFlag(base::FormatFlags::Host)) { + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kCurrentHostFormatSpecifier, + m_currentHost); + } + // Ignore Level::Global and Level::Unknown +} + +// TypedConfigurations + +TypedConfigurations::TypedConfigurations(Configurations* configurations, + base::LogStreamsReferenceMap* logStreamsReference) { + m_configurations = configurations; + m_logStreamsReference = logStreamsReference; + build(m_configurations); +} + +TypedConfigurations::TypedConfigurations(const TypedConfigurations& other) { + this->m_configurations = other.m_configurations; + this->m_logStreamsReference = other.m_logStreamsReference; + build(m_configurations); +} + +bool TypedConfigurations::enabled(Level level) { + return getConfigByVal(level, &m_enabledMap, "enabled"); +} + +bool TypedConfigurations::toFile(Level level) { + return getConfigByVal(level, &m_toFileMap, "toFile"); +} + +const std::string& TypedConfigurations::filename(Level level) { + return getConfigByRef(level, &m_filenameMap, "filename"); +} + +bool TypedConfigurations::toStandardOutput(Level level) { + return getConfigByVal(level, &m_toStandardOutputMap, "toStandardOutput"); +} + +const base::LogFormat& TypedConfigurations::logFormat(Level level) { + return getConfigByRef(level, &m_logFormatMap, "logFormat"); +} + +const base::SubsecondPrecision& TypedConfigurations::subsecondPrecision(Level level) { + return getConfigByRef(level, &m_subsecondPrecisionMap, "subsecondPrecision"); +} + +const base::MillisecondsWidth& TypedConfigurations::millisecondsWidth(Level level) { + return getConfigByRef(level, &m_subsecondPrecisionMap, "millisecondsWidth"); +} + +bool TypedConfigurations::performanceTracking(Level level) { + return getConfigByVal(level, &m_performanceTrackingMap, "performanceTracking"); +} + +base::type::fstream_t* TypedConfigurations::fileStream(Level level) { + return getConfigByRef(level, &m_fileStreamMap, "fileStream").get(); +} + +std::size_t TypedConfigurations::maxLogFileSize(Level level) { + return getConfigByVal(level, &m_maxLogFileSizeMap, "maxLogFileSize"); +} + +std::size_t TypedConfigurations::logFlushThreshold(Level level) { + return getConfigByVal(level, &m_logFlushThresholdMap, "logFlushThreshold"); +} + +void TypedConfigurations::build(Configurations* configurations) { + base::threading::ScopedLock scopedLock(lock()); + auto getBool = [] (std::string boolStr) -> bool { // Pass by value for trimming + base::utils::Str::trim(boolStr); + return (boolStr == "TRUE" || boolStr == "true" || boolStr == "1"); + }; + std::vector withFileSizeLimit; + for (Configurations::const_iterator it = configurations->begin(); it != configurations->end(); ++it) { + Configuration* conf = *it; + // We cannot use switch on strong enums because Intel C++ dont support them yet + if (conf->configurationType() == ConfigurationType::Enabled) { + setValue(conf->level(), getBool(conf->value()), &m_enabledMap); + } else if (conf->configurationType() == ConfigurationType::ToFile) { + setValue(conf->level(), getBool(conf->value()), &m_toFileMap); + } else if (conf->configurationType() == ConfigurationType::ToStandardOutput) { + setValue(conf->level(), getBool(conf->value()), &m_toStandardOutputMap); + } else if (conf->configurationType() == ConfigurationType::Filename) { + // We do not yet configure filename but we will configure in another + // loop. This is because if file cannot be created, we will force ToFile + // to be false. Because configuring logger is not necessarily performance + // sensative operation, we can live with another loop; (by the way this loop + // is not very heavy either) + } else if (conf->configurationType() == ConfigurationType::Format) { + setValue(conf->level(), base::LogFormat(conf->level(), + base::type::string_t(conf->value().begin(), conf->value().end())), &m_logFormatMap); + } else if (conf->configurationType() == ConfigurationType::SubsecondPrecision) { + setValue(Level::Global, + base::SubsecondPrecision(static_cast(getULong(conf->value()))), &m_subsecondPrecisionMap); + } else if (conf->configurationType() == ConfigurationType::PerformanceTracking) { + setValue(Level::Global, getBool(conf->value()), &m_performanceTrackingMap); + } else if (conf->configurationType() == ConfigurationType::MaxLogFileSize) { + auto v = getULong(conf->value()); + setValue(conf->level(), static_cast(v), &m_maxLogFileSizeMap); + if (v != 0) { + withFileSizeLimit.push_back(conf); + } + } else if (conf->configurationType() == ConfigurationType::LogFlushThreshold) { + setValue(conf->level(), static_cast(getULong(conf->value())), &m_logFlushThresholdMap); + } + } + // As mentioned earlier, we will now set filename configuration in separate loop to deal with non-existent files + for (Configurations::const_iterator it = configurations->begin(); it != configurations->end(); ++it) { + Configuration* conf = *it; + if (conf->configurationType() == ConfigurationType::Filename) { + insertFile(conf->level(), conf->value()); + } + } + for (std::vector::iterator conf = withFileSizeLimit.begin(); + conf != withFileSizeLimit.end(); ++conf) { + // This is not unsafe as mutex is locked in currect scope + unsafeValidateFileRolling((*conf)->level(), base::defaultPreRollOutCallback); + } +} + +unsigned long TypedConfigurations::getULong(std::string confVal) { + bool valid = true; + base::utils::Str::trim(confVal); + valid = !confVal.empty() && std::find_if(confVal.begin(), confVal.end(), + [](char c) { + return !base::utils::Str::isDigit(c); + }) == confVal.end(); + if (!valid) { + valid = false; + ELPP_ASSERT(valid, "Configuration value not a valid integer [" << confVal << "]"); + return 0; + } + return atol(confVal.c_str()); +} + +std::string TypedConfigurations::resolveFilename(const std::string& filename) { + std::string resultingFilename = filename; + std::size_t dateIndex = std::string::npos; + std::string dateTimeFormatSpecifierStr = std::string(base::consts::kDateTimeFormatSpecifierForFilename); + if ((dateIndex = resultingFilename.find(dateTimeFormatSpecifierStr.c_str())) != std::string::npos) { + while (dateIndex > 0 && resultingFilename[dateIndex - 1] == base::consts::kFormatSpecifierChar) { + dateIndex = resultingFilename.find(dateTimeFormatSpecifierStr.c_str(), dateIndex + 1); + } + if (dateIndex != std::string::npos) { + const char* ptr = resultingFilename.c_str() + dateIndex; + // Goto end of specifier + ptr += dateTimeFormatSpecifierStr.size(); + std::string fmt; + if ((resultingFilename.size() > dateIndex) && (ptr[0] == '{')) { + // User has provided format for date/time + ++ptr; + int count = 1; // Start by 1 in order to remove starting brace + std::stringstream ss; + for (; *ptr; ++ptr, ++count) { + if (*ptr == '}') { + ++count; // In order to remove ending brace + break; + } + ss << *ptr; + } + resultingFilename.erase(dateIndex + dateTimeFormatSpecifierStr.size(), count); + fmt = ss.str(); + } else { + fmt = std::string(base::consts::kDefaultDateTimeFormatInFilename); + } + base::SubsecondPrecision ssPrec(3); + std::string now = base::utils::DateTime::getDateTime(fmt.c_str(), &ssPrec); + base::utils::Str::replaceAll(now, '/', '-'); // Replace path element since we are dealing with filename + base::utils::Str::replaceAll(resultingFilename, dateTimeFormatSpecifierStr, now); + } + } + return resultingFilename; +} + +void TypedConfigurations::insertFile(Level level, const std::string& fullFilename) { + std::string resolvedFilename = resolveFilename(fullFilename); + if (resolvedFilename.empty()) { + std::cerr << "Could not load empty file for logging, please re-check your configurations for level [" + << LevelHelper::convertToString(level) << "]"; + } + std::string filePath = base::utils::File::extractPathFromFilename(resolvedFilename, base::consts::kFilePathSeperator); + if (filePath.size() < resolvedFilename.size()) { + base::utils::File::createPath(filePath); + } + auto create = [&](Level level) { + base::LogStreamsReferenceMap::iterator filestreamIter = m_logStreamsReference->find(resolvedFilename); + base::type::fstream_t* fs = nullptr; + if (filestreamIter == m_logStreamsReference->end()) { + // We need a completely new stream, nothing to share with + fs = base::utils::File::newFileStream(resolvedFilename); + m_filenameMap.insert(std::make_pair(level, resolvedFilename)); + m_fileStreamMap.insert(std::make_pair(level, base::FileStreamPtr(fs))); + m_logStreamsReference->insert(std::make_pair(resolvedFilename, base::FileStreamPtr(m_fileStreamMap.at(level)))); + } else { + // Woops! we have an existing one, share it! + m_filenameMap.insert(std::make_pair(level, filestreamIter->first)); + m_fileStreamMap.insert(std::make_pair(level, base::FileStreamPtr(filestreamIter->second))); + fs = filestreamIter->second.get(); + } + if (fs == nullptr) { + // We display bad file error from newFileStream() + ELPP_INTERNAL_ERROR("Setting [TO_FILE] of [" + << LevelHelper::convertToString(level) << "] to FALSE", false); + setValue(level, false, &m_toFileMap); + } + }; + // If we dont have file conf for any level, create it for Level::Global first + // otherwise create for specified level + create(m_filenameMap.empty() && m_fileStreamMap.empty() ? Level::Global : level); +} + +bool TypedConfigurations::unsafeValidateFileRolling(Level level, const PreRollOutCallback& preRollOutCallback) { + base::type::fstream_t* fs = unsafeGetConfigByRef(level, &m_fileStreamMap, "fileStream").get(); + if (fs == nullptr) { + return true; + } + std::size_t maxLogFileSize = unsafeGetConfigByVal(level, &m_maxLogFileSizeMap, "maxLogFileSize"); + std::size_t currFileSize = base::utils::File::getSizeOfFile(fs); + if (maxLogFileSize != 0 && currFileSize >= maxLogFileSize) { + std::string fname = unsafeGetConfigByRef(level, &m_filenameMap, "filename"); + ELPP_INTERNAL_INFO(1, "Truncating log file [" << fname << "] as a result of configurations for level [" + << LevelHelper::convertToString(level) << "]"); + fs->close(); + preRollOutCallback(fname.c_str(), currFileSize, level); + fs->open(fname, std::fstream::out | std::fstream::trunc); + return true; + } + return false; +} + +// RegisteredHitCounters + +bool RegisteredHitCounters::validateEveryN(const char* filename, base::type::LineNumber lineNumber, std::size_t n) { + base::threading::ScopedLock scopedLock(lock()); + base::HitCounter* counter = get(filename, lineNumber); + if (counter == nullptr) { + registerNew(counter = new base::HitCounter(filename, lineNumber)); + } + counter->validateHitCounts(n); + bool result = (n >= 1 && counter->hitCounts() != 0 && counter->hitCounts() % n == 0); + return result; +} + +/// @brief Validates counter for hits >= N, i.e, registers new if does not exist otherwise updates original one +/// @return True if validation resulted in triggering hit. Meaning logs should be written everytime true is returned +bool RegisteredHitCounters::validateAfterN(const char* filename, base::type::LineNumber lineNumber, std::size_t n) { + base::threading::ScopedLock scopedLock(lock()); + base::HitCounter* counter = get(filename, lineNumber); + if (counter == nullptr) { + registerNew(counter = new base::HitCounter(filename, lineNumber)); + } + // Do not use validateHitCounts here since we do not want to reset counter here + // Note the >= instead of > because we are incrementing + // after this check + if (counter->hitCounts() >= n) + return true; + counter->increment(); + return false; +} + +/// @brief Validates counter for hits are <= n, i.e, registers new if does not exist otherwise updates original one +/// @return True if validation resulted in triggering hit. Meaning logs should be written everytime true is returned +bool RegisteredHitCounters::validateNTimes(const char* filename, base::type::LineNumber lineNumber, std::size_t n) { + base::threading::ScopedLock scopedLock(lock()); + base::HitCounter* counter = get(filename, lineNumber); + if (counter == nullptr) { + registerNew(counter = new base::HitCounter(filename, lineNumber)); + } + counter->increment(); + // Do not use validateHitCounts here since we do not want to reset counter here + if (counter->hitCounts() <= n) + return true; + return false; +} + +// RegisteredLoggers + +RegisteredLoggers::RegisteredLoggers(const LogBuilderPtr& defaultLogBuilder) : + m_defaultLogBuilder(defaultLogBuilder) { + m_defaultConfigurations.setToDefault(); +} + +Logger* RegisteredLoggers::get(const std::string& id, bool forceCreation) { + base::threading::ScopedLock scopedLock(lock()); + Logger* logger_ = base::utils::Registry::get(id); + if (logger_ == nullptr && forceCreation) { + bool validId = Logger::isValidId(id); + if (!validId) { + ELPP_ASSERT(validId, "Invalid logger ID [" << id << "]. Not registering this logger."); + return nullptr; + } + logger_ = new Logger(id, m_defaultConfigurations, &m_logStreamsReference); + logger_->m_logBuilder = m_defaultLogBuilder; + registerNew(id, logger_); + LoggerRegistrationCallback* callback = nullptr; + for (const std::pair& h + : m_loggerRegistrationCallbacks) { + callback = h.second.get(); + if (callback != nullptr && callback->enabled()) { + callback->handle(logger_); + } + } + } + return logger_; +} + +bool RegisteredLoggers::remove(const std::string& id) { + if (id == base::consts::kDefaultLoggerId) { + return false; + } + // get has internal lock + Logger* logger = base::utils::Registry::get(id); + if (logger != nullptr) { + // unregister has internal lock + unregister(logger); + } + return true; +} + +void RegisteredLoggers::unsafeFlushAll(void) { + ELPP_INTERNAL_INFO(1, "Flushing all log files"); + for (base::LogStreamsReferenceMap::iterator it = m_logStreamsReference.begin(); + it != m_logStreamsReference.end(); ++it) { + if (it->second.get() == nullptr) continue; + it->second->flush(); + } +} + +// VRegistry + +VRegistry::VRegistry(base::type::VerboseLevel level, base::type::EnumType* pFlags) : m_level(level), m_pFlags(pFlags) { +} + +/// @brief Sets verbose level. Accepted range is 0-9 +void VRegistry::setLevel(base::type::VerboseLevel level) { + base::threading::ScopedLock scopedLock(lock()); + if (level > 9) + m_level = base::consts::kMaxVerboseLevel; + else + m_level = level; +} + +void VRegistry::setModules(const char* modules) { + base::threading::ScopedLock scopedLock(lock()); + auto addSuffix = [](std::stringstream& ss, const char* sfx, const char* prev) { + if (prev != nullptr && base::utils::Str::endsWith(ss.str(), std::string(prev))) { + std::string chr(ss.str().substr(0, ss.str().size() - strlen(prev))); + ss.str(std::string("")); + ss << chr; + } + if (base::utils::Str::endsWith(ss.str(), std::string(sfx))) { + std::string chr(ss.str().substr(0, ss.str().size() - strlen(sfx))); + ss.str(std::string("")); + ss << chr; + } + ss << sfx; + }; + auto insert = [&](std::stringstream& ss, base::type::VerboseLevel level) { + if (!base::utils::hasFlag(LoggingFlag::DisableVModulesExtensions, *m_pFlags)) { + addSuffix(ss, ".h", nullptr); + m_modules.insert(std::make_pair(ss.str(), level)); + addSuffix(ss, ".c", ".h"); + m_modules.insert(std::make_pair(ss.str(), level)); + addSuffix(ss, ".cpp", ".c"); + m_modules.insert(std::make_pair(ss.str(), level)); + addSuffix(ss, ".cc", ".cpp"); + m_modules.insert(std::make_pair(ss.str(), level)); + addSuffix(ss, ".cxx", ".cc"); + m_modules.insert(std::make_pair(ss.str(), level)); + addSuffix(ss, ".-inl.h", ".cxx"); + m_modules.insert(std::make_pair(ss.str(), level)); + addSuffix(ss, ".hxx", ".-inl.h"); + m_modules.insert(std::make_pair(ss.str(), level)); + addSuffix(ss, ".hpp", ".hxx"); + m_modules.insert(std::make_pair(ss.str(), level)); + addSuffix(ss, ".hh", ".hpp"); + } + m_modules.insert(std::make_pair(ss.str(), level)); + }; + bool isMod = true; + bool isLevel = false; + std::stringstream ss; + int level = -1; + for (; *modules; ++modules) { + switch (*modules) { + case '=': + isLevel = true; + isMod = false; + break; + case ',': + isLevel = false; + isMod = true; + if (!ss.str().empty() && level != -1) { + insert(ss, static_cast(level)); + ss.str(std::string("")); + level = -1; + } + break; + default: + if (isMod) { + ss << *modules; + } else if (isLevel) { + if (isdigit(*modules)) { + level = static_cast(*modules) - 48; + } + } + break; + } + } + if (!ss.str().empty() && level != -1) { + insert(ss, static_cast(level)); + } +} + +bool VRegistry::allowed(base::type::VerboseLevel vlevel, const char* file) { + base::threading::ScopedLock scopedLock(lock()); + if (m_modules.empty() || file == nullptr) { + return vlevel <= m_level; + } else { + char baseFilename[base::consts::kSourceFilenameMaxLength] = ""; + base::utils::File::buildBaseFilename(file, baseFilename); + std::unordered_map::iterator it = m_modules.begin(); + for (; it != m_modules.end(); ++it) { + if (base::utils::Str::wildCardMatch(baseFilename, it->first.c_str())) { + return vlevel <= it->second; + } + } + if (base::utils::hasFlag(LoggingFlag::AllowVerboseIfModuleNotSpecified, *m_pFlags)) { + return true; + } + return false; + } +} + +void VRegistry::setFromArgs(const base::utils::CommandLineArgs* commandLineArgs) { + if (commandLineArgs->hasParam("-v") || commandLineArgs->hasParam("--verbose") || + commandLineArgs->hasParam("-V") || commandLineArgs->hasParam("--VERBOSE")) { + setLevel(base::consts::kMaxVerboseLevel); + } else if (commandLineArgs->hasParamWithValue("--v")) { + setLevel(static_cast(atoi(commandLineArgs->getParamValue("--v")))); + } else if (commandLineArgs->hasParamWithValue("--V")) { + setLevel(static_cast(atoi(commandLineArgs->getParamValue("--V")))); + } else if ((commandLineArgs->hasParamWithValue("-vmodule")) && vModulesEnabled()) { + setModules(commandLineArgs->getParamValue("-vmodule")); + } else if (commandLineArgs->hasParamWithValue("-VMODULE") && vModulesEnabled()) { + setModules(commandLineArgs->getParamValue("-VMODULE")); + } +} + +#if !defined(ELPP_DEFAULT_LOGGING_FLAGS) +# define ELPP_DEFAULT_LOGGING_FLAGS 0x0 +#endif // !defined(ELPP_DEFAULT_LOGGING_FLAGS) +// Storage +#if ELPP_ASYNC_LOGGING +Storage::Storage(const LogBuilderPtr& defaultLogBuilder, base::IWorker* asyncDispatchWorker) : +#else +Storage::Storage(const LogBuilderPtr& defaultLogBuilder) : +#endif // ELPP_ASYNC_LOGGING + m_registeredHitCounters(new base::RegisteredHitCounters()), + m_registeredLoggers(new base::RegisteredLoggers(defaultLogBuilder)), + m_flags(ELPP_DEFAULT_LOGGING_FLAGS), + m_vRegistry(new base::VRegistry(0, &m_flags)), + +#if ELPP_ASYNC_LOGGING + m_asyncLogQueue(new base::AsyncLogQueue()), + m_asyncDispatchWorker(asyncDispatchWorker), +#endif // ELPP_ASYNC_LOGGING + + m_preRollOutCallback(base::defaultPreRollOutCallback) { + // Register default logger + m_registeredLoggers->get(std::string(base::consts::kDefaultLoggerId)); + // We register default logger anyway (worse case it's not going to register) just in case + m_registeredLoggers->get("default"); + +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + // Register performance logger and reconfigure format + Logger* performanceLogger = m_registeredLoggers->get(std::string(base::consts::kPerformanceLoggerId)); + m_registeredLoggers->get("performance"); + performanceLogger->configurations()->setGlobally(ConfigurationType::Format, std::string("%datetime %level %msg")); + performanceLogger->reconfigure(); +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + +#if defined(ELPP_SYSLOG) + // Register syslog logger and reconfigure format + Logger* sysLogLogger = m_registeredLoggers->get(std::string(base::consts::kSysLogLoggerId)); + sysLogLogger->configurations()->setGlobally(ConfigurationType::Format, std::string("%level: %msg")); + sysLogLogger->reconfigure(); +#endif // defined(ELPP_SYSLOG) + addFlag(LoggingFlag::AllowVerboseIfModuleNotSpecified); +#if ELPP_ASYNC_LOGGING + installLogDispatchCallback(std::string("AsyncLogDispatchCallback")); +#else + installLogDispatchCallback(std::string("DefaultLogDispatchCallback")); +#endif // ELPP_ASYNC_LOGGING +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + installPerformanceTrackingCallback + (std::string("DefaultPerformanceTrackingCallback")); +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + ELPP_INTERNAL_INFO(1, "Easylogging++ has been initialized"); +#if ELPP_ASYNC_LOGGING + m_asyncDispatchWorker->start(); +#endif // ELPP_ASYNC_LOGGING +} + +Storage::~Storage(void) { + ELPP_INTERNAL_INFO(4, "Destroying storage"); +#if ELPP_ASYNC_LOGGING + ELPP_INTERNAL_INFO(5, "Replacing log dispatch callback to synchronous"); + uninstallLogDispatchCallback(std::string("AsyncLogDispatchCallback")); + installLogDispatchCallback(std::string("DefaultLogDispatchCallback")); + ELPP_INTERNAL_INFO(5, "Destroying asyncDispatchWorker"); + base::utils::safeDelete(m_asyncDispatchWorker); + ELPP_INTERNAL_INFO(5, "Destroying asyncLogQueue"); + base::utils::safeDelete(m_asyncLogQueue); +#endif // ELPP_ASYNC_LOGGING + ELPP_INTERNAL_INFO(5, "Destroying registeredHitCounters"); + base::utils::safeDelete(m_registeredHitCounters); + ELPP_INTERNAL_INFO(5, "Destroying registeredLoggers"); + base::utils::safeDelete(m_registeredLoggers); + ELPP_INTERNAL_INFO(5, "Destroying vRegistry"); + base::utils::safeDelete(m_vRegistry); +} + +bool Storage::hasCustomFormatSpecifier(const char* formatSpecifier) { + base::threading::ScopedLock scopedLock(customFormatSpecifiersLock()); + return std::find(m_customFormatSpecifiers.begin(), m_customFormatSpecifiers.end(), + formatSpecifier) != m_customFormatSpecifiers.end(); +} + +void Storage::installCustomFormatSpecifier(const CustomFormatSpecifier& customFormatSpecifier) { + if (hasCustomFormatSpecifier(customFormatSpecifier.formatSpecifier())) { + return; + } + base::threading::ScopedLock scopedLock(customFormatSpecifiersLock()); + m_customFormatSpecifiers.push_back(customFormatSpecifier); +} + +bool Storage::uninstallCustomFormatSpecifier(const char* formatSpecifier) { + base::threading::ScopedLock scopedLock(customFormatSpecifiersLock()); + std::vector::iterator it = std::find(m_customFormatSpecifiers.begin(), + m_customFormatSpecifiers.end(), formatSpecifier); + if (it != m_customFormatSpecifiers.end() && strcmp(formatSpecifier, it->formatSpecifier()) == 0) { + m_customFormatSpecifiers.erase(it); + return true; + } + return false; +} + +void Storage::setApplicationArguments(int argc, char** argv) { + m_commandLineArgs.setArgs(argc, argv); + m_vRegistry->setFromArgs(commandLineArgs()); + // default log file +#if !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) + if (m_commandLineArgs.hasParamWithValue(base::consts::kDefaultLogFileParam)) { + Configurations c; + c.setGlobally(ConfigurationType::Filename, + std::string(m_commandLineArgs.getParamValue(base::consts::kDefaultLogFileParam))); + registeredLoggers()->setDefaultConfigurations(c); + for (base::RegisteredLoggers::iterator it = registeredLoggers()->begin(); + it != registeredLoggers()->end(); ++it) { + it->second->configure(c); + } + } +#endif // !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) +#if defined(ELPP_LOGGING_FLAGS_FROM_ARG) + if (m_commandLineArgs.hasParamWithValue(base::consts::kLoggingFlagsParam)) { + int userInput = atoi(m_commandLineArgs.getParamValue(base::consts::kLoggingFlagsParam)); + if (ELPP_DEFAULT_LOGGING_FLAGS == 0x0) { + m_flags = userInput; + } else { + base::utils::addFlag(userInput, &m_flags); + } + } +#endif // defined(ELPP_LOGGING_FLAGS_FROM_ARG) +} + +} // namespace base + +// LogDispatchCallback +void LogDispatchCallback::handle(const LogDispatchData* data) { +#if defined(ELPP_THREAD_SAFE) + base::threading::ScopedLock scopedLock(m_fileLocksMapLock); + std::string filename = data->logMessage()->logger()->typedConfigurations()->filename(data->logMessage()->level()); + auto lock = m_fileLocks.find(filename); + if (lock == m_fileLocks.end()) { + m_fileLocks.emplace(std::make_pair(filename, std::unique_ptr(new base::threading::Mutex))); + } +#endif +} + +base::threading::Mutex& LogDispatchCallback::fileHandle(const LogDispatchData* data) { + auto it = m_fileLocks.find(data->logMessage()->logger()->typedConfigurations()->filename(data->logMessage()->level())); + return *(it->second.get()); +} + +namespace base { +// DefaultLogDispatchCallback + +void DefaultLogDispatchCallback::handle(const LogDispatchData* data) { +#if defined(ELPP_THREAD_SAFE) + LogDispatchCallback::handle(data); + base::threading::ScopedLock scopedLock(fileHandle(data)); +#endif + m_data = data; + dispatch(m_data->logMessage()->logger()->logBuilder()->build(m_data->logMessage(), + m_data->dispatchAction() == base::DispatchAction::NormalLog)); +} + +void DefaultLogDispatchCallback::dispatch(base::type::string_t&& logLine) { + if (m_data->dispatchAction() == base::DispatchAction::NormalLog) { + if (m_data->logMessage()->logger()->m_typedConfigurations->toFile(m_data->logMessage()->level())) { + base::type::fstream_t* fs = m_data->logMessage()->logger()->m_typedConfigurations->fileStream( + m_data->logMessage()->level()); + if (fs != nullptr) { + fs->write(logLine.c_str(), logLine.size()); + if (fs->fail()) { + ELPP_INTERNAL_ERROR("Unable to write log to file [" + << m_data->logMessage()->logger()->m_typedConfigurations->filename(m_data->logMessage()->level()) << "].\n" + << "Few possible reasons (could be something else):\n" << " * Permission denied\n" + << " * Disk full\n" << " * Disk is not writable", true); + } else { + if (ELPP->hasFlag(LoggingFlag::ImmediateFlush) + || (m_data->logMessage()->logger()->isFlushNeeded(m_data->logMessage()->level()))) { + m_data->logMessage()->logger()->flush(m_data->logMessage()->level(), fs); + } + } + } else { + ELPP_INTERNAL_ERROR("Log file for [" << LevelHelper::convertToString(m_data->logMessage()->level()) << "] " + << "has not been configured but [TO_FILE] is configured to TRUE. [Logger ID: " + << m_data->logMessage()->logger()->id() << "]", false); + } + } + if (m_data->logMessage()->logger()->m_typedConfigurations->toStandardOutput(m_data->logMessage()->level())) { + if (ELPP->hasFlag(LoggingFlag::ColoredTerminalOutput)) + m_data->logMessage()->logger()->logBuilder()->convertToColoredOutput(&logLine, m_data->logMessage()->level()); + ELPP_COUT << ELPP_COUT_LINE(logLine); + } + } +#if defined(ELPP_SYSLOG) + else if (m_data->dispatchAction() == base::DispatchAction::SysLog) { + // Determine syslog priority + int sysLogPriority = 0; + if (m_data->logMessage()->level() == Level::Fatal) + sysLogPriority = LOG_EMERG; + else if (m_data->logMessage()->level() == Level::Error) + sysLogPriority = LOG_ERR; + else if (m_data->logMessage()->level() == Level::Warning) + sysLogPriority = LOG_WARNING; + else if (m_data->logMessage()->level() == Level::Info) + sysLogPriority = LOG_INFO; + else if (m_data->logMessage()->level() == Level::Debug) + sysLogPriority = LOG_DEBUG; + else + sysLogPriority = LOG_NOTICE; +# if defined(ELPP_UNICODE) + char* line = base::utils::Str::wcharPtrToCharPtr(logLine.c_str()); + syslog(sysLogPriority, "%s", line); + free(line); +# else + syslog(sysLogPriority, "%s", logLine.c_str()); +# endif + } +#endif // defined(ELPP_SYSLOG) +} + +#if ELPP_ASYNC_LOGGING + +// AsyncLogDispatchCallback + +void AsyncLogDispatchCallback::handle(const LogDispatchData* data) { + base::type::string_t logLine = data->logMessage()->logger()->logBuilder()->build(data->logMessage(), + data->dispatchAction() == base::DispatchAction::NormalLog); + if (data->dispatchAction() == base::DispatchAction::NormalLog + && data->logMessage()->logger()->typedConfigurations()->toStandardOutput(data->logMessage()->level())) { + if (ELPP->hasFlag(LoggingFlag::ColoredTerminalOutput)) + data->logMessage()->logger()->logBuilder()->convertToColoredOutput(&logLine, data->logMessage()->level()); + ELPP_COUT << ELPP_COUT_LINE(logLine); + } + // Save resources and only queue if we want to write to file otherwise just ignore handler + if (data->logMessage()->logger()->typedConfigurations()->toFile(data->logMessage()->level())) { + ELPP->asyncLogQueue()->push(AsyncLogItem(*(data->logMessage()), *data, logLine)); + } +} + +// AsyncDispatchWorker +AsyncDispatchWorker::AsyncDispatchWorker() { + setContinueRunning(false); +} + +AsyncDispatchWorker::~AsyncDispatchWorker() { + setContinueRunning(false); + ELPP_INTERNAL_INFO(6, "Stopping dispatch worker - Cleaning log queue"); + clean(); + ELPP_INTERNAL_INFO(6, "Log queue cleaned"); +} + +bool AsyncDispatchWorker::clean(void) { + std::mutex m; + std::unique_lock lk(m); + cv.wait(lk, [] { return !ELPP->asyncLogQueue()->empty(); }); + emptyQueue(); + lk.unlock(); + cv.notify_one(); + return ELPP->asyncLogQueue()->empty(); +} + +void AsyncDispatchWorker::emptyQueue(void) { + while (!ELPP->asyncLogQueue()->empty()) { + AsyncLogItem data = ELPP->asyncLogQueue()->next(); + handle(&data); + base::threading::msleep(100); + } +} + +void AsyncDispatchWorker::start(void) { + base::threading::msleep(5000); // 5s (why?) + setContinueRunning(true); + std::thread t1(&AsyncDispatchWorker::run, this); + t1.join(); +} + +void AsyncDispatchWorker::handle(AsyncLogItem* logItem) { + LogDispatchData* data = logItem->data(); + LogMessage* logMessage = logItem->logMessage(); + Logger* logger = logMessage->logger(); + base::TypedConfigurations* conf = logger->typedConfigurations(); + base::type::string_t logLine = logItem->logLine(); + if (data->dispatchAction() == base::DispatchAction::NormalLog) { + if (conf->toFile(logMessage->level())) { + base::type::fstream_t* fs = conf->fileStream(logMessage->level()); + if (fs != nullptr) { + fs->write(logLine.c_str(), logLine.size()); + if (fs->fail()) { + ELPP_INTERNAL_ERROR("Unable to write log to file [" + << conf->filename(logMessage->level()) << "].\n" + << "Few possible reasons (could be something else):\n" << " * Permission denied\n" + << " * Disk full\n" << " * Disk is not writable", true); + } else { + if (ELPP->hasFlag(LoggingFlag::ImmediateFlush) || (logger->isFlushNeeded(logMessage->level()))) { + logger->flush(logMessage->level(), fs); + } + } + } else { + ELPP_INTERNAL_ERROR("Log file for [" << LevelHelper::convertToString(logMessage->level()) << "] " + << "has not been configured but [TO_FILE] is configured to TRUE. [Logger ID: " << logger->id() << "]", false); + } + } + } +# if defined(ELPP_SYSLOG) + else if (data->dispatchAction() == base::DispatchAction::SysLog) { + // Determine syslog priority + int sysLogPriority = 0; + if (logMessage->level() == Level::Fatal) + sysLogPriority = LOG_EMERG; + else if (logMessage->level() == Level::Error) + sysLogPriority = LOG_ERR; + else if (logMessage->level() == Level::Warning) + sysLogPriority = LOG_WARNING; + else if (logMessage->level() == Level::Info) + sysLogPriority = LOG_INFO; + else if (logMessage->level() == Level::Debug) + sysLogPriority = LOG_DEBUG; + else + sysLogPriority = LOG_NOTICE; +# if defined(ELPP_UNICODE) + char* line = base::utils::Str::wcharPtrToCharPtr(logLine.c_str()); + syslog(sysLogPriority, "%s", line); + free(line); +# else + syslog(sysLogPriority, "%s", logLine.c_str()); +# endif + } +# endif // defined(ELPP_SYSLOG) +} + +void AsyncDispatchWorker::run(void) { + while (continueRunning()) { + emptyQueue(); + base::threading::msleep(10); // 10ms + } +} +#endif // ELPP_ASYNC_LOGGING + +// DefaultLogBuilder + +base::type::string_t DefaultLogBuilder::build(const LogMessage* logMessage, bool appendNewLine) const { + base::TypedConfigurations* tc = logMessage->logger()->typedConfigurations(); + const base::LogFormat* logFormat = &tc->logFormat(logMessage->level()); + base::type::string_t logLine = logFormat->format(); + char buff[base::consts::kSourceFilenameMaxLength + base::consts::kSourceLineMaxLength] = ""; + const char* bufLim = buff + sizeof(buff); + if (logFormat->hasFlag(base::FormatFlags::AppName)) { + // App name + base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kAppNameFormatSpecifier, + logMessage->logger()->parentApplicationName()); + } + if (logFormat->hasFlag(base::FormatFlags::ThreadId)) { + // Thread ID + base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kThreadIdFormatSpecifier, + ELPP->getThreadName(base::threading::getCurrentThreadId())); + } + if (logFormat->hasFlag(base::FormatFlags::DateTime)) { + // DateTime + base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kDateTimeFormatSpecifier, + base::utils::DateTime::getDateTime(logFormat->dateTimeFormat().c_str(), + &tc->subsecondPrecision(logMessage->level()))); + } + if (logFormat->hasFlag(base::FormatFlags::Function)) { + // Function + base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kLogFunctionFormatSpecifier, logMessage->func()); + } + if (logFormat->hasFlag(base::FormatFlags::File)) { + // File + base::utils::Str::clearBuff(buff, base::consts::kSourceFilenameMaxLength); + base::utils::File::buildStrippedFilename(logMessage->file().c_str(), buff); + base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kLogFileFormatSpecifier, std::string(buff)); + } + if (logFormat->hasFlag(base::FormatFlags::FileBase)) { + // FileBase + base::utils::Str::clearBuff(buff, base::consts::kSourceFilenameMaxLength); + base::utils::File::buildBaseFilename(logMessage->file(), buff); + base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kLogFileBaseFormatSpecifier, std::string(buff)); + } + if (logFormat->hasFlag(base::FormatFlags::Line)) { + // Line + char* buf = base::utils::Str::clearBuff(buff, base::consts::kSourceLineMaxLength); + buf = base::utils::Str::convertAndAddToBuff(logMessage->line(), base::consts::kSourceLineMaxLength, buf, bufLim, false); + base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kLogLineFormatSpecifier, std::string(buff)); + } + if (logFormat->hasFlag(base::FormatFlags::Location)) { + // Location + char* buf = base::utils::Str::clearBuff(buff, + base::consts::kSourceFilenameMaxLength + base::consts::kSourceLineMaxLength); + base::utils::File::buildStrippedFilename(logMessage->file().c_str(), buff); + buf = base::utils::Str::addToBuff(buff, buf, bufLim); + buf = base::utils::Str::addToBuff(":", buf, bufLim); + buf = base::utils::Str::convertAndAddToBuff(logMessage->line(), base::consts::kSourceLineMaxLength, buf, bufLim, + false); + base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kLogLocationFormatSpecifier, std::string(buff)); + } + if (logMessage->level() == Level::Verbose && logFormat->hasFlag(base::FormatFlags::VerboseLevel)) { + // Verbose level + char* buf = base::utils::Str::clearBuff(buff, 1); + buf = base::utils::Str::convertAndAddToBuff(logMessage->verboseLevel(), 1, buf, bufLim, false); + base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kVerboseLevelFormatSpecifier, std::string(buff)); + } + if (logFormat->hasFlag(base::FormatFlags::LogMessage)) { + // Log message + base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kMessageFormatSpecifier, logMessage->message()); + } +#if !defined(ELPP_DISABLE_CUSTOM_FORMAT_SPECIFIERS) + el::base::threading::ScopedLock lock_(ELPP->customFormatSpecifiersLock()); + ELPP_UNUSED(lock_); + for (std::vector::const_iterator it = ELPP->customFormatSpecifiers()->begin(); + it != ELPP->customFormatSpecifiers()->end(); ++it) { + std::string fs(it->formatSpecifier()); + base::type::string_t wcsFormatSpecifier(fs.begin(), fs.end()); + base::utils::Str::replaceFirstWithEscape(logLine, wcsFormatSpecifier, it->resolver()(logMessage)); + } +#endif // !defined(ELPP_DISABLE_CUSTOM_FORMAT_SPECIFIERS) + if (appendNewLine) logLine += ELPP_LITERAL("\n"); + return logLine; +} + +// LogDispatcher + +void LogDispatcher::dispatch(void) { + if (m_proceed && m_dispatchAction == base::DispatchAction::None) { + m_proceed = false; + } + if (!m_proceed) { + return; + } +#ifndef ELPP_NO_GLOBAL_LOCK + // see https://github.com/muflihun/easyloggingpp/issues/580 + // global lock is turned off by default unless + // ELPP_NO_GLOBAL_LOCK is defined + base::threading::ScopedLock scopedLock(ELPP->lock()); +#endif + base::TypedConfigurations* tc = m_logMessage->logger()->m_typedConfigurations; + if (ELPP->hasFlag(LoggingFlag::StrictLogFileSizeCheck)) { + tc->validateFileRolling(m_logMessage->level(), ELPP->preRollOutCallback()); + } + LogDispatchCallback* callback = nullptr; + LogDispatchData data; + for (const std::pair& h + : ELPP->m_logDispatchCallbacks) { + callback = h.second.get(); + if (callback != nullptr && callback->enabled()) { + data.setLogMessage(m_logMessage); + data.setDispatchAction(m_dispatchAction); + callback->handle(&data); + } + } +} + +// MessageBuilder + +void MessageBuilder::initialize(Logger* logger) { + m_logger = logger; + m_containerLogSeperator = ELPP->hasFlag(LoggingFlag::NewLineForContainer) ? + ELPP_LITERAL("\n ") : ELPP_LITERAL(", "); +} + +MessageBuilder& MessageBuilder::operator<<(const wchar_t* msg) { + if (msg == nullptr) { + m_logger->stream() << base::consts::kNullPointer; + return *this; + } +# if defined(ELPP_UNICODE) + m_logger->stream() << msg; +# else + char* buff_ = base::utils::Str::wcharPtrToCharPtr(msg); + m_logger->stream() << buff_; + free(buff_); +# endif + if (ELPP->hasFlag(LoggingFlag::AutoSpacing)) { + m_logger->stream() << " "; + } + return *this; +} + +// Writer + +Writer& Writer::construct(Logger* logger, bool needLock) { + m_logger = logger; + initializeLogger(logger->id(), false, needLock); + m_messageBuilder.initialize(m_logger); + return *this; +} + +Writer& Writer::construct(int count, const char* loggerIds, ...) { + if (ELPP->hasFlag(LoggingFlag::MultiLoggerSupport)) { + va_list loggersList; + va_start(loggersList, loggerIds); + const char* id = loggerIds; + m_loggerIds.reserve(count); + for (int i = 0; i < count; ++i) { + m_loggerIds.push_back(std::string(id)); + id = va_arg(loggersList, const char*); + } + va_end(loggersList); + initializeLogger(m_loggerIds.at(0)); + } else { + initializeLogger(std::string(loggerIds)); + } + m_messageBuilder.initialize(m_logger); + return *this; +} + +void Writer::initializeLogger(const std::string& loggerId, bool lookup, bool needLock) { + if (lookup) { + m_logger = ELPP->registeredLoggers()->get(loggerId, ELPP->hasFlag(LoggingFlag::CreateLoggerAutomatically)); + } + if (m_logger == nullptr) { + { + if (!ELPP->registeredLoggers()->has(std::string(base::consts::kDefaultLoggerId))) { + // Somehow default logger has been unregistered. Not good! Register again + ELPP->registeredLoggers()->get(std::string(base::consts::kDefaultLoggerId)); + } + } + Writer(Level::Debug, m_file, m_line, m_func).construct(1, base::consts::kDefaultLoggerId) + << "Logger [" << loggerId << "] is not registered yet!"; + m_proceed = false; + } else { + if (needLock) { + m_logger->acquireLock(); // This should not be unlocked by checking m_proceed because + // m_proceed can be changed by lines below + } + if (ELPP->hasFlag(LoggingFlag::HierarchicalLogging)) { + m_proceed = m_level == Level::Verbose ? m_logger->enabled(m_level) : + LevelHelper::castToInt(m_level) >= LevelHelper::castToInt(ELPP->m_loggingLevel); + } else { + m_proceed = m_logger->enabled(m_level); + } + } +} + +void Writer::processDispatch() { +#if ELPP_LOGGING_ENABLED + if (ELPP->hasFlag(LoggingFlag::MultiLoggerSupport)) { + bool firstDispatched = false; + base::type::string_t logMessage; + std::size_t i = 0; + do { + if (m_proceed) { + if (firstDispatched) { + m_logger->stream() << logMessage; + } else { + firstDispatched = true; + if (m_loggerIds.size() > 1) { + logMessage = m_logger->stream().str(); + } + } + triggerDispatch(); + } else if (m_logger != nullptr) { + m_logger->stream().str(ELPP_LITERAL("")); + m_logger->releaseLock(); + } + if (i + 1 < m_loggerIds.size()) { + initializeLogger(m_loggerIds.at(i + 1)); + } + } while (++i < m_loggerIds.size()); + } else { + if (m_proceed) { + triggerDispatch(); + } else if (m_logger != nullptr) { + m_logger->stream().str(ELPP_LITERAL("")); + m_logger->releaseLock(); + } + } +#else + if (m_logger != nullptr) { + m_logger->stream().str(ELPP_LITERAL("")); + m_logger->releaseLock(); + } +#endif // ELPP_LOGGING_ENABLED +} + +void Writer::triggerDispatch(void) { + if (m_proceed) { + if (m_msg == nullptr) { + LogMessage msg(m_level, m_file, m_line, m_func, m_verboseLevel, + m_logger); + base::LogDispatcher(m_proceed, &msg, m_dispatchAction).dispatch(); + } else { + base::LogDispatcher(m_proceed, m_msg, m_dispatchAction).dispatch(); + } + } + if (m_logger != nullptr) { + m_logger->stream().str(ELPP_LITERAL("")); + m_logger->releaseLock(); + } + if (m_proceed && m_level == Level::Fatal + && !ELPP->hasFlag(LoggingFlag::DisableApplicationAbortOnFatalLog)) { + base::Writer(Level::Warning, m_file, m_line, m_func).construct(1, base::consts::kDefaultLoggerId) + << "Aborting application. Reason: Fatal log at [" << m_file << ":" << m_line << "]"; + std::stringstream reasonStream; + reasonStream << "Fatal log at [" << m_file << ":" << m_line << "]" + << " If you wish to disable 'abort on fatal log' please use " + << "el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog)"; + base::utils::abort(1, reasonStream.str()); + } + m_proceed = false; +} + +// PErrorWriter + +PErrorWriter::~PErrorWriter(void) { + if (m_proceed) { +#if ELPP_COMPILER_MSVC + char buff[256]; + strerror_s(buff, 256, errno); + m_logger->stream() << ": " << buff << " [" << errno << "]"; +#else + m_logger->stream() << ": " << strerror(errno) << " [" << errno << "]"; +#endif + } +} + +// PerformanceTracker + +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + +PerformanceTracker::PerformanceTracker(const std::string& blockName, + base::TimestampUnit timestampUnit, + const std::string& loggerId, + bool scopedLog, Level level) : + m_blockName(blockName), m_timestampUnit(timestampUnit), m_loggerId(loggerId), m_scopedLog(scopedLog), + m_level(level), m_hasChecked(false), m_lastCheckpointId(std::string()), m_enabled(false) { +#if !defined(ELPP_DISABLE_PERFORMANCE_TRACKING) && ELPP_LOGGING_ENABLED + // We store it locally so that if user happen to change configuration by the end of scope + // or before calling checkpoint, we still depend on state of configuraton at time of construction + el::Logger* loggerPtr = ELPP->registeredLoggers()->get(loggerId, false); + m_enabled = loggerPtr != nullptr && loggerPtr->m_typedConfigurations->performanceTracking(m_level); + if (m_enabled) { + base::utils::DateTime::gettimeofday(&m_startTime); + } +#endif // !defined(ELPP_DISABLE_PERFORMANCE_TRACKING) && ELPP_LOGGING_ENABLED +} + +PerformanceTracker::~PerformanceTracker(void) { +#if !defined(ELPP_DISABLE_PERFORMANCE_TRACKING) && ELPP_LOGGING_ENABLED + if (m_enabled) { + base::threading::ScopedLock scopedLock(lock()); + if (m_scopedLog) { + base::utils::DateTime::gettimeofday(&m_endTime); + base::type::string_t formattedTime = getFormattedTimeTaken(); + PerformanceTrackingData data(PerformanceTrackingData::DataType::Complete); + data.init(this); + data.m_formattedTimeTaken = formattedTime; + PerformanceTrackingCallback* callback = nullptr; + for (const std::pair& h + : ELPP->m_performanceTrackingCallbacks) { + callback = h.second.get(); + if (callback != nullptr && callback->enabled()) { + callback->handle(&data); + } + } + } + } +#endif // !defined(ELPP_DISABLE_PERFORMANCE_TRACKING) +} + +void PerformanceTracker::checkpoint(const std::string& id, const char* file, base::type::LineNumber line, + const char* func) { +#if !defined(ELPP_DISABLE_PERFORMANCE_TRACKING) && ELPP_LOGGING_ENABLED + if (m_enabled) { + base::threading::ScopedLock scopedLock(lock()); + base::utils::DateTime::gettimeofday(&m_endTime); + base::type::string_t formattedTime = m_hasChecked ? getFormattedTimeTaken(m_lastCheckpointTime) : ELPP_LITERAL(""); + PerformanceTrackingData data(PerformanceTrackingData::DataType::Checkpoint); + data.init(this); + data.m_checkpointId = id; + data.m_file = file; + data.m_line = line; + data.m_func = func; + data.m_formattedTimeTaken = formattedTime; + PerformanceTrackingCallback* callback = nullptr; + for (const std::pair& h + : ELPP->m_performanceTrackingCallbacks) { + callback = h.second.get(); + if (callback != nullptr && callback->enabled()) { + callback->handle(&data); + } + } + base::utils::DateTime::gettimeofday(&m_lastCheckpointTime); + m_hasChecked = true; + m_lastCheckpointId = id; + } +#endif // !defined(ELPP_DISABLE_PERFORMANCE_TRACKING) && ELPP_LOGGING_ENABLED + ELPP_UNUSED(id); + ELPP_UNUSED(file); + ELPP_UNUSED(line); + ELPP_UNUSED(func); +} + +const base::type::string_t PerformanceTracker::getFormattedTimeTaken(struct timeval startTime) const { + if (ELPP->hasFlag(LoggingFlag::FixedTimeFormat)) { + base::type::stringstream_t ss; + ss << base::utils::DateTime::getTimeDifference(m_endTime, + startTime, m_timestampUnit) << " " << base::consts::kTimeFormats[static_cast + (m_timestampUnit)].unit; + return ss.str(); + } + return base::utils::DateTime::formatTime(base::utils::DateTime::getTimeDifference(m_endTime, + startTime, m_timestampUnit), m_timestampUnit); +} + +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + +namespace debug { +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) + +// StackTrace + +StackTrace::StackTraceEntry::StackTraceEntry(std::size_t index, const std::string& loc, const std::string& demang, + const std::string& hex, + const std::string& addr) : + m_index(index), + m_location(loc), + m_demangled(demang), + m_hex(hex), + m_addr(addr) { +} + +std::ostream& operator<<(std::ostream& ss, const StackTrace::StackTraceEntry& si) { + ss << "[" << si.m_index << "] " << si.m_location << (si.m_hex.empty() ? "" : "+") << si.m_hex << " " << si.m_addr << + (si.m_demangled.empty() ? "" : ":") << si.m_demangled; + return ss; +} + +std::ostream& operator<<(std::ostream& os, const StackTrace& st) { + std::vector::const_iterator it = st.m_stack.begin(); + while (it != st.m_stack.end()) { + os << " " << *it++ << "\n"; + } + return os; +} + +void StackTrace::generateNew(void) { +#if ELPP_STACKTRACE + m_stack.clear(); + void* stack[kMaxStack]; + unsigned int size = backtrace(stack, kMaxStack); + char** strings = backtrace_symbols(stack, size); + if (size > kStackStart) { // Skip StackTrace c'tor and generateNew + for (std::size_t i = kStackStart; i < size; ++i) { + std::string mangName; + std::string location; + std::string hex; + std::string addr; + + // entry: 2 crash.cpp.bin 0x0000000101552be5 _ZN2el4base5debug10StackTraceC1Ev + 21 + const std::string line(strings[i]); + auto p = line.find("_"); + if (p != std::string::npos) { + mangName = line.substr(p); + mangName = mangName.substr(0, mangName.find(" +")); + } + p = line.find("0x"); + if (p != std::string::npos) { + addr = line.substr(p); + addr = addr.substr(0, addr.find("_")); + } + // Perform demangling if parsed properly + if (!mangName.empty()) { + int status = 0; + char* demangName = abi::__cxa_demangle(mangName.data(), 0, 0, &status); + // if demangling is successful, output the demangled function name + if (status == 0) { + // Success (see http://gcc.gnu.org/onlinedocs/libstdc++/libstdc++-html-USERS-4.3/a01696.html) + StackTraceEntry entry(i - 1, location, demangName, hex, addr); + m_stack.push_back(entry); + } else { + // Not successful - we will use mangled name + StackTraceEntry entry(i - 1, location, mangName, hex, addr); + m_stack.push_back(entry); + } + free(demangName); + } else { + StackTraceEntry entry(i - 1, line); + m_stack.push_back(entry); + } + } + } + free(strings); +#else + ELPP_INTERNAL_INFO(1, "Stacktrace generation not supported for selected compiler"); +#endif // ELPP_STACKTRACE +} + +// Static helper functions + +static std::string crashReason(int sig) { + std::stringstream ss; + bool foundReason = false; + for (int i = 0; i < base::consts::kCrashSignalsCount; ++i) { + if (base::consts::kCrashSignals[i].numb == sig) { + ss << "Application has crashed due to [" << base::consts::kCrashSignals[i].name << "] signal"; + if (ELPP->hasFlag(el::LoggingFlag::LogDetailedCrashReason)) { + ss << std::endl << + " " << base::consts::kCrashSignals[i].brief << std::endl << + " " << base::consts::kCrashSignals[i].detail; + } + foundReason = true; + } + } + if (!foundReason) { + ss << "Application has crashed due to unknown signal [" << sig << "]"; + } + return ss.str(); +} +/// @brief Logs reason of crash from sig +static void logCrashReason(int sig, bool stackTraceIfAvailable, Level level, const char* logger) { + if (sig == SIGINT && ELPP->hasFlag(el::LoggingFlag::IgnoreSigInt)) { + return; + } + std::stringstream ss; + ss << "CRASH HANDLED; "; + ss << crashReason(sig); +#if ELPP_STACKTRACE + if (stackTraceIfAvailable) { + ss << std::endl << " ======= Backtrace: =========" << std::endl << base::debug::StackTrace(); + } +#else + ELPP_UNUSED(stackTraceIfAvailable); +#endif // ELPP_STACKTRACE + ELPP_WRITE_LOG(el::base::Writer, level, base::DispatchAction::NormalLog, logger) << ss.str(); +} + +static inline void crashAbort(int sig) { + base::utils::abort(sig, std::string()); +} + +/// @brief Default application crash handler +/// +/// @detail This function writes log using 'default' logger, prints stack trace for GCC based compilers and aborts program. +static inline void defaultCrashHandler(int sig) { + base::debug::logCrashReason(sig, true, Level::Fatal, base::consts::kDefaultLoggerId); + base::debug::crashAbort(sig); +} + +// CrashHandler + +CrashHandler::CrashHandler(bool useDefault) { + if (useDefault) { + setHandler(defaultCrashHandler); + } +} + +void CrashHandler::setHandler(const Handler& cHandler) { + m_handler = cHandler; +#if defined(ELPP_HANDLE_SIGABRT) + int i = 0; // SIGABRT is at base::consts::kCrashSignals[0] +#else + int i = 1; +#endif // defined(ELPP_HANDLE_SIGABRT) + for (; i < base::consts::kCrashSignalsCount; ++i) { + m_handler = signal(base::consts::kCrashSignals[i].numb, cHandler); + } +} + +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) +} // namespace debug +} // namespace base + +// el + +// Helpers + +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) + +void Helpers::crashAbort(int sig, const char* sourceFile, unsigned int long line) { + std::stringstream ss; + ss << base::debug::crashReason(sig).c_str(); + ss << " - [Called el::Helpers::crashAbort(" << sig << ")]"; + if (sourceFile != nullptr && strlen(sourceFile) > 0) { + ss << " - Source: " << sourceFile; + if (line > 0) + ss << ":" << line; + else + ss << " (line number not specified)"; + } + base::utils::abort(sig, ss.str()); +} + +void Helpers::logCrashReason(int sig, bool stackTraceIfAvailable, Level level, const char* logger) { + el::base::debug::logCrashReason(sig, stackTraceIfAvailable, level, logger); +} + +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) + +// Loggers + +Logger* Loggers::getLogger(const std::string& identity, bool registerIfNotAvailable) { + return ELPP->registeredLoggers()->get(identity, registerIfNotAvailable); +} + +void Loggers::setDefaultLogBuilder(el::LogBuilderPtr& logBuilderPtr) { + ELPP->registeredLoggers()->setDefaultLogBuilder(logBuilderPtr); +} + +bool Loggers::unregisterLogger(const std::string& identity) { + return ELPP->registeredLoggers()->remove(identity); +} + +bool Loggers::hasLogger(const std::string& identity) { + return ELPP->registeredLoggers()->has(identity); +} + +Logger* Loggers::reconfigureLogger(Logger* logger, const Configurations& configurations) { + if (!logger) return nullptr; + logger->configure(configurations); + return logger; +} + +Logger* Loggers::reconfigureLogger(const std::string& identity, const Configurations& configurations) { + return Loggers::reconfigureLogger(Loggers::getLogger(identity), configurations); +} + +Logger* Loggers::reconfigureLogger(const std::string& identity, ConfigurationType configurationType, + const std::string& value) { + Logger* logger = Loggers::getLogger(identity); + if (logger == nullptr) { + return nullptr; + } + logger->configurations()->set(Level::Global, configurationType, value); + logger->reconfigure(); + return logger; +} + +void Loggers::reconfigureAllLoggers(const Configurations& configurations) { + for (base::RegisteredLoggers::iterator it = ELPP->registeredLoggers()->begin(); + it != ELPP->registeredLoggers()->end(); ++it) { + Loggers::reconfigureLogger(it->second, configurations); + } +} + +void Loggers::reconfigureAllLoggers(Level level, ConfigurationType configurationType, + const std::string& value) { + for (base::RegisteredLoggers::iterator it = ELPP->registeredLoggers()->begin(); + it != ELPP->registeredLoggers()->end(); ++it) { + Logger* logger = it->second; + logger->configurations()->set(level, configurationType, value); + logger->reconfigure(); + } +} + +void Loggers::setDefaultConfigurations(const Configurations& configurations, bool reconfigureExistingLoggers) { + ELPP->registeredLoggers()->setDefaultConfigurations(configurations); + if (reconfigureExistingLoggers) { + Loggers::reconfigureAllLoggers(configurations); + } +} + +const Configurations* Loggers::defaultConfigurations(void) { + return ELPP->registeredLoggers()->defaultConfigurations(); +} + +const base::LogStreamsReferenceMap* Loggers::logStreamsReference(void) { + return ELPP->registeredLoggers()->logStreamsReference(); +} + +base::TypedConfigurations Loggers::defaultTypedConfigurations(void) { + return base::TypedConfigurations( + ELPP->registeredLoggers()->defaultConfigurations(), + ELPP->registeredLoggers()->logStreamsReference()); +} + +std::vector* Loggers::populateAllLoggerIds(std::vector* targetList) { + targetList->clear(); + for (base::RegisteredLoggers::iterator it = ELPP->registeredLoggers()->list().begin(); + it != ELPP->registeredLoggers()->list().end(); ++it) { + targetList->push_back(it->first); + } + return targetList; +} + +void Loggers::configureFromGlobal(const char* globalConfigurationFilePath) { + std::ifstream gcfStream(globalConfigurationFilePath, std::ifstream::in); + ELPP_ASSERT(gcfStream.is_open(), "Unable to open global configuration file [" << globalConfigurationFilePath + << "] for parsing."); + std::string line = std::string(); + std::stringstream ss; + Logger* logger = nullptr; + auto configure = [&](void) { + ELPP_INTERNAL_INFO(8, "Configuring logger: '" << logger->id() << "' with configurations \n" << ss.str() + << "\n--------------"); + Configurations c; + c.parseFromText(ss.str()); + logger->configure(c); + }; + while (gcfStream.good()) { + std::getline(gcfStream, line); + ELPP_INTERNAL_INFO(1, "Parsing line: " << line); + base::utils::Str::trim(line); + if (Configurations::Parser::isComment(line)) continue; + Configurations::Parser::ignoreComments(&line); + base::utils::Str::trim(line); + if (line.size() > 2 && base::utils::Str::startsWith(line, std::string(base::consts::kConfigurationLoggerId))) { + if (!ss.str().empty() && logger != nullptr) { + configure(); + } + ss.str(std::string("")); + line = line.substr(2); + base::utils::Str::trim(line); + if (line.size() > 1) { + ELPP_INTERNAL_INFO(1, "Getting logger: '" << line << "'"); + logger = getLogger(line); + } + } else { + ss << line << "\n"; + } + } + if (!ss.str().empty() && logger != nullptr) { + configure(); + } +} + +bool Loggers::configureFromArg(const char* argKey) { +#if defined(ELPP_DISABLE_CONFIGURATION_FROM_PROGRAM_ARGS) + ELPP_UNUSED(argKey); +#else + if (!Helpers::commandLineArgs()->hasParamWithValue(argKey)) { + return false; + } + configureFromGlobal(Helpers::commandLineArgs()->getParamValue(argKey)); +#endif // defined(ELPP_DISABLE_CONFIGURATION_FROM_PROGRAM_ARGS) + return true; +} + +void Loggers::flushAll(void) { + ELPP->registeredLoggers()->flushAll(); +} + +void Loggers::setVerboseLevel(base::type::VerboseLevel level) { + ELPP->vRegistry()->setLevel(level); +} + +base::type::VerboseLevel Loggers::verboseLevel(void) { + return ELPP->vRegistry()->level(); +} + +void Loggers::setVModules(const char* modules) { + if (ELPP->vRegistry()->vModulesEnabled()) { + ELPP->vRegistry()->setModules(modules); + } +} + +void Loggers::clearVModules(void) { + ELPP->vRegistry()->clearModules(); +} + +// VersionInfo + +const std::string VersionInfo::version(void) { + return std::string("9.96.7"); +} +/// @brief Release date of current version +const std::string VersionInfo::releaseDate(void) { + return std::string("24-11-2018 0728hrs"); +} + +} // namespace el diff --git a/cpp/src/utils/easylogging++.h b/cpp/src/utils/easylogging++.h new file mode 100644 index 0000000000..62a7c5a423 --- /dev/null +++ b/cpp/src/utils/easylogging++.h @@ -0,0 +1,4569 @@ +// +// Bismillah ar-Rahmaan ar-Raheem +// +// Easylogging++ v9.96.7 +// Single-header only, cross-platform logging library for C++ applications +// +// Copyright (c) 2012-2018 Zuhd Web Services +// Copyright (c) 2012-2018 @abumusamq +// +// This library is released under the MIT Licence. +// https://github.com/zuhd-org/easyloggingpp/blob/master/LICENSE +// +// https://zuhd.org +// http://muflihun.com +// + +#ifndef EASYLOGGINGPP_H +#define EASYLOGGINGPP_H +// Compilers and C++0x/C++11 Evaluation +#if __cplusplus >= 201103L +# define ELPP_CXX11 1 +#endif // __cplusplus >= 201103L +#if (defined(__GNUC__)) +# define ELPP_COMPILER_GCC 1 +#else +# define ELPP_COMPILER_GCC 0 +#endif +#if ELPP_COMPILER_GCC +# define ELPP_GCC_VERSION (__GNUC__ * 10000 \ ++ __GNUC_MINOR__ * 100 \ ++ __GNUC_PATCHLEVEL__) +# if defined(__GXX_EXPERIMENTAL_CXX0X__) +# define ELPP_CXX0X 1 +# endif +#endif +// Visual C++ +#if defined(_MSC_VER) +# define ELPP_COMPILER_MSVC 1 +#else +# define ELPP_COMPILER_MSVC 0 +#endif +#define ELPP_CRT_DBG_WARNINGS ELPP_COMPILER_MSVC +#if ELPP_COMPILER_MSVC +# if (_MSC_VER == 1600) +# define ELPP_CXX0X 1 +# elif(_MSC_VER >= 1700) +# define ELPP_CXX11 1 +# endif +#endif +// Clang++ +#if (defined(__clang__) && (__clang__ == 1)) +# define ELPP_COMPILER_CLANG 1 +#else +# define ELPP_COMPILER_CLANG 0 +#endif +#if ELPP_COMPILER_CLANG +# if __has_include() +# include // Make __GLIBCXX__ defined when using libstdc++ +# if !defined(__GLIBCXX__) || __GLIBCXX__ >= 20150426 +# define ELPP_CLANG_SUPPORTS_THREAD +# endif // !defined(__GLIBCXX__) || __GLIBCXX__ >= 20150426 +# endif // __has_include() +#endif +#if (defined(__MINGW32__) || defined(__MINGW64__)) +# define ELPP_MINGW 1 +#else +# define ELPP_MINGW 0 +#endif +#if (defined(__CYGWIN__) && (__CYGWIN__ == 1)) +# define ELPP_CYGWIN 1 +#else +# define ELPP_CYGWIN 0 +#endif +#if (defined(__INTEL_COMPILER)) +# define ELPP_COMPILER_INTEL 1 +#else +# define ELPP_COMPILER_INTEL 0 +#endif +// Operating System Evaluation +// Windows +#if (defined(_WIN32) || defined(_WIN64)) +# define ELPP_OS_WINDOWS 1 +#else +# define ELPP_OS_WINDOWS 0 +#endif +// Linux +#if (defined(__linux) || defined(__linux__)) +# define ELPP_OS_LINUX 1 +#else +# define ELPP_OS_LINUX 0 +#endif +#if (defined(__APPLE__)) +# define ELPP_OS_MAC 1 +#else +# define ELPP_OS_MAC 0 +#endif +#if (defined(__FreeBSD__) || defined(__FreeBSD_kernel__)) +# define ELPP_OS_FREEBSD 1 +#else +# define ELPP_OS_FREEBSD 0 +#endif +#if (defined(__sun)) +# define ELPP_OS_SOLARIS 1 +#else +# define ELPP_OS_SOLARIS 0 +#endif +#if (defined(_AIX)) +# define ELPP_OS_AIX 1 +#else +# define ELPP_OS_AIX 0 +#endif +#if (defined(__NetBSD__)) +# define ELPP_OS_NETBSD 1 +#else +# define ELPP_OS_NETBSD 0 +#endif +#if defined(__EMSCRIPTEN__) +# define ELPP_OS_EMSCRIPTEN 1 +#else +# define ELPP_OS_EMSCRIPTEN 0 +#endif +// Unix +#if ((ELPP_OS_LINUX || ELPP_OS_MAC || ELPP_OS_FREEBSD || ELPP_OS_NETBSD || ELPP_OS_SOLARIS || ELPP_OS_AIX || ELPP_OS_EMSCRIPTEN) && (!ELPP_OS_WINDOWS)) +# define ELPP_OS_UNIX 1 +#else +# define ELPP_OS_UNIX 0 +#endif +#if (defined(__ANDROID__)) +# define ELPP_OS_ANDROID 1 +#else +# define ELPP_OS_ANDROID 0 +#endif +// Evaluating Cygwin as *nix OS +#if !ELPP_OS_UNIX && !ELPP_OS_WINDOWS && ELPP_CYGWIN +# undef ELPP_OS_UNIX +# undef ELPP_OS_LINUX +# define ELPP_OS_UNIX 1 +# define ELPP_OS_LINUX 1 +#endif // !ELPP_OS_UNIX && !ELPP_OS_WINDOWS && ELPP_CYGWIN +#if !defined(ELPP_INTERNAL_DEBUGGING_OUT_INFO) +# define ELPP_INTERNAL_DEBUGGING_OUT_INFO std::cout +#endif // !defined(ELPP_INTERNAL_DEBUGGING_OUT) +#if !defined(ELPP_INTERNAL_DEBUGGING_OUT_ERROR) +# define ELPP_INTERNAL_DEBUGGING_OUT_ERROR std::cerr +#endif // !defined(ELPP_INTERNAL_DEBUGGING_OUT) +#if !defined(ELPP_INTERNAL_DEBUGGING_ENDL) +# define ELPP_INTERNAL_DEBUGGING_ENDL std::endl +#endif // !defined(ELPP_INTERNAL_DEBUGGING_OUT) +#if !defined(ELPP_INTERNAL_DEBUGGING_MSG) +# define ELPP_INTERNAL_DEBUGGING_MSG(msg) msg +#endif // !defined(ELPP_INTERNAL_DEBUGGING_OUT) +// Internal Assertions and errors +#if !defined(ELPP_DISABLE_ASSERT) +# if (defined(ELPP_DEBUG_ASSERT_FAILURE)) +# define ELPP_ASSERT(expr, msg) if (!(expr)) { \ +std::stringstream internalInfoStream; internalInfoStream << msg; \ +ELPP_INTERNAL_DEBUGGING_OUT_ERROR \ +<< "EASYLOGGING++ ASSERTION FAILED (LINE: " << __LINE__ << ") [" #expr << "] WITH MESSAGE \"" \ +<< ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStream.str()) << "\"" << ELPP_INTERNAL_DEBUGGING_ENDL; base::utils::abort(1, \ +"ELPP Assertion failure, please define ELPP_DEBUG_ASSERT_FAILURE"); } +# else +# define ELPP_ASSERT(expr, msg) if (!(expr)) { \ +std::stringstream internalInfoStream; internalInfoStream << msg; \ +ELPP_INTERNAL_DEBUGGING_OUT_ERROR\ +<< "ASSERTION FAILURE FROM EASYLOGGING++ (LINE: " \ +<< __LINE__ << ") [" #expr << "] WITH MESSAGE \"" << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStream.str()) << "\"" \ +<< ELPP_INTERNAL_DEBUGGING_ENDL; } +# endif // (defined(ELPP_DEBUG_ASSERT_FAILURE)) +#else +# define ELPP_ASSERT(x, y) +#endif //(!defined(ELPP_DISABLE_ASSERT) +#if ELPP_COMPILER_MSVC +# define ELPP_INTERNAL_DEBUGGING_WRITE_PERROR \ +{ char buff[256]; strerror_s(buff, 256, errno); \ +ELPP_INTERNAL_DEBUGGING_OUT_ERROR << ": " << buff << " [" << errno << "]";} (void)0 +#else +# define ELPP_INTERNAL_DEBUGGING_WRITE_PERROR \ +ELPP_INTERNAL_DEBUGGING_OUT_ERROR << ": " << strerror(errno) << " [" << errno << "]"; (void)0 +#endif // ELPP_COMPILER_MSVC +#if defined(ELPP_DEBUG_ERRORS) +# if !defined(ELPP_INTERNAL_ERROR) +# define ELPP_INTERNAL_ERROR(msg, pe) { \ +std::stringstream internalInfoStream; internalInfoStream << " " << msg; \ +ELPP_INTERNAL_DEBUGGING_OUT_ERROR \ +<< "ERROR FROM EASYLOGGING++ (LINE: " << __LINE__ << ") " \ +<< ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStream.str()) << ELPP_INTERNAL_DEBUGGING_ENDL; \ +if (pe) { ELPP_INTERNAL_DEBUGGING_OUT_ERROR << " "; ELPP_INTERNAL_DEBUGGING_WRITE_PERROR; }} (void)0 +# endif +#else +# undef ELPP_INTERNAL_INFO +# define ELPP_INTERNAL_ERROR(msg, pe) +#endif // defined(ELPP_DEBUG_ERRORS) +#if (defined(ELPP_DEBUG_INFO)) +# if !(defined(ELPP_INTERNAL_INFO_LEVEL)) +# define ELPP_INTERNAL_INFO_LEVEL 9 +# endif // !(defined(ELPP_INTERNAL_INFO_LEVEL)) +# if !defined(ELPP_INTERNAL_INFO) +# define ELPP_INTERNAL_INFO(lvl, msg) { if (lvl <= ELPP_INTERNAL_INFO_LEVEL) { \ +std::stringstream internalInfoStream; internalInfoStream << " " << msg; \ +ELPP_INTERNAL_DEBUGGING_OUT_INFO << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStream.str()) \ +<< ELPP_INTERNAL_DEBUGGING_ENDL; }} +# endif +#else +# undef ELPP_INTERNAL_INFO +# define ELPP_INTERNAL_INFO(lvl, msg) +#endif // (defined(ELPP_DEBUG_INFO)) +#if (defined(ELPP_FEATURE_ALL)) || (defined(ELPP_FEATURE_CRASH_LOG)) +# if (ELPP_COMPILER_GCC && !ELPP_MINGW && !ELPP_OS_ANDROID && !ELPP_OS_EMSCRIPTEN) +# define ELPP_STACKTRACE 1 +# else +# if ELPP_COMPILER_MSVC +# pragma message("Stack trace not available for this compiler") +# else +# warning "Stack trace not available for this compiler"; +# endif // ELPP_COMPILER_MSVC +# define ELPP_STACKTRACE 0 +# endif // ELPP_COMPILER_GCC +#else +# define ELPP_STACKTRACE 0 +#endif // (defined(ELPP_FEATURE_ALL)) || (defined(ELPP_FEATURE_CRASH_LOG)) +// Miscellaneous macros +#define ELPP_UNUSED(x) (void)x +#if ELPP_OS_UNIX +// Log file permissions for unix-based systems +# define ELPP_LOG_PERMS S_IRUSR | S_IWUSR | S_IXUSR | S_IWGRP | S_IRGRP | S_IXGRP | S_IWOTH | S_IXOTH +#endif // ELPP_OS_UNIX +#if defined(ELPP_AS_DLL) && ELPP_COMPILER_MSVC +# if defined(ELPP_EXPORT_SYMBOLS) +# define ELPP_EXPORT __declspec(dllexport) +# else +# define ELPP_EXPORT __declspec(dllimport) +# endif // defined(ELPP_EXPORT_SYMBOLS) +#else +# define ELPP_EXPORT +#endif // defined(ELPP_AS_DLL) && ELPP_COMPILER_MSVC +// Some special functions that are VC++ specific +#undef STRTOK +#undef STRERROR +#undef STRCAT +#undef STRCPY +#if ELPP_CRT_DBG_WARNINGS +# define STRTOK(a, b, c) strtok_s(a, b, c) +# define STRERROR(a, b, c) strerror_s(a, b, c) +# define STRCAT(a, b, len) strcat_s(a, len, b) +# define STRCPY(a, b, len) strcpy_s(a, len, b) +#else +# define STRTOK(a, b, c) strtok(a, b) +# define STRERROR(a, b, c) strerror(c) +# define STRCAT(a, b, len) strcat(a, b) +# define STRCPY(a, b, len) strcpy(a, b) +#endif +// Compiler specific support evaluations +#if (ELPP_MINGW && !defined(ELPP_FORCE_USE_STD_THREAD)) +# define ELPP_USE_STD_THREADING 0 +#else +# if ((ELPP_COMPILER_CLANG && defined(ELPP_CLANG_SUPPORTS_THREAD)) || \ + (!ELPP_COMPILER_CLANG && defined(ELPP_CXX11)) || \ + defined(ELPP_FORCE_USE_STD_THREAD)) +# define ELPP_USE_STD_THREADING 1 +# else +# define ELPP_USE_STD_THREADING 0 +# endif +#endif +#undef ELPP_FINAL +#if ELPP_COMPILER_INTEL || (ELPP_GCC_VERSION < 40702) +# define ELPP_FINAL +#else +# define ELPP_FINAL final +#endif // ELPP_COMPILER_INTEL || (ELPP_GCC_VERSION < 40702) +#if defined(ELPP_EXPERIMENTAL_ASYNC) +# define ELPP_ASYNC_LOGGING 1 +#else +# define ELPP_ASYNC_LOGGING 0 +#endif // defined(ELPP_EXPERIMENTAL_ASYNC) +#if defined(ELPP_THREAD_SAFE) || ELPP_ASYNC_LOGGING +# define ELPP_THREADING_ENABLED 1 +#else +# define ELPP_THREADING_ENABLED 0 +#endif // defined(ELPP_THREAD_SAFE) || ELPP_ASYNC_LOGGING +// Function macro ELPP_FUNC +#undef ELPP_FUNC +#if ELPP_COMPILER_MSVC // Visual C++ +# define ELPP_FUNC __FUNCSIG__ +#elif ELPP_COMPILER_GCC // GCC +# define ELPP_FUNC __PRETTY_FUNCTION__ +#elif ELPP_COMPILER_INTEL // Intel C++ +# define ELPP_FUNC __PRETTY_FUNCTION__ +#elif ELPP_COMPILER_CLANG // Clang++ +# define ELPP_FUNC __PRETTY_FUNCTION__ +#else +# if defined(__func__) +# define ELPP_FUNC __func__ +# else +# define ELPP_FUNC "" +# endif // defined(__func__) +#endif // defined(_MSC_VER) +#undef ELPP_VARIADIC_TEMPLATES_SUPPORTED +// Keep following line commented until features are fixed +#define ELPP_VARIADIC_TEMPLATES_SUPPORTED \ +(ELPP_COMPILER_GCC || ELPP_COMPILER_CLANG || ELPP_COMPILER_INTEL || (ELPP_COMPILER_MSVC && _MSC_VER >= 1800)) +// Logging Enable/Disable macros +#if defined(ELPP_DISABLE_LOGS) +#define ELPP_LOGGING_ENABLED 0 +#else +#define ELPP_LOGGING_ENABLED 1 +#endif +#if (!defined(ELPP_DISABLE_DEBUG_LOGS) && (ELPP_LOGGING_ENABLED)) +# define ELPP_DEBUG_LOG 1 +#else +# define ELPP_DEBUG_LOG 0 +#endif // (!defined(ELPP_DISABLE_DEBUG_LOGS) && (ELPP_LOGGING_ENABLED)) +#if (!defined(ELPP_DISABLE_INFO_LOGS) && (ELPP_LOGGING_ENABLED)) +# define ELPP_INFO_LOG 1 +#else +# define ELPP_INFO_LOG 0 +#endif // (!defined(ELPP_DISABLE_INFO_LOGS) && (ELPP_LOGGING_ENABLED)) +#if (!defined(ELPP_DISABLE_WARNING_LOGS) && (ELPP_LOGGING_ENABLED)) +# define ELPP_WARNING_LOG 1 +#else +# define ELPP_WARNING_LOG 0 +#endif // (!defined(ELPP_DISABLE_WARNING_LOGS) && (ELPP_LOGGING_ENABLED)) +#if (!defined(ELPP_DISABLE_ERROR_LOGS) && (ELPP_LOGGING_ENABLED)) +# define ELPP_ERROR_LOG 1 +#else +# define ELPP_ERROR_LOG 0 +#endif // (!defined(ELPP_DISABLE_ERROR_LOGS) && (ELPP_LOGGING_ENABLED)) +#if (!defined(ELPP_DISABLE_FATAL_LOGS) && (ELPP_LOGGING_ENABLED)) +# define ELPP_FATAL_LOG 1 +#else +# define ELPP_FATAL_LOG 0 +#endif // (!defined(ELPP_DISABLE_FATAL_LOGS) && (ELPP_LOGGING_ENABLED)) +#if (!defined(ELPP_DISABLE_TRACE_LOGS) && (ELPP_LOGGING_ENABLED)) +# define ELPP_TRACE_LOG 1 +#else +# define ELPP_TRACE_LOG 0 +#endif // (!defined(ELPP_DISABLE_TRACE_LOGS) && (ELPP_LOGGING_ENABLED)) +#if (!defined(ELPP_DISABLE_VERBOSE_LOGS) && (ELPP_LOGGING_ENABLED)) +# define ELPP_VERBOSE_LOG 1 +#else +# define ELPP_VERBOSE_LOG 0 +#endif // (!defined(ELPP_DISABLE_VERBOSE_LOGS) && (ELPP_LOGGING_ENABLED)) +#if (!(ELPP_CXX0X || ELPP_CXX11)) +# error "C++0x (or higher) support not detected! (Is `-std=c++11' missing?)" +#endif // (!(ELPP_CXX0X || ELPP_CXX11)) +// Headers +#if defined(ELPP_SYSLOG) +# include +#endif // defined(ELPP_SYSLOG) +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(ELPP_UNICODE) +# include +# if ELPP_OS_WINDOWS +# include +# endif // ELPP_OS_WINDOWS +#endif // defined(ELPP_UNICODE) +#if ELPP_STACKTRACE +# include +# include +#endif // ELPP_STACKTRACE +#if ELPP_OS_ANDROID +# include +#endif // ELPP_OS_ANDROID +#if ELPP_OS_UNIX +# include +# include +#elif ELPP_OS_WINDOWS +# include +# include +# if defined(WIN32_LEAN_AND_MEAN) +# if defined(ELPP_WINSOCK2) +# include +# else +# include +# endif // defined(ELPP_WINSOCK2) +# endif // defined(WIN32_LEAN_AND_MEAN) +#endif // ELPP_OS_UNIX +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if ELPP_THREADING_ENABLED +# if ELPP_USE_STD_THREADING +# include +# include +# else +# if ELPP_OS_UNIX +# include +# endif // ELPP_OS_UNIX +# endif // ELPP_USE_STD_THREADING +#endif // ELPP_THREADING_ENABLED +#if ELPP_ASYNC_LOGGING +# if defined(ELPP_NO_SLEEP_FOR) +# include +# endif // defined(ELPP_NO_SLEEP_FOR) +# include +# include +# include +#endif // ELPP_ASYNC_LOGGING +#if defined(ELPP_STL_LOGGING) +// For logging STL based templates +# include +# include +# include +# include +# include +# include +# if defined(ELPP_LOG_STD_ARRAY) +# include +# endif // defined(ELPP_LOG_STD_ARRAY) +# if defined(ELPP_LOG_UNORDERED_SET) +# include +# endif // defined(ELPP_UNORDERED_SET) +#endif // defined(ELPP_STL_LOGGING) +#if defined(ELPP_QT_LOGGING) +// For logging Qt based classes & templates +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +#endif // defined(ELPP_QT_LOGGING) +#if defined(ELPP_BOOST_LOGGING) +// For logging boost based classes & templates +# include +# include +# include +# include +# include +# include +# include +# include +#endif // defined(ELPP_BOOST_LOGGING) +#if defined(ELPP_WXWIDGETS_LOGGING) +// For logging wxWidgets based classes & templates +# include +#endif // defined(ELPP_WXWIDGETS_LOGGING) +#if defined(ELPP_UTC_DATETIME) +# define elpptime_r gmtime_r +# define elpptime_s gmtime_s +# define elpptime gmtime +#else +# define elpptime_r localtime_r +# define elpptime_s localtime_s +# define elpptime localtime +#endif // defined(ELPP_UTC_DATETIME) +// Forward declarations +namespace el { +class Logger; +class LogMessage; +class PerformanceTrackingData; +class Loggers; +class Helpers; +template class Callback; +class LogDispatchCallback; +class PerformanceTrackingCallback; +class LoggerRegistrationCallback; +class LogDispatchData; +namespace base { +class Storage; +class RegisteredLoggers; +class PerformanceTracker; +class MessageBuilder; +class Writer; +class PErrorWriter; +class LogDispatcher; +class DefaultLogBuilder; +class DefaultLogDispatchCallback; +#if ELPP_ASYNC_LOGGING +class AsyncLogDispatchCallback; +class AsyncDispatchWorker; +#endif // ELPP_ASYNC_LOGGING +class DefaultPerformanceTrackingCallback; +} // namespace base +} // namespace el +/// @brief Easylogging++ entry namespace +namespace el { +/// @brief Namespace containing base/internal functionality used by Easylogging++ +namespace base { +/// @brief Data types used by Easylogging++ +namespace type { +#undef ELPP_LITERAL +#undef ELPP_STRLEN +#undef ELPP_COUT +#if defined(ELPP_UNICODE) +# define ELPP_LITERAL(txt) L##txt +# define ELPP_STRLEN wcslen +# if defined ELPP_CUSTOM_COUT +# define ELPP_COUT ELPP_CUSTOM_COUT +# else +# define ELPP_COUT std::wcout +# endif // defined ELPP_CUSTOM_COUT +typedef wchar_t char_t; +typedef std::wstring string_t; +typedef std::wstringstream stringstream_t; +typedef std::wfstream fstream_t; +typedef std::wostream ostream_t; +#else +# define ELPP_LITERAL(txt) txt +# define ELPP_STRLEN strlen +# if defined ELPP_CUSTOM_COUT +# define ELPP_COUT ELPP_CUSTOM_COUT +# else +# define ELPP_COUT std::cout +# endif // defined ELPP_CUSTOM_COUT +typedef char char_t; +typedef std::string string_t; +typedef std::stringstream stringstream_t; +typedef std::fstream fstream_t; +typedef std::ostream ostream_t; +#endif // defined(ELPP_UNICODE) +#if defined(ELPP_CUSTOM_COUT_LINE) +# define ELPP_COUT_LINE(logLine) ELPP_CUSTOM_COUT_LINE(logLine) +#else +# define ELPP_COUT_LINE(logLine) logLine << std::flush +#endif // defined(ELPP_CUSTOM_COUT_LINE) +typedef unsigned int EnumType; +typedef unsigned short VerboseLevel; +typedef unsigned long int LineNumber; +typedef std::shared_ptr StoragePointer; +typedef std::shared_ptr LogDispatchCallbackPtr; +typedef std::shared_ptr PerformanceTrackingCallbackPtr; +typedef std::shared_ptr LoggerRegistrationCallbackPtr; +typedef std::unique_ptr PerformanceTrackerPtr; +} // namespace type +/// @brief Internal helper class that prevent copy constructor for class +/// +/// @detail When using this class simply inherit it privately +class NoCopy { + protected: + NoCopy(void) {} + private: + NoCopy(const NoCopy&); + NoCopy& operator=(const NoCopy&); +}; +/// @brief Internal helper class that makes all default constructors private. +/// +/// @detail This prevents initializing class making it static unless an explicit constructor is declared. +/// When using this class simply inherit it privately +class StaticClass { + private: + StaticClass(void); + StaticClass(const StaticClass&); + StaticClass& operator=(const StaticClass&); +}; +} // namespace base +/// @brief Represents enumeration for severity level used to determine level of logging +/// +/// @detail With Easylogging++, developers may disable or enable any level regardless of +/// what the severity is. Or they can choose to log using hierarchical logging flag +enum class Level : base::type::EnumType { + /// @brief Generic level that represents all the levels. Useful when setting global configuration for all levels + Global = 1, + /// @brief Information that can be useful to back-trace certain events - mostly useful than debug logs. + Trace = 2, + /// @brief Informational events most useful for developers to debug application + Debug = 4, + /// @brief Severe error information that will presumably abort application + Fatal = 8, + /// @brief Information representing errors in application but application will keep running + Error = 16, + /// @brief Useful when application has potentially harmful situtaions + Warning = 32, + /// @brief Information that can be highly useful and vary with verbose logging level. + Verbose = 64, + /// @brief Mainly useful to represent current progress of application + Info = 128, + /// @brief Represents unknown level + Unknown = 1010 +}; +} // namespace el +namespace std { +template<> struct hash { + public: + std::size_t operator()(const el::Level& l) const { + return hash {}(static_cast(l)); + } +}; +} +namespace el { +/// @brief Static class that contains helper functions for el::Level +class LevelHelper : base::StaticClass { + public: + /// @brief Represents minimum valid level. Useful when iterating through enum. + static const base::type::EnumType kMinValid = static_cast(Level::Trace); + /// @brief Represents maximum valid level. This is used internally and you should not need it. + static const base::type::EnumType kMaxValid = static_cast(Level::Info); + /// @brief Casts level to int, useful for iterating through enum. + static base::type::EnumType castToInt(Level level) { + return static_cast(level); + } + /// @brief Casts int(ushort) to level, useful for iterating through enum. + static Level castFromInt(base::type::EnumType l) { + return static_cast(l); + } + /// @brief Converts level to associated const char* + /// @return Upper case string based level. + static const char* convertToString(Level level); + /// @brief Converts from levelStr to Level + /// @param levelStr Upper case string based level. + /// Lower case is also valid but providing upper case is recommended. + static Level convertFromString(const char* levelStr); + /// @brief Applies specified function to each level starting from startIndex + /// @param startIndex initial value to start the iteration from. This is passed as pointer and + /// is left-shifted so this can be used inside function (fn) to represent current level. + /// @param fn function to apply with each level. This bool represent whether or not to stop iterating through levels. + static void forEachLevel(base::type::EnumType* startIndex, const std::function& fn); +}; +/// @brief Represents enumeration of ConfigurationType used to configure or access certain aspect +/// of logging +enum class ConfigurationType : base::type::EnumType { + /// @brief Determines whether or not corresponding level and logger of logging is enabled + /// You may disable all logs by using el::Level::Global + Enabled = 1, + /// @brief Whether or not to write corresponding log to log file + ToFile = 2, + /// @brief Whether or not to write corresponding level and logger log to standard output. + /// By standard output meaning termnal, command prompt etc + ToStandardOutput = 4, + /// @brief Determines format of logging corresponding level and logger. + Format = 8, + /// @brief Determines log file (full path) to write logs to for correponding level and logger + Filename = 16, + /// @brief Specifies precision of the subsecond part. It should be within range (1-6). + SubsecondPrecision = 32, + /// @brief Alias of SubsecondPrecision (for backward compatibility) + MillisecondsWidth = SubsecondPrecision, + /// @brief Determines whether or not performance tracking is enabled. + /// + /// @detail This does not depend on logger or level. Performance tracking always uses 'performance' logger + PerformanceTracking = 64, + /// @brief Specifies log file max size. + /// + /// @detail If file size of corresponding log file (for corresponding level) is >= specified size, log file will + /// be truncated and re-initiated. + MaxLogFileSize = 128, + /// @brief Specifies number of log entries to hold until we flush pending log data + LogFlushThreshold = 256, + /// @brief Represents unknown configuration + Unknown = 1010 +}; +/// @brief Static class that contains helper functions for el::ConfigurationType +class ConfigurationTypeHelper : base::StaticClass { + public: + /// @brief Represents minimum valid configuration type. Useful when iterating through enum. + static const base::type::EnumType kMinValid = static_cast(ConfigurationType::Enabled); + /// @brief Represents maximum valid configuration type. This is used internally and you should not need it. + static const base::type::EnumType kMaxValid = static_cast(ConfigurationType::MaxLogFileSize); + /// @brief Casts configuration type to int, useful for iterating through enum. + static base::type::EnumType castToInt(ConfigurationType configurationType) { + return static_cast(configurationType); + } + /// @brief Casts int(ushort) to configurationt type, useful for iterating through enum. + static ConfigurationType castFromInt(base::type::EnumType c) { + return static_cast(c); + } + /// @brief Converts configuration type to associated const char* + /// @returns Upper case string based configuration type. + static const char* convertToString(ConfigurationType configurationType); + /// @brief Converts from configStr to ConfigurationType + /// @param configStr Upper case string based configuration type. + /// Lower case is also valid but providing upper case is recommended. + static ConfigurationType convertFromString(const char* configStr); + /// @brief Applies specified function to each configuration type starting from startIndex + /// @param startIndex initial value to start the iteration from. This is passed by pointer and is left-shifted + /// so this can be used inside function (fn) to represent current configuration type. + /// @param fn function to apply with each configuration type. + /// This bool represent whether or not to stop iterating through configurations. + static inline void forEachConfigType(base::type::EnumType* startIndex, const std::function& fn); +}; +/// @brief Flags used while writing logs. This flags are set by user +enum class LoggingFlag : base::type::EnumType { + /// @brief Makes sure we have new line for each container log entry + NewLineForContainer = 1, + /// @brief Makes sure if -vmodule is used and does not specifies a module, then verbose + /// logging is allowed via that module. + AllowVerboseIfModuleNotSpecified = 2, + /// @brief When handling crashes by default, detailed crash reason will be logged as well + LogDetailedCrashReason = 4, + /// @brief Allows to disable application abortion when logged using FATAL level + DisableApplicationAbortOnFatalLog = 8, + /// @brief Flushes log with every log-entry (performance sensative) - Disabled by default + ImmediateFlush = 16, + /// @brief Enables strict file rolling + StrictLogFileSizeCheck = 32, + /// @brief Make terminal output colorful for supported terminals + ColoredTerminalOutput = 64, + /// @brief Supports use of multiple logging in same macro, e.g, CLOG(INFO, "default", "network") + MultiLoggerSupport = 128, + /// @brief Disables comparing performance tracker's checkpoints + DisablePerformanceTrackingCheckpointComparison = 256, + /// @brief Disable VModules + DisableVModules = 512, + /// @brief Disable VModules extensions + DisableVModulesExtensions = 1024, + /// @brief Enables hierarchical logging + HierarchicalLogging = 2048, + /// @brief Creates logger automatically when not available + CreateLoggerAutomatically = 4096, + /// @brief Adds spaces b/w logs that separated by left-shift operator + AutoSpacing = 8192, + /// @brief Preserves time format and does not convert it to sec, hour etc (performance tracking only) + FixedTimeFormat = 16384, + // @brief Ignore SIGINT or crash + IgnoreSigInt = 32768, +}; +namespace base { +/// @brief Namespace containing constants used internally. +namespace consts { +static const char kFormatSpecifierCharValue = 'v'; +static const char kFormatSpecifierChar = '%'; +static const unsigned int kMaxLogPerCounter = 100000; +static const unsigned int kMaxLogPerContainer = 100; +static const unsigned int kDefaultSubsecondPrecision = 3; + +#ifdef ELPP_DEFAULT_LOGGER +static const char* kDefaultLoggerId = ELPP_DEFAULT_LOGGER; +#else +static const char* kDefaultLoggerId = "default"; +#endif + +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) +#ifdef ELPP_DEFAULT_PERFORMANCE_LOGGER +static const char* kPerformanceLoggerId = ELPP_DEFAULT_PERFORMANCE_LOGGER; +#else +static const char* kPerformanceLoggerId = "performance"; +#endif // ELPP_DEFAULT_PERFORMANCE_LOGGER +#endif + +#if defined(ELPP_SYSLOG) +static const char* kSysLogLoggerId = "syslog"; +#endif // defined(ELPP_SYSLOG) + +#if ELPP_OS_WINDOWS +static const char* kFilePathSeperator = "\\"; +#else +static const char* kFilePathSeperator = "/"; +#endif // ELPP_OS_WINDOWS + +static const std::size_t kSourceFilenameMaxLength = 100; +static const std::size_t kSourceLineMaxLength = 10; +static const Level kPerformanceTrackerDefaultLevel = Level::Info; +const struct { + double value; + const base::type::char_t* unit; +} kTimeFormats[] = { + { 1000.0f, ELPP_LITERAL("us") }, + { 1000.0f, ELPP_LITERAL("ms") }, + { 60.0f, ELPP_LITERAL("seconds") }, + { 60.0f, ELPP_LITERAL("minutes") }, + { 24.0f, ELPP_LITERAL("hours") }, + { 7.0f, ELPP_LITERAL("days") } +}; +static const int kTimeFormatsCount = sizeof(kTimeFormats) / sizeof(kTimeFormats[0]); +const struct { + int numb; + const char* name; + const char* brief; + const char* detail; +} kCrashSignals[] = { + // NOTE: Do not re-order, if you do please check CrashHandler(bool) constructor and CrashHandler::setHandler(..) + { + SIGABRT, "SIGABRT", "Abnormal termination", + "Program was abnormally terminated." + }, + { + SIGFPE, "SIGFPE", "Erroneous arithmetic operation", + "Arithemetic operation issue such as division by zero or operation resulting in overflow." + }, + { + SIGILL, "SIGILL", "Illegal instruction", + "Generally due to a corruption in the code or to an attempt to execute data." + }, + { + SIGSEGV, "SIGSEGV", "Invalid access to memory", + "Program is trying to read an invalid (unallocated, deleted or corrupted) or inaccessible memory." + }, + { + SIGINT, "SIGINT", "Interactive attention signal", + "Interruption generated (generally) by user or operating system." + }, +}; +static const int kCrashSignalsCount = sizeof(kCrashSignals) / sizeof(kCrashSignals[0]); +} // namespace consts +} // namespace base +typedef std::function PreRollOutCallback; +namespace base { +static inline void defaultPreRollOutCallback(const char*, std::size_t, Level level) {} +/// @brief Enum to represent timestamp unit +enum class TimestampUnit : base::type::EnumType { + Microsecond = 0, Millisecond = 1, Second = 2, Minute = 3, Hour = 4, Day = 5 +}; +/// @brief Format flags used to determine specifiers that are active for performance improvements. +enum class FormatFlags : base::type::EnumType { + DateTime = 1 << 1, + LoggerId = 1 << 2, + File = 1 << 3, + Line = 1 << 4, + Location = 1 << 5, + Function = 1 << 6, + User = 1 << 7, + Host = 1 << 8, + LogMessage = 1 << 9, + VerboseLevel = 1 << 10, + AppName = 1 << 11, + ThreadId = 1 << 12, + Level = 1 << 13, + FileBase = 1 << 14, + LevelShort = 1 << 15 +}; +/// @brief A subsecond precision class containing actual width and offset of the subsecond part +class SubsecondPrecision { + public: + SubsecondPrecision(void) { + init(base::consts::kDefaultSubsecondPrecision); + } + explicit SubsecondPrecision(int width) { + init(width); + } + bool operator==(const SubsecondPrecision& ssPrec) { + return m_width == ssPrec.m_width && m_offset == ssPrec.m_offset; + } + int m_width; + unsigned int m_offset; + private: + void init(int width); +}; +/// @brief Type alias of SubsecondPrecision +typedef SubsecondPrecision MillisecondsWidth; +/// @brief Namespace containing utility functions/static classes used internally +namespace utils { +/// @brief Deletes memory safely and points to null +template +static +typename std::enable_if::value, void>::type +safeDelete(T*& pointer) { + if (pointer == nullptr) + return; + delete pointer; + pointer = nullptr; +} +/// @brief Bitwise operations for C++11 strong enum class. This casts e into Flag_T and returns value after bitwise operation +/// Use these function as
flag = bitwise::Or(MyEnum::val1, flag);
+namespace bitwise { +template +static inline base::type::EnumType And(Enum e, base::type::EnumType flag) { + return static_cast(flag) & static_cast(e); +} +template +static inline base::type::EnumType Not(Enum e, base::type::EnumType flag) { + return static_cast(flag) & ~(static_cast(e)); +} +template +static inline base::type::EnumType Or(Enum e, base::type::EnumType flag) { + return static_cast(flag) | static_cast(e); +} +} // namespace bitwise +template +static inline void addFlag(Enum e, base::type::EnumType* flag) { + *flag = base::utils::bitwise::Or(e, *flag); +} +template +static inline void removeFlag(Enum e, base::type::EnumType* flag) { + *flag = base::utils::bitwise::Not(e, *flag); +} +template +static inline bool hasFlag(Enum e, base::type::EnumType flag) { + return base::utils::bitwise::And(e, flag) > 0x0; +} +} // namespace utils +namespace threading { +#if ELPP_THREADING_ENABLED +# if !ELPP_USE_STD_THREADING +namespace internal { +/// @brief A mutex wrapper for compiler that dont yet support std::recursive_mutex +class Mutex : base::NoCopy { + public: + Mutex(void) { +# if ELPP_OS_UNIX + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&m_underlyingMutex, &attr); + pthread_mutexattr_destroy(&attr); +# elif ELPP_OS_WINDOWS + InitializeCriticalSection(&m_underlyingMutex); +# endif // ELPP_OS_UNIX + } + + virtual ~Mutex(void) { +# if ELPP_OS_UNIX + pthread_mutex_destroy(&m_underlyingMutex); +# elif ELPP_OS_WINDOWS + DeleteCriticalSection(&m_underlyingMutex); +# endif // ELPP_OS_UNIX + } + + inline void lock(void) { +# if ELPP_OS_UNIX + pthread_mutex_lock(&m_underlyingMutex); +# elif ELPP_OS_WINDOWS + EnterCriticalSection(&m_underlyingMutex); +# endif // ELPP_OS_UNIX + } + + inline bool try_lock(void) { +# if ELPP_OS_UNIX + return (pthread_mutex_trylock(&m_underlyingMutex) == 0); +# elif ELPP_OS_WINDOWS + return TryEnterCriticalSection(&m_underlyingMutex); +# endif // ELPP_OS_UNIX + } + + inline void unlock(void) { +# if ELPP_OS_UNIX + pthread_mutex_unlock(&m_underlyingMutex); +# elif ELPP_OS_WINDOWS + LeaveCriticalSection(&m_underlyingMutex); +# endif // ELPP_OS_UNIX + } + + private: +# if ELPP_OS_UNIX + pthread_mutex_t m_underlyingMutex; +# elif ELPP_OS_WINDOWS + CRITICAL_SECTION m_underlyingMutex; +# endif // ELPP_OS_UNIX +}; +/// @brief Scoped lock for compiler that dont yet support std::lock_guard +template +class ScopedLock : base::NoCopy { + public: + explicit ScopedLock(M& mutex) { + m_mutex = &mutex; + m_mutex->lock(); + } + + virtual ~ScopedLock(void) { + m_mutex->unlock(); + } + private: + M* m_mutex; + ScopedLock(void); +}; +} // namespace internal +typedef base::threading::internal::Mutex Mutex; +typedef base::threading::internal::ScopedLock ScopedLock; +# else +typedef std::recursive_mutex Mutex; +typedef std::lock_guard ScopedLock; +# endif // !ELPP_USE_STD_THREADING +#else +namespace internal { +/// @brief Mutex wrapper used when multi-threading is disabled. +class NoMutex : base::NoCopy { + public: + NoMutex(void) {} + inline void lock(void) {} + inline bool try_lock(void) { + return true; + } + inline void unlock(void) {} +}; +/// @brief Lock guard wrapper used when multi-threading is disabled. +template +class NoScopedLock : base::NoCopy { + public: + explicit NoScopedLock(Mutex&) { + } + virtual ~NoScopedLock(void) { + } + private: + NoScopedLock(void); +}; +} // namespace internal +typedef base::threading::internal::NoMutex Mutex; +typedef base::threading::internal::NoScopedLock ScopedLock; +#endif // ELPP_THREADING_ENABLED +/// @brief Base of thread safe class, this class is inheritable-only +class ThreadSafe { + public: + virtual inline void acquireLock(void) ELPP_FINAL { m_mutex.lock(); } + virtual inline void releaseLock(void) ELPP_FINAL { m_mutex.unlock(); } + virtual inline base::threading::Mutex& lock(void) ELPP_FINAL { return m_mutex; } + protected: + ThreadSafe(void) {} + virtual ~ThreadSafe(void) {} + private: + base::threading::Mutex m_mutex; +}; + +#if ELPP_THREADING_ENABLED +# if !ELPP_USE_STD_THREADING +/// @brief Gets ID of currently running threading in windows systems. On unix, nothing is returned. +static std::string getCurrentThreadId(void) { + std::stringstream ss; +# if (ELPP_OS_WINDOWS) + ss << GetCurrentThreadId(); +# endif // (ELPP_OS_WINDOWS) + return ss.str(); +} +# else +/// @brief Gets ID of currently running threading using std::this_thread::get_id() +static std::string getCurrentThreadId(void) { + std::stringstream ss; + ss << std::this_thread::get_id(); + return ss.str(); +} +# endif // !ELPP_USE_STD_THREADING +#else +static inline std::string getCurrentThreadId(void) { + return std::string(); +} +#endif // ELPP_THREADING_ENABLED +} // namespace threading +namespace utils { +class File : base::StaticClass { + public: + /// @brief Creates new out file stream for specified filename. + /// @return Pointer to newly created fstream or nullptr + static base::type::fstream_t* newFileStream(const std::string& filename); + + /// @brief Gets size of file provided in stream + static std::size_t getSizeOfFile(base::type::fstream_t* fs); + + /// @brief Determines whether or not provided path exist in current file system + static bool pathExists(const char* path, bool considerFile = false); + + /// @brief Creates specified path on file system + /// @param path Path to create. + static bool createPath(const std::string& path); + /// @brief Extracts path of filename with leading slash + static std::string extractPathFromFilename(const std::string& fullPath, + const char* seperator = base::consts::kFilePathSeperator); + /// @brief builds stripped filename and puts it in buff + static void buildStrippedFilename(const char* filename, char buff[], + std::size_t limit = base::consts::kSourceFilenameMaxLength); + /// @brief builds base filename and puts it in buff + static void buildBaseFilename(const std::string& fullPath, char buff[], + std::size_t limit = base::consts::kSourceFilenameMaxLength, + const char* seperator = base::consts::kFilePathSeperator); +}; +/// @brief String utilities helper class used internally. You should not use it. +class Str : base::StaticClass { + public: + /// @brief Checks if character is digit. Dont use libc implementation of it to prevent locale issues. + static inline bool isDigit(char c) { + return c >= '0' && c <= '9'; + } + + /// @brief Matches wildcards, '*' and '?' only supported. + static bool wildCardMatch(const char* str, const char* pattern); + + static std::string& ltrim(std::string& str); + static std::string& rtrim(std::string& str); + static std::string& trim(std::string& str); + + /// @brief Determines whether or not str starts with specified string + /// @param str String to check + /// @param start String to check against + /// @return Returns true if starts with specified string, false otherwise + static bool startsWith(const std::string& str, const std::string& start); + + /// @brief Determines whether or not str ends with specified string + /// @param str String to check + /// @param end String to check against + /// @return Returns true if ends with specified string, false otherwise + static bool endsWith(const std::string& str, const std::string& end); + + /// @brief Replaces all instances of replaceWhat with 'replaceWith'. Original variable is changed for performance. + /// @param [in,out] str String to replace from + /// @param replaceWhat Character to replace + /// @param replaceWith Character to replace with + /// @return Modified version of str + static std::string& replaceAll(std::string& str, char replaceWhat, char replaceWith); + + /// @brief Replaces all instances of 'replaceWhat' with 'replaceWith'. (String version) Replaces in place + /// @param str String to replace from + /// @param replaceWhat Character to replace + /// @param replaceWith Character to replace with + /// @return Modified (original) str + static std::string& replaceAll(std::string& str, const std::string& replaceWhat, + const std::string& replaceWith); + + static void replaceFirstWithEscape(base::type::string_t& str, const base::type::string_t& replaceWhat, + const base::type::string_t& replaceWith); +#if defined(ELPP_UNICODE) + static void replaceFirstWithEscape(base::type::string_t& str, const base::type::string_t& replaceWhat, + const std::string& replaceWith); +#endif // defined(ELPP_UNICODE) + /// @brief Converts string to uppercase + /// @param str String to convert + /// @return Uppercase string + static std::string& toUpper(std::string& str); + + /// @brief Compares cstring equality - uses strcmp + static bool cStringEq(const char* s1, const char* s2); + + /// @brief Compares cstring equality (case-insensitive) - uses toupper(char) + /// Dont use strcasecmp because of CRT (VC++) + static bool cStringCaseEq(const char* s1, const char* s2); + + /// @brief Returns true if c exist in str + static bool contains(const char* str, char c); + + static char* convertAndAddToBuff(std::size_t n, int len, char* buf, const char* bufLim, bool zeroPadded = true); + static char* addToBuff(const char* str, char* buf, const char* bufLim); + static char* clearBuff(char buff[], std::size_t lim); + + /// @brief Converst wchar* to char* + /// NOTE: Need to free return value after use! + static char* wcharPtrToCharPtr(const wchar_t* line); +}; +/// @brief Operating System helper static class used internally. You should not use it. +class OS : base::StaticClass { + public: +#if ELPP_OS_WINDOWS + /// @brief Gets environment variables for Windows based OS. + /// We are not using getenv(const char*) because of CRT deprecation + /// @param varname Variable name to get environment variable value for + /// @return If variable exist the value of it otherwise nullptr + static const char* getWindowsEnvironmentVariable(const char* varname); +#endif // ELPP_OS_WINDOWS +#if ELPP_OS_ANDROID + /// @brief Reads android property value + static std::string getProperty(const char* prop); + + /// @brief Reads android device name + static std::string getDeviceName(void); +#endif // ELPP_OS_ANDROID + + /// @brief Runs command on terminal and returns the output. + /// + /// @detail This is applicable only on unix based systems, for all other OS, an empty string is returned. + /// @param command Bash command + /// @return Result of bash output or empty string if no result found. + static const std::string getBashOutput(const char* command); + + /// @brief Gets environment variable. This is cross-platform and CRT safe (for VC++) + /// @param variableName Environment variable name + /// @param defaultVal If no environment variable or value found the value to return by default + /// @param alternativeBashCommand If environment variable not found what would be alternative bash command + /// in order to look for value user is looking for. E.g, for 'user' alternative command will 'whoami' + static std::string getEnvironmentVariable(const char* variableName, const char* defaultVal, + const char* alternativeBashCommand = nullptr); + /// @brief Gets current username. + static std::string currentUser(void); + + /// @brief Gets current host name or computer name. + /// + /// @detail For android systems this is device name with its manufacturer and model seperated by hyphen + static std::string currentHost(void); + /// @brief Whether or not terminal supports colors + static bool termSupportsColor(void); +}; +/// @brief Contains utilities for cross-platform date/time. This class make use of el::base::utils::Str +class DateTime : base::StaticClass { + public: + /// @brief Cross platform gettimeofday for Windows and unix platform. This can be used to determine current microsecond. + /// + /// @detail For unix system it uses gettimeofday(timeval*, timezone*) and for Windows, a seperate implementation is provided + /// @param [in,out] tv Pointer that gets updated + static void gettimeofday(struct timeval* tv); + + /// @brief Gets current date and time with a subsecond part. + /// @param format User provided date/time format + /// @param ssPrec A pointer to base::SubsecondPrecision from configuration (non-null) + /// @returns string based date time in specified format. + static std::string getDateTime(const char* format, const base::SubsecondPrecision* ssPrec); + + /// @brief Converts timeval (struct from ctime) to string using specified format and subsecond precision + static std::string timevalToString(struct timeval tval, const char* format, + const el::base::SubsecondPrecision* ssPrec); + + /// @brief Formats time to get unit accordingly, units like second if > 1000 or minutes if > 60000 etc + static base::type::string_t formatTime(unsigned long long time, base::TimestampUnit timestampUnit); + + /// @brief Gets time difference in milli/micro second depending on timestampUnit + static unsigned long long getTimeDifference(const struct timeval& endTime, const struct timeval& startTime, + base::TimestampUnit timestampUnit); + + + static struct ::tm* buildTimeInfo(struct timeval* currTime, struct ::tm* timeInfo); + private: + static char* parseFormat(char* buf, std::size_t bufSz, const char* format, const struct tm* tInfo, + std::size_t msec, const base::SubsecondPrecision* ssPrec); +}; +/// @brief Command line arguments for application if specified using el::Helpers::setArgs(..) or START_EASYLOGGINGPP(..) +class CommandLineArgs { + public: + CommandLineArgs(void) { + setArgs(0, static_cast(nullptr)); + } + CommandLineArgs(int argc, const char** argv) { + setArgs(argc, argv); + } + CommandLineArgs(int argc, char** argv) { + setArgs(argc, argv); + } + virtual ~CommandLineArgs(void) {} + /// @brief Sets arguments and parses them + inline void setArgs(int argc, const char** argv) { + setArgs(argc, const_cast(argv)); + } + /// @brief Sets arguments and parses them + void setArgs(int argc, char** argv); + /// @brief Returns true if arguments contain paramKey with a value (seperated by '=') + bool hasParamWithValue(const char* paramKey) const; + /// @brief Returns value of arguments + /// @see hasParamWithValue(const char*) + const char* getParamValue(const char* paramKey) const; + /// @brief Return true if arguments has a param (not having a value) i,e without '=' + bool hasParam(const char* paramKey) const; + /// @brief Returns true if no params available. This exclude argv[0] + bool empty(void) const; + /// @brief Returns total number of arguments. This exclude argv[0] + std::size_t size(void) const; + friend base::type::ostream_t& operator<<(base::type::ostream_t& os, const CommandLineArgs& c); + + private: + int m_argc; + char** m_argv; + std::unordered_map m_paramsWithValue; + std::vector m_params; +}; +/// @brief Abstract registry (aka repository) that provides basic interface for pointer repository specified by T_Ptr type. +/// +/// @detail Most of the functions are virtual final methods but anything implementing this abstract class should implement +/// unregisterAll() and deepCopy(const AbstractRegistry&) and write registerNew() method according to container +/// and few more methods; get() to find element, unregister() to unregister single entry. +/// Please note that this is thread-unsafe and should also implement thread-safety mechanisms in implementation. +template +class AbstractRegistry : public base::threading::ThreadSafe { + public: + typedef typename Container::iterator iterator; + typedef typename Container::const_iterator const_iterator; + + /// @brief Default constructor + AbstractRegistry(void) {} + + /// @brief Move constructor that is useful for base classes + AbstractRegistry(AbstractRegistry&& sr) { + if (this == &sr) { + return; + } + unregisterAll(); + m_list = std::move(sr.m_list); + } + + bool operator==(const AbstractRegistry& other) { + if (size() != other.size()) { + return false; + } + for (std::size_t i = 0; i < m_list.size(); ++i) { + if (m_list.at(i) != other.m_list.at(i)) { + return false; + } + } + return true; + } + + bool operator!=(const AbstractRegistry& other) { + if (size() != other.size()) { + return true; + } + for (std::size_t i = 0; i < m_list.size(); ++i) { + if (m_list.at(i) != other.m_list.at(i)) { + return true; + } + } + return false; + } + + /// @brief Assignment move operator + AbstractRegistry& operator=(AbstractRegistry&& sr) { + if (this == &sr) { + return *this; + } + unregisterAll(); + m_list = std::move(sr.m_list); + return *this; + } + + virtual ~AbstractRegistry(void) { + } + + /// @return Iterator pointer from start of repository + virtual inline iterator begin(void) ELPP_FINAL { + return m_list.begin(); + } + + /// @return Iterator pointer from end of repository + virtual inline iterator end(void) ELPP_FINAL { + return m_list.end(); + } + + + /// @return Constant iterator pointer from start of repository + virtual inline const_iterator cbegin(void) const ELPP_FINAL { + return m_list.cbegin(); + } + + /// @return End of repository + virtual inline const_iterator cend(void) const ELPP_FINAL { + return m_list.cend(); + } + + /// @return Whether or not repository is empty + virtual inline bool empty(void) const ELPP_FINAL { + return m_list.empty(); + } + + /// @return Size of repository + virtual inline std::size_t size(void) const ELPP_FINAL { + return m_list.size(); + } + + /// @brief Returns underlying container by reference + virtual inline Container& list(void) ELPP_FINAL { + return m_list; + } + + /// @brief Returns underlying container by constant reference. + virtual inline const Container& list(void) const ELPP_FINAL { + return m_list; + } + + /// @brief Unregisters all the pointers from current repository. + virtual void unregisterAll(void) = 0; + + protected: + virtual void deepCopy(const AbstractRegistry&) = 0; + void reinitDeepCopy(const AbstractRegistry& sr) { + unregisterAll(); + deepCopy(sr); + } + + private: + Container m_list; +}; + +/// @brief A pointer registry mechanism to manage memory and provide search functionalities. (non-predicate version) +/// +/// @detail NOTE: This is thread-unsafe implementation (although it contains lock function, it does not use these functions) +/// of AbstractRegistry. Any implementation of this class should be +/// explicitly (by using lock functions) +template +class Registry : public AbstractRegistry> { + public: + typedef typename Registry::iterator iterator; + typedef typename Registry::const_iterator const_iterator; + + Registry(void) {} + + /// @brief Copy constructor that is useful for base classes. Try to avoid this constructor, use move constructor. + Registry(const Registry& sr) : AbstractRegistry>() { + if (this == &sr) { + return; + } + this->reinitDeepCopy(sr); + } + + /// @brief Assignment operator that unregisters all the existing registeries and deeply copies each of repo element + /// @see unregisterAll() + /// @see deepCopy(const AbstractRegistry&) + Registry& operator=(const Registry& sr) { + if (this == &sr) { + return *this; + } + this->reinitDeepCopy(sr); + return *this; + } + + virtual ~Registry(void) { + unregisterAll(); + } + + protected: + virtual void unregisterAll(void) ELPP_FINAL { + if (!this->empty()) { + for (auto&& curr : this->list()) { + base::utils::safeDelete(curr.second); + } + this->list().clear(); + } + } + +/// @brief Registers new registry to repository. + virtual void registerNew(const T_Key& uniqKey, T_Ptr* ptr) ELPP_FINAL { + unregister(uniqKey); + this->list().insert(std::make_pair(uniqKey, ptr)); + } + +/// @brief Unregisters single entry mapped to specified unique key + void unregister(const T_Key& uniqKey) { + T_Ptr* existing = get(uniqKey); + if (existing != nullptr) { + this->list().erase(uniqKey); + base::utils::safeDelete(existing); + } + } + +/// @brief Gets pointer from repository. If none found, nullptr is returned. + T_Ptr* get(const T_Key& uniqKey) { + iterator it = this->list().find(uniqKey); + return it == this->list().end() + ? nullptr + : it->second; + } + + private: + virtual void deepCopy(const AbstractRegistry>& sr) ELPP_FINAL { + for (const_iterator it = sr.cbegin(); it != sr.cend(); ++it) { + registerNew(it->first, new T_Ptr(*it->second)); + } + } +}; + +/// @brief A pointer registry mechanism to manage memory and provide search functionalities. (predicate version) +/// +/// @detail NOTE: This is thread-unsafe implementation of AbstractRegistry. Any implementation of this class +/// should be made thread-safe explicitly +template +class RegistryWithPred : public AbstractRegistry> { + public: + typedef typename RegistryWithPred::iterator iterator; + typedef typename RegistryWithPred::const_iterator const_iterator; + + RegistryWithPred(void) { + } + + virtual ~RegistryWithPred(void) { + unregisterAll(); + } + + /// @brief Copy constructor that is useful for base classes. Try to avoid this constructor, use move constructor. + RegistryWithPred(const RegistryWithPred& sr) : AbstractRegistry>() { + if (this == &sr) { + return; + } + this->reinitDeepCopy(sr); + } + + /// @brief Assignment operator that unregisters all the existing registeries and deeply copies each of repo element + /// @see unregisterAll() + /// @see deepCopy(const AbstractRegistry&) + RegistryWithPred& operator=(const RegistryWithPred& sr) { + if (this == &sr) { + return *this; + } + this->reinitDeepCopy(sr); + return *this; + } + + friend base::type::ostream_t& operator<<(base::type::ostream_t& os, const RegistryWithPred& sr) { + for (const_iterator it = sr.list().begin(); it != sr.list().end(); ++it) { + os << ELPP_LITERAL(" ") << **it << ELPP_LITERAL("\n"); + } + return os; + } + + protected: + virtual void unregisterAll(void) ELPP_FINAL { + if (!this->empty()) { + for (auto&& curr : this->list()) { + base::utils::safeDelete(curr); + } + this->list().clear(); + } + } + + virtual void unregister(T_Ptr*& ptr) ELPP_FINAL { + if (ptr) { + iterator iter = this->begin(); + for (; iter != this->end(); ++iter) { + if (ptr == *iter) { + break; + } + } + if (iter != this->end() && *iter != nullptr) { + this->list().erase(iter); + base::utils::safeDelete(*iter); + } + } + } + + virtual inline void registerNew(T_Ptr* ptr) ELPP_FINAL { + this->list().push_back(ptr); + } + +/// @brief Gets pointer from repository with speicifed arguments. Arguments are passed to predicate +/// in order to validate pointer. + template + T_Ptr* get(const T& arg1, const T2 arg2) { + iterator iter = std::find_if(this->list().begin(), this->list().end(), Pred(arg1, arg2)); + if (iter != this->list().end() && *iter != nullptr) { + return *iter; + } + return nullptr; + } + + private: + virtual void deepCopy(const AbstractRegistry>& sr) { + for (const_iterator it = sr.list().begin(); it != sr.list().end(); ++it) { + registerNew(new T_Ptr(**it)); + } + } +}; +class Utils { + public: + template + static bool installCallback(const std::string& id, std::unordered_map* mapT) { + if (mapT->find(id) == mapT->end()) { + mapT->insert(std::make_pair(id, TPtr(new T()))); + return true; + } + return false; + } + + template + static void uninstallCallback(const std::string& id, std::unordered_map* mapT) { + if (mapT->find(id) != mapT->end()) { + mapT->erase(id); + } + } + + template + static T* callback(const std::string& id, std::unordered_map* mapT) { + typename std::unordered_map::iterator iter = mapT->find(id); + if (iter != mapT->end()) { + return static_cast(iter->second.get()); + } + return nullptr; + } +}; +} // namespace utils +} // namespace base +/// @brief Base of Easylogging++ friendly class +/// +/// @detail After inheriting this class publicly, implement pure-virtual function `void log(std::ostream&) const` +class Loggable { + public: + virtual ~Loggable(void) {} + virtual void log(el::base::type::ostream_t&) const = 0; + private: + friend inline el::base::type::ostream_t& operator<<(el::base::type::ostream_t& os, const Loggable& loggable) { + loggable.log(os); + return os; + } +}; +namespace base { +/// @brief Represents log format containing flags and date format. This is used internally to start initial log +class LogFormat : public Loggable { + public: + LogFormat(void); + LogFormat(Level level, const base::type::string_t& format); + LogFormat(const LogFormat& logFormat); + LogFormat(LogFormat&& logFormat); + LogFormat& operator=(const LogFormat& logFormat); + virtual ~LogFormat(void) {} + bool operator==(const LogFormat& other); + + /// @brief Updates format to be used while logging. + /// @param userFormat User provided format + void parseFromFormat(const base::type::string_t& userFormat); + + inline Level level(void) const { + return m_level; + } + + inline const base::type::string_t& userFormat(void) const { + return m_userFormat; + } + + inline const base::type::string_t& format(void) const { + return m_format; + } + + inline const std::string& dateTimeFormat(void) const { + return m_dateTimeFormat; + } + + inline base::type::EnumType flags(void) const { + return m_flags; + } + + inline bool hasFlag(base::FormatFlags flag) const { + return base::utils::hasFlag(flag, m_flags); + } + + virtual void log(el::base::type::ostream_t& os) const { + os << m_format; + } + + protected: + /// @brief Updates date time format if available in currFormat. + /// @param index Index where %datetime, %date or %time was found + /// @param [in,out] currFormat current format that is being used to format + virtual void updateDateFormat(std::size_t index, base::type::string_t& currFormat) ELPP_FINAL; + + /// @brief Updates %level from format. This is so that we dont have to do it at log-writing-time. It uses m_format and m_level + virtual void updateFormatSpec(void) ELPP_FINAL; + + inline void addFlag(base::FormatFlags flag) { + base::utils::addFlag(flag, &m_flags); + } + + private: + Level m_level; + base::type::string_t m_userFormat; + base::type::string_t m_format; + std::string m_dateTimeFormat; + base::type::EnumType m_flags; + std::string m_currentUser; + std::string m_currentHost; + friend class el::Logger; // To resolve loggerId format specifier easily +}; +} // namespace base +/// @brief Resolving function for format specifier +typedef std::function FormatSpecifierValueResolver; +/// @brief User-provided custom format specifier +/// @see el::Helpers::installCustomFormatSpecifier +/// @see FormatSpecifierValueResolver +class CustomFormatSpecifier { + public: + CustomFormatSpecifier(const char* formatSpecifier, const FormatSpecifierValueResolver& resolver) : + m_formatSpecifier(formatSpecifier), m_resolver(resolver) {} + inline const char* formatSpecifier(void) const { + return m_formatSpecifier; + } + inline const FormatSpecifierValueResolver& resolver(void) const { + return m_resolver; + } + inline bool operator==(const char* formatSpecifier) { + return strcmp(m_formatSpecifier, formatSpecifier) == 0; + } + + private: + const char* m_formatSpecifier; + FormatSpecifierValueResolver m_resolver; +}; +/// @brief Represents single configuration that has representing level, configuration type and a string based value. +/// +/// @detail String based value means any value either its boolean, integer or string itself, it will be embedded inside quotes +/// and will be parsed later. +/// +/// Consider some examples below: +/// * el::Configuration confEnabledInfo(el::Level::Info, el::ConfigurationType::Enabled, "true"); +/// * el::Configuration confMaxLogFileSizeInfo(el::Level::Info, el::ConfigurationType::MaxLogFileSize, "2048"); +/// * el::Configuration confFilenameInfo(el::Level::Info, el::ConfigurationType::Filename, "/var/log/my.log"); +class Configuration : public Loggable { + public: + Configuration(const Configuration& c); + Configuration& operator=(const Configuration& c); + + virtual ~Configuration(void) { + } + + /// @brief Full constructor used to sets value of configuration + Configuration(Level level, ConfigurationType configurationType, const std::string& value); + + /// @brief Gets level of current configuration + inline Level level(void) const { + return m_level; + } + + /// @brief Gets configuration type of current configuration + inline ConfigurationType configurationType(void) const { + return m_configurationType; + } + + /// @brief Gets string based configuration value + inline const std::string& value(void) const { + return m_value; + } + + /// @brief Set string based configuration value + /// @param value Value to set. Values have to be std::string; For boolean values use "true", "false", for any integral values + /// use them in quotes. They will be parsed when configuring + inline void setValue(const std::string& value) { + m_value = value; + } + + virtual void log(el::base::type::ostream_t& os) const; + + /// @brief Used to find configuration from configuration (pointers) repository. Avoid using it. + class Predicate { + public: + Predicate(Level level, ConfigurationType configurationType); + + bool operator()(const Configuration* conf) const; + + private: + Level m_level; + ConfigurationType m_configurationType; + }; + + private: + Level m_level; + ConfigurationType m_configurationType; + std::string m_value; +}; + +/// @brief Thread-safe Configuration repository +/// +/// @detail This repository represents configurations for all the levels and configuration type mapped to a value. +class Configurations : public base::utils::RegistryWithPred { + public: + /// @brief Default constructor with empty repository + Configurations(void); + + /// @brief Constructor used to set configurations using configuration file. + /// @param configurationFile Full path to configuration file + /// @param useDefaultsForRemaining Lets you set the remaining configurations to default. + /// @param base If provided, this configuration will be based off existing repository that this argument is pointing to. + /// @see parseFromFile(const std::string&, Configurations* base) + /// @see setRemainingToDefault() + Configurations(const std::string& configurationFile, bool useDefaultsForRemaining = true, + Configurations* base = nullptr); + + virtual ~Configurations(void) { + } + + /// @brief Parses configuration from file. + /// @param configurationFile Full path to configuration file + /// @param base Configurations to base new configuration repository off. This value is used when you want to use + /// existing Configurations to base all the values and then set rest of configuration via configuration file. + /// @return True if successfully parsed, false otherwise. You may define 'ELPP_DEBUG_ASSERT_FAILURE' to make sure you + /// do not proceed without successful parse. + bool parseFromFile(const std::string& configurationFile, Configurations* base = nullptr); + + /// @brief Parse configurations from configuration string. + /// + /// @detail This configuration string has same syntax as configuration file contents. Make sure all the necessary + /// new line characters are provided. + /// @param base Configurations to base new configuration repository off. This value is used when you want to use + /// existing Configurations to base all the values and then set rest of configuration via configuration text. + /// @return True if successfully parsed, false otherwise. You may define 'ELPP_DEBUG_ASSERT_FAILURE' to make sure you + /// do not proceed without successful parse. + bool parseFromText(const std::string& configurationsString, Configurations* base = nullptr); + + /// @brief Sets configuration based-off an existing configurations. + /// @param base Pointer to existing configurations. + void setFromBase(Configurations* base); + + /// @brief Determines whether or not specified configuration type exists in the repository. + /// + /// @detail Returns as soon as first level is found. + /// @param configurationType Type of configuration to check existence for. + bool hasConfiguration(ConfigurationType configurationType); + + /// @brief Determines whether or not specified configuration type exists for specified level + /// @param level Level to check + /// @param configurationType Type of configuration to check existence for. + bool hasConfiguration(Level level, ConfigurationType configurationType); + + /// @brief Sets value of configuration for specified level. + /// + /// @detail Any existing configuration for specified level will be replaced. Also note that configuration types + /// ConfigurationType::SubsecondPrecision and ConfigurationType::PerformanceTracking will be ignored if not set for + /// Level::Global because these configurations are not dependant on level. + /// @param level Level to set configuration for (el::Level). + /// @param configurationType Type of configuration (el::ConfigurationType) + /// @param value A string based value. Regardless of what the data type of configuration is, it will always be string + /// from users' point of view. This is then parsed later to be used internally. + /// @see Configuration::setValue(const std::string& value) + /// @see el::Level + /// @see el::ConfigurationType + void set(Level level, ConfigurationType configurationType, const std::string& value); + + /// @brief Sets single configuration based on other single configuration. + /// @see set(Level level, ConfigurationType configurationType, const std::string& value) + void set(Configuration* conf); + + inline Configuration* get(Level level, ConfigurationType configurationType) { + base::threading::ScopedLock scopedLock(lock()); + return RegistryWithPred::get(level, configurationType); + } + + /// @brief Sets configuration for all levels. + /// @param configurationType Type of configuration + /// @param value String based value + /// @see Configurations::set(Level level, ConfigurationType configurationType, const std::string& value) + inline void setGlobally(ConfigurationType configurationType, const std::string& value) { + setGlobally(configurationType, value, false); + } + + /// @brief Clears repository so that all the configurations are unset + inline void clear(void) { + base::threading::ScopedLock scopedLock(lock()); + unregisterAll(); + } + + /// @brief Gets configuration file used in parsing this configurations. + /// + /// @detail If this repository was set manually or by text this returns empty string. + inline const std::string& configurationFile(void) const { + return m_configurationFile; + } + + /// @brief Sets configurations to "factory based" configurations. + void setToDefault(void); + + /// @brief Lets you set the remaining configurations to default. + /// + /// @detail By remaining, it means that the level/type a configuration does not exist for. + /// This function is useful when you want to minimize chances of failures, e.g, if you have a configuration file that sets + /// configuration for all the configurations except for Enabled or not, we use this so that ENABLED is set to default i.e, + /// true. If you dont do this explicitly (either by calling this function or by using second param in Constructor + /// and try to access a value, an error is thrown + void setRemainingToDefault(void); + + /// @brief Parser used internally to parse configurations from file or text. + /// + /// @detail This class makes use of base::utils::Str. + /// You should not need this unless you are working on some tool for Easylogging++ + class Parser : base::StaticClass { + public: + /// @brief Parses configuration from file. + /// @param configurationFile Full path to configuration file + /// @param sender Sender configurations pointer. Usually 'this' is used from calling class + /// @param base Configurations to base new configuration repository off. This value is used when you want to use + /// existing Configurations to base all the values and then set rest of configuration via configuration file. + /// @return True if successfully parsed, false otherwise. You may define '_STOP_ON_FIRSTELPP_ASSERTION' to make sure you + /// do not proceed without successful parse. + static bool parseFromFile(const std::string& configurationFile, Configurations* sender, + Configurations* base = nullptr); + + /// @brief Parse configurations from configuration string. + /// + /// @detail This configuration string has same syntax as configuration file contents. Make sure all the necessary + /// new line characters are provided. You may define '_STOP_ON_FIRSTELPP_ASSERTION' to make sure you + /// do not proceed without successful parse (This is recommended) + /// @param configurationsString the configuration in plain text format + /// @param sender Sender configurations pointer. Usually 'this' is used from calling class + /// @param base Configurations to base new configuration repository off. This value is used when you want to use + /// existing Configurations to base all the values and then set rest of configuration via configuration text. + /// @return True if successfully parsed, false otherwise. + static bool parseFromText(const std::string& configurationsString, Configurations* sender, + Configurations* base = nullptr); + + private: + friend class el::Loggers; + static void ignoreComments(std::string* line); + static bool isLevel(const std::string& line); + static bool isComment(const std::string& line); + static inline bool isConfig(const std::string& line); + static bool parseLine(std::string* line, std::string* currConfigStr, std::string* currLevelStr, Level* currLevel, + Configurations* conf); + }; + + private: + std::string m_configurationFile; + bool m_isFromFile; + friend class el::Loggers; + + /// @brief Unsafely sets configuration if does not already exist + void unsafeSetIfNotExist(Level level, ConfigurationType configurationType, const std::string& value); + + /// @brief Thread unsafe set + void unsafeSet(Level level, ConfigurationType configurationType, const std::string& value); + + /// @brief Sets configurations for all levels including Level::Global if includeGlobalLevel is true + /// @see Configurations::setGlobally(ConfigurationType configurationType, const std::string& value) + void setGlobally(ConfigurationType configurationType, const std::string& value, bool includeGlobalLevel); + + /// @brief Sets configurations (Unsafely) for all levels including Level::Global if includeGlobalLevel is true + /// @see Configurations::setGlobally(ConfigurationType configurationType, const std::string& value) + void unsafeSetGlobally(ConfigurationType configurationType, const std::string& value, bool includeGlobalLevel); +}; + +namespace base { +typedef std::shared_ptr FileStreamPtr; +typedef std::unordered_map LogStreamsReferenceMap; +/// @brief Configurations with data types. +/// +/// @detail el::Configurations have string based values. This is whats used internally in order to read correct configurations. +/// This is to perform faster while writing logs using correct configurations. +/// +/// This is thread safe and final class containing non-virtual destructor (means nothing should inherit this class) +class TypedConfigurations : public base::threading::ThreadSafe { + public: + /// @brief Constructor to initialize (construct) the object off el::Configurations + /// @param configurations Configurations pointer/reference to base this typed configurations off. + /// @param logStreamsReference Use ELPP->registeredLoggers()->logStreamsReference() + TypedConfigurations(Configurations* configurations, base::LogStreamsReferenceMap* logStreamsReference); + + TypedConfigurations(const TypedConfigurations& other); + + virtual ~TypedConfigurations(void) { + } + + const Configurations* configurations(void) const { + return m_configurations; + } + + bool enabled(Level level); + bool toFile(Level level); + const std::string& filename(Level level); + bool toStandardOutput(Level level); + const base::LogFormat& logFormat(Level level); + const base::SubsecondPrecision& subsecondPrecision(Level level = Level::Global); + const base::MillisecondsWidth& millisecondsWidth(Level level = Level::Global); + bool performanceTracking(Level level = Level::Global); + base::type::fstream_t* fileStream(Level level); + std::size_t maxLogFileSize(Level level); + std::size_t logFlushThreshold(Level level); + + private: + Configurations* m_configurations; + std::unordered_map m_enabledMap; + std::unordered_map m_toFileMap; + std::unordered_map m_filenameMap; + std::unordered_map m_toStandardOutputMap; + std::unordered_map m_logFormatMap; + std::unordered_map m_subsecondPrecisionMap; + std::unordered_map m_performanceTrackingMap; + std::unordered_map m_fileStreamMap; + std::unordered_map m_maxLogFileSizeMap; + std::unordered_map m_logFlushThresholdMap; + base::LogStreamsReferenceMap* m_logStreamsReference; + + friend class el::Helpers; + friend class el::base::MessageBuilder; + friend class el::base::Writer; + friend class el::base::DefaultLogDispatchCallback; + friend class el::base::LogDispatcher; + + template + inline Conf_T getConfigByVal(Level level, const std::unordered_map* confMap, const char* confName) { + base::threading::ScopedLock scopedLock(lock()); + return unsafeGetConfigByVal(level, confMap, confName); // This is not unsafe anymore - mutex locked in scope + } + + template + inline Conf_T& getConfigByRef(Level level, std::unordered_map* confMap, const char* confName) { + base::threading::ScopedLock scopedLock(lock()); + return unsafeGetConfigByRef(level, confMap, confName); // This is not unsafe anymore - mutex locked in scope + } + + template + Conf_T unsafeGetConfigByVal(Level level, const std::unordered_map* confMap, const char* confName) { + ELPP_UNUSED(confName); + typename std::unordered_map::const_iterator it = confMap->find(level); + if (it == confMap->end()) { + try { + return confMap->at(Level::Global); + } catch (...) { + ELPP_INTERNAL_ERROR("Unable to get configuration [" << confName << "] for level [" + << LevelHelper::convertToString(level) << "]" + << std::endl << "Please ensure you have properly configured logger.", false); + return Conf_T(); + } + } + return it->second; + } + + template + Conf_T& unsafeGetConfigByRef(Level level, std::unordered_map* confMap, const char* confName) { + ELPP_UNUSED(confName); + typename std::unordered_map::iterator it = confMap->find(level); + if (it == confMap->end()) { + try { + return confMap->at(Level::Global); + } catch (...) { + ELPP_INTERNAL_ERROR("Unable to get configuration [" << confName << "] for level [" + << LevelHelper::convertToString(level) << "]" + << std::endl << "Please ensure you have properly configured logger.", false); + } + } + return it->second; + } + + template + void setValue(Level level, const Conf_T& value, std::unordered_map* confMap, + bool includeGlobalLevel = true) { + // If map is empty and we are allowed to add into generic level (Level::Global), do it! + if (confMap->empty() && includeGlobalLevel) { + confMap->insert(std::make_pair(Level::Global, value)); + return; + } + // If same value exist in generic level already, dont add it to explicit level + typename std::unordered_map::iterator it = confMap->find(Level::Global); + if (it != confMap->end() && it->second == value) { + return; + } + // Now make sure we dont double up values if we really need to add it to explicit level + it = confMap->find(level); + if (it == confMap->end()) { + // Value not found for level, add new + confMap->insert(std::make_pair(level, value)); + } else { + // Value found, just update value + confMap->at(level) = value; + } + } + + void build(Configurations* configurations); + unsigned long getULong(std::string confVal); + std::string resolveFilename(const std::string& filename); + void insertFile(Level level, const std::string& fullFilename); + bool unsafeValidateFileRolling(Level level, const PreRollOutCallback& preRollOutCallback); + + inline bool validateFileRolling(Level level, const PreRollOutCallback& preRollOutCallback) { + base::threading::ScopedLock scopedLock(lock()); + return unsafeValidateFileRolling(level, preRollOutCallback); + } +}; +/// @brief Class that keeps record of current line hit for occasional logging +class HitCounter { + public: + HitCounter(void) : + m_filename(""), + m_lineNumber(0), + m_hitCounts(0) { + } + + HitCounter(const char* filename, base::type::LineNumber lineNumber) : + m_filename(filename), + m_lineNumber(lineNumber), + m_hitCounts(0) { + } + + HitCounter(const HitCounter& hitCounter) : + m_filename(hitCounter.m_filename), + m_lineNumber(hitCounter.m_lineNumber), + m_hitCounts(hitCounter.m_hitCounts) { + } + + HitCounter& operator=(const HitCounter& hitCounter) { + if (&hitCounter != this) { + m_filename = hitCounter.m_filename; + m_lineNumber = hitCounter.m_lineNumber; + m_hitCounts = hitCounter.m_hitCounts; + } + return *this; + } + + virtual ~HitCounter(void) { + } + + /// @brief Resets location of current hit counter + inline void resetLocation(const char* filename, base::type::LineNumber lineNumber) { + m_filename = filename; + m_lineNumber = lineNumber; + } + + /// @brief Validates hit counts and resets it if necessary + inline void validateHitCounts(std::size_t n) { + if (m_hitCounts >= base::consts::kMaxLogPerCounter) { + m_hitCounts = (n >= 1 ? base::consts::kMaxLogPerCounter % n : 0); + } + ++m_hitCounts; + } + + inline const char* filename(void) const { + return m_filename; + } + + inline base::type::LineNumber lineNumber(void) const { + return m_lineNumber; + } + + inline std::size_t hitCounts(void) const { + return m_hitCounts; + } + + inline void increment(void) { + ++m_hitCounts; + } + + class Predicate { + public: + Predicate(const char* filename, base::type::LineNumber lineNumber) + : m_filename(filename), + m_lineNumber(lineNumber) { + } + inline bool operator()(const HitCounter* counter) { + return ((counter != nullptr) && + (strcmp(counter->m_filename, m_filename) == 0) && + (counter->m_lineNumber == m_lineNumber)); + } + + private: + const char* m_filename; + base::type::LineNumber m_lineNumber; + }; + + private: + const char* m_filename; + base::type::LineNumber m_lineNumber; + std::size_t m_hitCounts; +}; +/// @brief Repository for hit counters used across the application +class RegisteredHitCounters : public base::utils::RegistryWithPred { + public: + /// @brief Validates counter for every N, i.e, registers new if does not exist otherwise updates original one + /// @return True if validation resulted in triggering hit. Meaning logs should be written everytime true is returned + bool validateEveryN(const char* filename, base::type::LineNumber lineNumber, std::size_t n); + + /// @brief Validates counter for hits >= N, i.e, registers new if does not exist otherwise updates original one + /// @return True if validation resulted in triggering hit. Meaning logs should be written everytime true is returned + bool validateAfterN(const char* filename, base::type::LineNumber lineNumber, std::size_t n); + + /// @brief Validates counter for hits are <= n, i.e, registers new if does not exist otherwise updates original one + /// @return True if validation resulted in triggering hit. Meaning logs should be written everytime true is returned + bool validateNTimes(const char* filename, base::type::LineNumber lineNumber, std::size_t n); + + /// @brief Gets hit counter registered at specified position + inline const base::HitCounter* getCounter(const char* filename, base::type::LineNumber lineNumber) { + base::threading::ScopedLock scopedLock(lock()); + return get(filename, lineNumber); + } +}; +/// @brief Action to be taken for dispatching +enum class DispatchAction : base::type::EnumType { + None = 1, NormalLog = 2, SysLog = 4 +}; +} // namespace base +template +class Callback : protected base::threading::ThreadSafe { + public: + Callback(void) : m_enabled(true) {} + inline bool enabled(void) const { + return m_enabled; + } + inline void setEnabled(bool enabled) { + base::threading::ScopedLock scopedLock(lock()); + m_enabled = enabled; + } + protected: + virtual void handle(const T* handlePtr) = 0; + private: + bool m_enabled; +}; +class LogDispatchData { + public: + LogDispatchData() : m_logMessage(nullptr), m_dispatchAction(base::DispatchAction::None) {} + inline const LogMessage* logMessage(void) const { + return m_logMessage; + } + inline base::DispatchAction dispatchAction(void) const { + return m_dispatchAction; + } + inline void setLogMessage(LogMessage* logMessage) { + m_logMessage = logMessage; + } + inline void setDispatchAction(base::DispatchAction dispatchAction) { + m_dispatchAction = dispatchAction; + } + private: + LogMessage* m_logMessage; + base::DispatchAction m_dispatchAction; + friend class base::LogDispatcher; + +}; +class LogDispatchCallback : public Callback { + protected: + virtual void handle(const LogDispatchData* data); + base::threading::Mutex& fileHandle(const LogDispatchData* data); + private: + friend class base::LogDispatcher; + std::unordered_map> m_fileLocks; + base::threading::Mutex m_fileLocksMapLock; +}; +class PerformanceTrackingCallback : public Callback { + private: + friend class base::PerformanceTracker; +}; +class LoggerRegistrationCallback : public Callback { + private: + friend class base::RegisteredLoggers; +}; +class LogBuilder : base::NoCopy { + public: + LogBuilder() : m_termSupportsColor(base::utils::OS::termSupportsColor()) {} + virtual ~LogBuilder(void) { + ELPP_INTERNAL_INFO(3, "Destroying log builder...") + } + virtual base::type::string_t build(const LogMessage* logMessage, bool appendNewLine) const = 0; + void convertToColoredOutput(base::type::string_t* logLine, Level level); + private: + bool m_termSupportsColor; + friend class el::base::DefaultLogDispatchCallback; +}; +typedef std::shared_ptr LogBuilderPtr; +/// @brief Represents a logger holding ID and configurations we need to write logs +/// +/// @detail This class does not write logs itself instead its used by writer to read configuations from. +class Logger : public base::threading::ThreadSafe, public Loggable { + public: + Logger(const std::string& id, base::LogStreamsReferenceMap* logStreamsReference); + Logger(const std::string& id, const Configurations& configurations, base::LogStreamsReferenceMap* logStreamsReference); + Logger(const Logger& logger); + Logger& operator=(const Logger& logger); + + virtual ~Logger(void) { + base::utils::safeDelete(m_typedConfigurations); + } + + virtual inline void log(el::base::type::ostream_t& os) const { + os << m_id.c_str(); + } + + /// @brief Configures the logger using specified configurations. + void configure(const Configurations& configurations); + + /// @brief Reconfigures logger using existing configurations + void reconfigure(void); + + inline const std::string& id(void) const { + return m_id; + } + + inline const std::string& parentApplicationName(void) const { + return m_parentApplicationName; + } + + inline void setParentApplicationName(const std::string& parentApplicationName) { + m_parentApplicationName = parentApplicationName; + } + + inline Configurations* configurations(void) { + return &m_configurations; + } + + inline base::TypedConfigurations* typedConfigurations(void) { + return m_typedConfigurations; + } + + static bool isValidId(const std::string& id); + + /// @brief Flushes logger to sync all log files for all levels + void flush(void); + + void flush(Level level, base::type::fstream_t* fs); + + inline bool isFlushNeeded(Level level) { + return ++m_unflushedCount.find(level)->second >= m_typedConfigurations->logFlushThreshold(level); + } + + inline LogBuilder* logBuilder(void) const { + return m_logBuilder.get(); + } + + inline void setLogBuilder(const LogBuilderPtr& logBuilder) { + m_logBuilder = logBuilder; + } + + inline bool enabled(Level level) const { + return m_typedConfigurations->enabled(level); + } + +#if ELPP_VARIADIC_TEMPLATES_SUPPORTED +# define LOGGER_LEVEL_WRITERS_SIGNATURES(FUNCTION_NAME)\ +template \ +inline void FUNCTION_NAME(const char*, const T&, const Args&...);\ +template \ +inline void FUNCTION_NAME(const T&); + + template + inline void verbose(int, const char*, const T&, const Args&...); + + template + inline void verbose(int, const T&); + + LOGGER_LEVEL_WRITERS_SIGNATURES(info) + LOGGER_LEVEL_WRITERS_SIGNATURES(debug) + LOGGER_LEVEL_WRITERS_SIGNATURES(warn) + LOGGER_LEVEL_WRITERS_SIGNATURES(error) + LOGGER_LEVEL_WRITERS_SIGNATURES(fatal) + LOGGER_LEVEL_WRITERS_SIGNATURES(trace) +# undef LOGGER_LEVEL_WRITERS_SIGNATURES +#endif // ELPP_VARIADIC_TEMPLATES_SUPPORTED + private: + std::string m_id; + base::TypedConfigurations* m_typedConfigurations; + base::type::stringstream_t m_stream; + std::string m_parentApplicationName; + bool m_isConfigured; + Configurations m_configurations; + std::unordered_map m_unflushedCount; + base::LogStreamsReferenceMap* m_logStreamsReference; + LogBuilderPtr m_logBuilder; + + friend class el::LogMessage; + friend class el::Loggers; + friend class el::Helpers; + friend class el::base::RegisteredLoggers; + friend class el::base::DefaultLogDispatchCallback; + friend class el::base::MessageBuilder; + friend class el::base::Writer; + friend class el::base::PErrorWriter; + friend class el::base::Storage; + friend class el::base::PerformanceTracker; + friend class el::base::LogDispatcher; + + Logger(void); + +#if ELPP_VARIADIC_TEMPLATES_SUPPORTED + template + void log_(Level, int, const char*, const T&, const Args&...); + + template + inline void log_(Level, int, const T&); + + template + void log(Level, const char*, const T&, const Args&...); + + template + inline void log(Level, const T&); +#endif // ELPP_VARIADIC_TEMPLATES_SUPPORTED + + void initUnflushedCount(void); + + inline base::type::stringstream_t& stream(void) { + return m_stream; + } + + void resolveLoggerFormatSpec(void) const; +}; +namespace base { +/// @brief Loggers repository +class RegisteredLoggers : public base::utils::Registry { + public: + explicit RegisteredLoggers(const LogBuilderPtr& defaultLogBuilder); + + virtual ~RegisteredLoggers(void) { + unsafeFlushAll(); + } + + inline void setDefaultConfigurations(const Configurations& configurations) { + base::threading::ScopedLock scopedLock(lock()); + m_defaultConfigurations.setFromBase(const_cast(&configurations)); + } + + inline Configurations* defaultConfigurations(void) { + return &m_defaultConfigurations; + } + + Logger* get(const std::string& id, bool forceCreation = true); + + template + inline bool installLoggerRegistrationCallback(const std::string& id) { + return base::utils::Utils::installCallback(id, + &m_loggerRegistrationCallbacks); + } + + template + inline void uninstallLoggerRegistrationCallback(const std::string& id) { + base::utils::Utils::uninstallCallback(id, &m_loggerRegistrationCallbacks); + } + + template + inline T* loggerRegistrationCallback(const std::string& id) { + return base::utils::Utils::callback(id, &m_loggerRegistrationCallbacks); + } + + bool remove(const std::string& id); + + inline bool has(const std::string& id) { + return get(id, false) != nullptr; + } + + inline void unregister(Logger*& logger) { + base::threading::ScopedLock scopedLock(lock()); + base::utils::Registry::unregister(logger->id()); + } + + inline base::LogStreamsReferenceMap* logStreamsReference(void) { + return &m_logStreamsReference; + } + + inline void flushAll(void) { + base::threading::ScopedLock scopedLock(lock()); + unsafeFlushAll(); + } + + inline void setDefaultLogBuilder(LogBuilderPtr& logBuilderPtr) { + base::threading::ScopedLock scopedLock(lock()); + m_defaultLogBuilder = logBuilderPtr; + } + + private: + LogBuilderPtr m_defaultLogBuilder; + Configurations m_defaultConfigurations; + base::LogStreamsReferenceMap m_logStreamsReference; + std::unordered_map m_loggerRegistrationCallbacks; + friend class el::base::Storage; + + void unsafeFlushAll(void); +}; +/// @brief Represents registries for verbose logging +class VRegistry : base::NoCopy, public base::threading::ThreadSafe { + public: + explicit VRegistry(base::type::VerboseLevel level, base::type::EnumType* pFlags); + + /// @brief Sets verbose level. Accepted range is 0-9 + void setLevel(base::type::VerboseLevel level); + + inline base::type::VerboseLevel level(void) const { + return m_level; + } + + inline void clearModules(void) { + base::threading::ScopedLock scopedLock(lock()); + m_modules.clear(); + } + + void setModules(const char* modules); + + bool allowed(base::type::VerboseLevel vlevel, const char* file); + + inline const std::unordered_map& modules(void) const { + return m_modules; + } + + void setFromArgs(const base::utils::CommandLineArgs* commandLineArgs); + + /// @brief Whether or not vModules enabled + inline bool vModulesEnabled(void) { + return !base::utils::hasFlag(LoggingFlag::DisableVModules, *m_pFlags); + } + + private: + base::type::VerboseLevel m_level; + base::type::EnumType* m_pFlags; + std::unordered_map m_modules; +}; +} // namespace base +class LogMessage { + public: + LogMessage(Level level, const std::string& file, base::type::LineNumber line, const std::string& func, + base::type::VerboseLevel verboseLevel, Logger* logger) : + m_level(level), m_file(file), m_line(line), m_func(func), + m_verboseLevel(verboseLevel), m_logger(logger), m_message(logger->stream().str()) { + } + inline Level level(void) const { + return m_level; + } + inline const std::string& file(void) const { + return m_file; + } + inline base::type::LineNumber line(void) const { + return m_line; + } + inline const std::string& func(void) const { + return m_func; + } + inline base::type::VerboseLevel verboseLevel(void) const { + return m_verboseLevel; + } + inline Logger* logger(void) const { + return m_logger; + } + inline const base::type::string_t& message(void) const { + return m_message; + } + private: + Level m_level; + std::string m_file; + base::type::LineNumber m_line; + std::string m_func; + base::type::VerboseLevel m_verboseLevel; + Logger* m_logger; + base::type::string_t m_message; +}; +namespace base { +#if ELPP_ASYNC_LOGGING +class AsyncLogItem { + public: + explicit AsyncLogItem(const LogMessage& logMessage, const LogDispatchData& data, const base::type::string_t& logLine) + : m_logMessage(logMessage), m_dispatchData(data), m_logLine(logLine) {} + virtual ~AsyncLogItem() {} + inline LogMessage* logMessage(void) { + return &m_logMessage; + } + inline LogDispatchData* data(void) { + return &m_dispatchData; + } + inline base::type::string_t logLine(void) { + return m_logLine; + } + private: + LogMessage m_logMessage; + LogDispatchData m_dispatchData; + base::type::string_t m_logLine; +}; +class AsyncLogQueue : public base::threading::ThreadSafe { + public: + virtual ~AsyncLogQueue() { + ELPP_INTERNAL_INFO(6, "~AsyncLogQueue"); + } + + inline AsyncLogItem next(void) { + base::threading::ScopedLock scopedLock(lock()); + AsyncLogItem result = m_queue.front(); + m_queue.pop(); + return result; + } + + inline void push(const AsyncLogItem& item) { + base::threading::ScopedLock scopedLock(lock()); + m_queue.push(item); + } + inline void pop(void) { + base::threading::ScopedLock scopedLock(lock()); + m_queue.pop(); + } + inline AsyncLogItem front(void) { + base::threading::ScopedLock scopedLock(lock()); + return m_queue.front(); + } + inline bool empty(void) { + base::threading::ScopedLock scopedLock(lock()); + return m_queue.empty(); + } + private: + std::queue m_queue; +}; +class IWorker { + public: + virtual ~IWorker() {} + virtual void start() = 0; +}; +#endif // ELPP_ASYNC_LOGGING +/// @brief Easylogging++ management storage +class Storage : base::NoCopy, public base::threading::ThreadSafe { + public: +#if ELPP_ASYNC_LOGGING + Storage(const LogBuilderPtr& defaultLogBuilder, base::IWorker* asyncDispatchWorker); +#else + explicit Storage(const LogBuilderPtr& defaultLogBuilder); +#endif // ELPP_ASYNC_LOGGING + + virtual ~Storage(void); + + inline bool validateEveryNCounter(const char* filename, base::type::LineNumber lineNumber, std::size_t occasion) { + return hitCounters()->validateEveryN(filename, lineNumber, occasion); + } + + inline bool validateAfterNCounter(const char* filename, base::type::LineNumber lineNumber, std::size_t n) { + return hitCounters()->validateAfterN(filename, lineNumber, n); + } + + inline bool validateNTimesCounter(const char* filename, base::type::LineNumber lineNumber, std::size_t n) { + return hitCounters()->validateNTimes(filename, lineNumber, n); + } + + inline base::RegisteredHitCounters* hitCounters(void) const { + return m_registeredHitCounters; + } + + inline base::RegisteredLoggers* registeredLoggers(void) const { + return m_registeredLoggers; + } + + inline base::VRegistry* vRegistry(void) const { + return m_vRegistry; + } + +#if ELPP_ASYNC_LOGGING + inline base::AsyncLogQueue* asyncLogQueue(void) const { + return m_asyncLogQueue; + } +#endif // ELPP_ASYNC_LOGGING + + inline const base::utils::CommandLineArgs* commandLineArgs(void) const { + return &m_commandLineArgs; + } + + inline void addFlag(LoggingFlag flag) { + base::utils::addFlag(flag, &m_flags); + } + + inline void removeFlag(LoggingFlag flag) { + base::utils::removeFlag(flag, &m_flags); + } + + inline bool hasFlag(LoggingFlag flag) const { + return base::utils::hasFlag(flag, m_flags); + } + + inline base::type::EnumType flags(void) const { + return m_flags; + } + + inline void setFlags(base::type::EnumType flags) { + m_flags = flags; + } + + inline void setPreRollOutCallback(const PreRollOutCallback& callback) { + m_preRollOutCallback = callback; + } + + inline void unsetPreRollOutCallback(void) { + m_preRollOutCallback = base::defaultPreRollOutCallback; + } + + inline PreRollOutCallback& preRollOutCallback(void) { + return m_preRollOutCallback; + } + + bool hasCustomFormatSpecifier(const char* formatSpecifier); + void installCustomFormatSpecifier(const CustomFormatSpecifier& customFormatSpecifier); + bool uninstallCustomFormatSpecifier(const char* formatSpecifier); + + const std::vector* customFormatSpecifiers(void) const { + return &m_customFormatSpecifiers; + } + + base::threading::Mutex& customFormatSpecifiersLock() { + return m_customFormatSpecifiersLock; + } + + inline void setLoggingLevel(Level level) { + m_loggingLevel = level; + } + + template + inline bool installLogDispatchCallback(const std::string& id) { + return base::utils::Utils::installCallback(id, &m_logDispatchCallbacks); + } + + template + inline void uninstallLogDispatchCallback(const std::string& id) { + base::utils::Utils::uninstallCallback(id, &m_logDispatchCallbacks); + } + template + inline T* logDispatchCallback(const std::string& id) { + return base::utils::Utils::callback(id, &m_logDispatchCallbacks); + } + +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + template + inline bool installPerformanceTrackingCallback(const std::string& id) { + return base::utils::Utils::installCallback(id, + &m_performanceTrackingCallbacks); + } + + template + inline void uninstallPerformanceTrackingCallback(const std::string& id) { + base::utils::Utils::uninstallCallback(id, + &m_performanceTrackingCallbacks); + } + + template + inline T* performanceTrackingCallback(const std::string& id) { + return base::utils::Utils::callback(id, &m_performanceTrackingCallbacks); + } +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + + /// @brief Sets thread name for current thread. Requires std::thread + inline void setThreadName(const std::string& name) { + if (name.empty()) return; + base::threading::ScopedLock scopedLock(m_threadNamesLock); + m_threadNames[base::threading::getCurrentThreadId()] = name; + } + + inline std::string getThreadName(const std::string& threadId) { + base::threading::ScopedLock scopedLock(m_threadNamesLock); + std::unordered_map::const_iterator it = m_threadNames.find(threadId); + if (it == m_threadNames.end()) { + return threadId; + } + return it->second; + } + private: + base::RegisteredHitCounters* m_registeredHitCounters; + base::RegisteredLoggers* m_registeredLoggers; + base::type::EnumType m_flags; + base::VRegistry* m_vRegistry; +#if ELPP_ASYNC_LOGGING + base::AsyncLogQueue* m_asyncLogQueue; + base::IWorker* m_asyncDispatchWorker; +#endif // ELPP_ASYNC_LOGGING + base::utils::CommandLineArgs m_commandLineArgs; + PreRollOutCallback m_preRollOutCallback; + std::unordered_map m_logDispatchCallbacks; + std::unordered_map m_performanceTrackingCallbacks; + std::unordered_map m_threadNames; + std::vector m_customFormatSpecifiers; + base::threading::Mutex m_customFormatSpecifiersLock; + base::threading::Mutex m_threadNamesLock; + Level m_loggingLevel; + + friend class el::Helpers; + friend class el::base::DefaultLogDispatchCallback; + friend class el::LogBuilder; + friend class el::base::MessageBuilder; + friend class el::base::Writer; + friend class el::base::PerformanceTracker; + friend class el::base::LogDispatcher; + + void setApplicationArguments(int argc, char** argv); + + inline void setApplicationArguments(int argc, const char** argv) { + setApplicationArguments(argc, const_cast(argv)); + } +}; +extern ELPP_EXPORT base::type::StoragePointer elStorage; +#define ELPP el::base::elStorage +class DefaultLogDispatchCallback : public LogDispatchCallback { + protected: + void handle(const LogDispatchData* data); + private: + const LogDispatchData* m_data; + void dispatch(base::type::string_t&& logLine); +}; +#if ELPP_ASYNC_LOGGING +class AsyncLogDispatchCallback : public LogDispatchCallback { + protected: + void handle(const LogDispatchData* data); +}; +class AsyncDispatchWorker : public base::IWorker, public base::threading::ThreadSafe { + public: + AsyncDispatchWorker(); + virtual ~AsyncDispatchWorker(); + + bool clean(void); + void emptyQueue(void); + virtual void start(void); + void handle(AsyncLogItem* logItem); + void run(void); + + void setContinueRunning(bool value) { + base::threading::ScopedLock scopedLock(m_continueRunningLock); + m_continueRunning = value; + } + + bool continueRunning(void) const { + return m_continueRunning; + } + private: + std::condition_variable cv; + bool m_continueRunning; + base::threading::Mutex m_continueRunningLock; +}; +#endif // ELPP_ASYNC_LOGGING +} // namespace base +namespace base { +class DefaultLogBuilder : public LogBuilder { + public: + base::type::string_t build(const LogMessage* logMessage, bool appendNewLine) const; +}; +/// @brief Dispatches log messages +class LogDispatcher : base::NoCopy { + public: + LogDispatcher(bool proceed, LogMessage* logMessage, base::DispatchAction dispatchAction) : + m_proceed(proceed), + m_logMessage(logMessage), + m_dispatchAction(std::move(dispatchAction)) { + } + + void dispatch(void); + + private: + bool m_proceed; + LogMessage* m_logMessage; + base::DispatchAction m_dispatchAction; +}; +#if defined(ELPP_STL_LOGGING) +/// @brief Workarounds to write some STL logs +/// +/// @detail There is workaround needed to loop through some stl containers. In order to do that, we need iterable containers +/// of same type and provide iterator interface and pass it on to writeIterator(). +/// Remember, this is passed by value in constructor so that we dont change original containers. +/// This operation is as expensive as Big-O(std::min(class_.size(), base::consts::kMaxLogPerContainer)) +namespace workarounds { +/// @brief Abstract IterableContainer template that provides interface for iterable classes of type T +template +class IterableContainer { + public: + typedef typename Container::iterator iterator; + typedef typename Container::const_iterator const_iterator; + IterableContainer(void) {} + virtual ~IterableContainer(void) {} + iterator begin(void) { + return getContainer().begin(); + } + iterator end(void) { + return getContainer().end(); + } + private: + virtual Container& getContainer(void) = 0; +}; +/// @brief Implements IterableContainer and provides iterable std::priority_queue class +template, typename Comparator = std::less> +class IterablePriorityQueue : public IterableContainer, + public std::priority_queue { + public: + IterablePriorityQueue(std::priority_queue queue_) { + std::size_t count_ = 0; + while (++count_ < base::consts::kMaxLogPerContainer && !queue_.empty()) { + this->push(queue_.top()); + queue_.pop(); + } + } + private: + inline Container& getContainer(void) { + return this->c; + } +}; +/// @brief Implements IterableContainer and provides iterable std::queue class +template> +class IterableQueue : public IterableContainer, public std::queue { + public: + IterableQueue(std::queue queue_) { + std::size_t count_ = 0; + while (++count_ < base::consts::kMaxLogPerContainer && !queue_.empty()) { + this->push(queue_.front()); + queue_.pop(); + } + } + private: + inline Container& getContainer(void) { + return this->c; + } +}; +/// @brief Implements IterableContainer and provides iterable std::stack class +template> +class IterableStack : public IterableContainer, public std::stack { + public: + IterableStack(std::stack stack_) { + std::size_t count_ = 0; + while (++count_ < base::consts::kMaxLogPerContainer && !stack_.empty()) { + this->push(stack_.top()); + stack_.pop(); + } + } + private: + inline Container& getContainer(void) { + return this->c; + } +}; +} // namespace workarounds +#endif // defined(ELPP_STL_LOGGING) +// Log message builder +class MessageBuilder { + public: + MessageBuilder(void) : m_logger(nullptr), m_containerLogSeperator(ELPP_LITERAL("")) {} + void initialize(Logger* logger); + +# define ELPP_SIMPLE_LOG(LOG_TYPE)\ +MessageBuilder& operator<<(LOG_TYPE msg) {\ +m_logger->stream() << msg;\ +if (ELPP->hasFlag(LoggingFlag::AutoSpacing)) {\ +m_logger->stream() << " ";\ +}\ +return *this;\ +} + + inline MessageBuilder& operator<<(const std::string& msg) { + return operator<<(msg.c_str()); + } + ELPP_SIMPLE_LOG(char) + ELPP_SIMPLE_LOG(bool) + ELPP_SIMPLE_LOG(signed short) + ELPP_SIMPLE_LOG(unsigned short) + ELPP_SIMPLE_LOG(signed int) + ELPP_SIMPLE_LOG(unsigned int) + ELPP_SIMPLE_LOG(signed long) + ELPP_SIMPLE_LOG(unsigned long) + ELPP_SIMPLE_LOG(float) + ELPP_SIMPLE_LOG(double) + ELPP_SIMPLE_LOG(char*) + ELPP_SIMPLE_LOG(const char*) + ELPP_SIMPLE_LOG(const void*) + ELPP_SIMPLE_LOG(long double) + inline MessageBuilder& operator<<(const std::wstring& msg) { + return operator<<(msg.c_str()); + } + MessageBuilder& operator<<(const wchar_t* msg); + // ostream manipulators + inline MessageBuilder& operator<<(std::ostream& (*OStreamMani)(std::ostream&)) { + m_logger->stream() << OStreamMani; + return *this; + } +#define ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(temp) \ +template \ +inline MessageBuilder& operator<<(const temp& template_inst) { \ +return writeIterator(template_inst.begin(), template_inst.end(), template_inst.size()); \ +} +#define ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(temp) \ +template \ +inline MessageBuilder& operator<<(const temp& template_inst) { \ +return writeIterator(template_inst.begin(), template_inst.end(), template_inst.size()); \ +} +#define ELPP_ITERATOR_CONTAINER_LOG_THREE_ARG(temp) \ +template \ +inline MessageBuilder& operator<<(const temp& template_inst) { \ +return writeIterator(template_inst.begin(), template_inst.end(), template_inst.size()); \ +} +#define ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(temp) \ +template \ +inline MessageBuilder& operator<<(const temp& template_inst) { \ +return writeIterator(template_inst.begin(), template_inst.end(), template_inst.size()); \ +} +#define ELPP_ITERATOR_CONTAINER_LOG_FIVE_ARG(temp) \ +template \ +inline MessageBuilder& operator<<(const temp& template_inst) { \ +return writeIterator(template_inst.begin(), template_inst.end(), template_inst.size()); \ +} + +#if defined(ELPP_STL_LOGGING) + ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(std::vector) + ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(std::list) + ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(std::deque) + ELPP_ITERATOR_CONTAINER_LOG_THREE_ARG(std::set) + ELPP_ITERATOR_CONTAINER_LOG_THREE_ARG(std::multiset) + ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(std::map) + ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(std::multimap) + template + inline MessageBuilder& operator<<(const std::queue& queue_) { + base::workarounds::IterableQueue iterableQueue_ = + static_cast >(queue_); + return writeIterator(iterableQueue_.begin(), iterableQueue_.end(), iterableQueue_.size()); + } + template + inline MessageBuilder& operator<<(const std::stack& stack_) { + base::workarounds::IterableStack iterableStack_ = + static_cast >(stack_); + return writeIterator(iterableStack_.begin(), iterableStack_.end(), iterableStack_.size()); + } + template + inline MessageBuilder& operator<<(const std::priority_queue& priorityQueue_) { + base::workarounds::IterablePriorityQueue iterablePriorityQueue_ = + static_cast >(priorityQueue_); + return writeIterator(iterablePriorityQueue_.begin(), iterablePriorityQueue_.end(), iterablePriorityQueue_.size()); + } + template + MessageBuilder& operator<<(const std::pair& pair_) { + m_logger->stream() << ELPP_LITERAL("("); + operator << (static_cast(pair_.first)); + m_logger->stream() << ELPP_LITERAL(", "); + operator << (static_cast(pair_.second)); + m_logger->stream() << ELPP_LITERAL(")"); + return *this; + } + template + MessageBuilder& operator<<(const std::bitset& bitset_) { + m_logger->stream() << ELPP_LITERAL("["); + operator << (bitset_.to_string()); + m_logger->stream() << ELPP_LITERAL("]"); + return *this; + } +# if defined(ELPP_LOG_STD_ARRAY) + template + inline MessageBuilder& operator<<(const std::array& array) { + return writeIterator(array.begin(), array.end(), array.size()); + } +# endif // defined(ELPP_LOG_STD_ARRAY) +# if defined(ELPP_LOG_UNORDERED_MAP) + ELPP_ITERATOR_CONTAINER_LOG_FIVE_ARG(std::unordered_map) + ELPP_ITERATOR_CONTAINER_LOG_FIVE_ARG(std::unordered_multimap) +# endif // defined(ELPP_LOG_UNORDERED_MAP) +# if defined(ELPP_LOG_UNORDERED_SET) + ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(std::unordered_set) + ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(std::unordered_multiset) +# endif // defined(ELPP_LOG_UNORDERED_SET) +#endif // defined(ELPP_STL_LOGGING) +#if defined(ELPP_QT_LOGGING) + inline MessageBuilder& operator<<(const QString& msg) { +# if defined(ELPP_UNICODE) + m_logger->stream() << msg.toStdWString(); +# else + m_logger->stream() << msg.toStdString(); +# endif // defined(ELPP_UNICODE) + return *this; + } + inline MessageBuilder& operator<<(const QByteArray& msg) { + return operator << (QString(msg)); + } + inline MessageBuilder& operator<<(const QStringRef& msg) { + return operator<<(msg.toString()); + } + inline MessageBuilder& operator<<(qint64 msg) { +# if defined(ELPP_UNICODE) + m_logger->stream() << QString::number(msg).toStdWString(); +# else + m_logger->stream() << QString::number(msg).toStdString(); +# endif // defined(ELPP_UNICODE) + return *this; + } + inline MessageBuilder& operator<<(quint64 msg) { +# if defined(ELPP_UNICODE) + m_logger->stream() << QString::number(msg).toStdWString(); +# else + m_logger->stream() << QString::number(msg).toStdString(); +# endif // defined(ELPP_UNICODE) + return *this; + } + inline MessageBuilder& operator<<(QChar msg) { + m_logger->stream() << msg.toLatin1(); + return *this; + } + inline MessageBuilder& operator<<(const QLatin1String& msg) { + m_logger->stream() << msg.latin1(); + return *this; + } + ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(QList) + ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(QVector) + ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(QQueue) + ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(QSet) + ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(QLinkedList) + ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(QStack) + template + MessageBuilder& operator<<(const QPair& pair_) { + m_logger->stream() << ELPP_LITERAL("("); + operator << (static_cast(pair_.first)); + m_logger->stream() << ELPP_LITERAL(", "); + operator << (static_cast(pair_.second)); + m_logger->stream() << ELPP_LITERAL(")"); + return *this; + } + template + MessageBuilder& operator<<(const QMap& map_) { + m_logger->stream() << ELPP_LITERAL("["); + QList keys = map_.keys(); + typename QList::const_iterator begin = keys.begin(); + typename QList::const_iterator end = keys.end(); + int max_ = static_cast(base::consts::kMaxLogPerContainer); // to prevent warning + for (int index_ = 0; begin != end && index_ < max_; ++index_, ++begin) { + m_logger->stream() << ELPP_LITERAL("("); + operator << (static_cast(*begin)); + m_logger->stream() << ELPP_LITERAL(", "); + operator << (static_cast(map_.value(*begin))); + m_logger->stream() << ELPP_LITERAL(")"); + m_logger->stream() << ((index_ < keys.size() -1) ? m_containerLogSeperator : ELPP_LITERAL("")); + } + if (begin != end) { + m_logger->stream() << ELPP_LITERAL("..."); + } + m_logger->stream() << ELPP_LITERAL("]"); + return *this; + } + template + inline MessageBuilder& operator<<(const QMultiMap& map_) { + operator << (static_cast>(map_)); + return *this; + } + template + MessageBuilder& operator<<(const QHash& hash_) { + m_logger->stream() << ELPP_LITERAL("["); + QList keys = hash_.keys(); + typename QList::const_iterator begin = keys.begin(); + typename QList::const_iterator end = keys.end(); + int max_ = static_cast(base::consts::kMaxLogPerContainer); // prevent type warning + for (int index_ = 0; begin != end && index_ < max_; ++index_, ++begin) { + m_logger->stream() << ELPP_LITERAL("("); + operator << (static_cast(*begin)); + m_logger->stream() << ELPP_LITERAL(", "); + operator << (static_cast(hash_.value(*begin))); + m_logger->stream() << ELPP_LITERAL(")"); + m_logger->stream() << ((index_ < keys.size() -1) ? m_containerLogSeperator : ELPP_LITERAL("")); + } + if (begin != end) { + m_logger->stream() << ELPP_LITERAL("..."); + } + m_logger->stream() << ELPP_LITERAL("]"); + return *this; + } + template + inline MessageBuilder& operator<<(const QMultiHash& multiHash_) { + operator << (static_cast>(multiHash_)); + return *this; + } +#endif // defined(ELPP_QT_LOGGING) +#if defined(ELPP_BOOST_LOGGING) + ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(boost::container::vector) + ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(boost::container::stable_vector) + ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(boost::container::list) + ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(boost::container::deque) + ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(boost::container::map) + ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(boost::container::flat_map) + ELPP_ITERATOR_CONTAINER_LOG_THREE_ARG(boost::container::set) + ELPP_ITERATOR_CONTAINER_LOG_THREE_ARG(boost::container::flat_set) +#endif // defined(ELPP_BOOST_LOGGING) + + /// @brief Macro used internally that can be used externally to make containers easylogging++ friendly + /// + /// @detail This macro expands to write an ostream& operator<< for container. This container is expected to + /// have begin() and end() methods that return respective iterators + /// @param ContainerType Type of container e.g, MyList from WX_DECLARE_LIST(int, MyList); in wxwidgets + /// @param SizeMethod Method used to get size of container. + /// @param ElementInstance Instance of element to be fed out. Insance name is "elem". See WXELPP_ENABLED macro + /// for an example usage +#define MAKE_CONTAINERELPP_FRIENDLY(ContainerType, SizeMethod, ElementInstance) \ +el::base::type::ostream_t& operator<<(el::base::type::ostream_t& ss, const ContainerType& container) {\ +const el::base::type::char_t* sep = ELPP->hasFlag(el::LoggingFlag::NewLineForContainer) ? \ +ELPP_LITERAL("\n ") : ELPP_LITERAL(", ");\ +ContainerType::const_iterator elem = container.begin();\ +ContainerType::const_iterator endElem = container.end();\ +std::size_t size_ = container.SizeMethod; \ +ss << ELPP_LITERAL("[");\ +for (std::size_t i = 0; elem != endElem && i < el::base::consts::kMaxLogPerContainer; ++i, ++elem) { \ +ss << ElementInstance;\ +ss << ((i < size_ - 1) ? sep : ELPP_LITERAL(""));\ +}\ +if (elem != endElem) {\ +ss << ELPP_LITERAL("...");\ +}\ +ss << ELPP_LITERAL("]");\ +return ss;\ +} +#if defined(ELPP_WXWIDGETS_LOGGING) + ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(wxVector) +# define ELPP_WX_PTR_ENABLED(ContainerType) MAKE_CONTAINERELPP_FRIENDLY(ContainerType, size(), *(*elem)) +# define ELPP_WX_ENABLED(ContainerType) MAKE_CONTAINERELPP_FRIENDLY(ContainerType, size(), (*elem)) +# define ELPP_WX_HASH_MAP_ENABLED(ContainerType) MAKE_CONTAINERELPP_FRIENDLY(ContainerType, size(), \ +ELPP_LITERAL("(") << elem->first << ELPP_LITERAL(", ") << elem->second << ELPP_LITERAL(")") +#else +# define ELPP_WX_PTR_ENABLED(ContainerType) +# define ELPP_WX_ENABLED(ContainerType) +# define ELPP_WX_HASH_MAP_ENABLED(ContainerType) +#endif // defined(ELPP_WXWIDGETS_LOGGING) + // Other classes + template + ELPP_SIMPLE_LOG(const Class&) +#undef ELPP_SIMPLE_LOG +#undef ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG +#undef ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG +#undef ELPP_ITERATOR_CONTAINER_LOG_THREE_ARG +#undef ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG +#undef ELPP_ITERATOR_CONTAINER_LOG_FIVE_ARG + private: + Logger* m_logger; + const base::type::char_t* m_containerLogSeperator; + + template + MessageBuilder& writeIterator(Iterator begin_, Iterator end_, std::size_t size_) { + m_logger->stream() << ELPP_LITERAL("["); + for (std::size_t i = 0; begin_ != end_ && i < base::consts::kMaxLogPerContainer; ++i, ++begin_) { + operator << (*begin_); + m_logger->stream() << ((i < size_ - 1) ? m_containerLogSeperator : ELPP_LITERAL("")); + } + if (begin_ != end_) { + m_logger->stream() << ELPP_LITERAL("..."); + } + m_logger->stream() << ELPP_LITERAL("]"); + if (ELPP->hasFlag(LoggingFlag::AutoSpacing)) { + m_logger->stream() << " "; + } + return *this; + } +}; +/// @brief Writes nothing - Used when certain log is disabled +class NullWriter : base::NoCopy { + public: + NullWriter(void) {} + + // Null manipulator + inline NullWriter& operator<<(std::ostream& (*)(std::ostream&)) { + return *this; + } + + template + inline NullWriter& operator<<(const T&) { + return *this; + } + + inline operator bool() { + return true; + } +}; +/// @brief Main entry point of each logging +class Writer : base::NoCopy { + public: + Writer(Level level, const char* file, base::type::LineNumber line, + const char* func, base::DispatchAction dispatchAction = base::DispatchAction::NormalLog, + base::type::VerboseLevel verboseLevel = 0) : + m_msg(nullptr), m_level(level), m_file(file), m_line(line), m_func(func), m_verboseLevel(verboseLevel), + m_logger(nullptr), m_proceed(false), m_dispatchAction(dispatchAction) { + } + + Writer(LogMessage* msg, base::DispatchAction dispatchAction = base::DispatchAction::NormalLog) : + m_msg(msg), m_level(msg != nullptr ? msg->level() : Level::Unknown), + m_line(0), m_logger(nullptr), m_proceed(false), m_dispatchAction(dispatchAction) { + } + + virtual ~Writer(void) { + processDispatch(); + } + + template + inline Writer& operator<<(const T& log) { +#if ELPP_LOGGING_ENABLED + if (m_proceed) { + m_messageBuilder << log; + } +#endif // ELPP_LOGGING_ENABLED + return *this; + } + + inline Writer& operator<<(std::ostream& (*log)(std::ostream&)) { +#if ELPP_LOGGING_ENABLED + if (m_proceed) { + m_messageBuilder << log; + } +#endif // ELPP_LOGGING_ENABLED + return *this; + } + + inline operator bool() { + return true; + } + + Writer& construct(Logger* logger, bool needLock = true); + Writer& construct(int count, const char* loggerIds, ...); + protected: + LogMessage* m_msg; + Level m_level; + const char* m_file; + const base::type::LineNumber m_line; + const char* m_func; + base::type::VerboseLevel m_verboseLevel; + Logger* m_logger; + bool m_proceed; + base::MessageBuilder m_messageBuilder; + base::DispatchAction m_dispatchAction; + std::vector m_loggerIds; + friend class el::Helpers; + + void initializeLogger(const std::string& loggerId, bool lookup = true, bool needLock = true); + void processDispatch(); + void triggerDispatch(void); +}; +class PErrorWriter : public base::Writer { + public: + PErrorWriter(Level level, const char* file, base::type::LineNumber line, + const char* func, base::DispatchAction dispatchAction = base::DispatchAction::NormalLog, + base::type::VerboseLevel verboseLevel = 0) : + base::Writer(level, file, line, func, dispatchAction, verboseLevel) { + } + + virtual ~PErrorWriter(void); +}; +} // namespace base +// Logging from Logger class. Why this is here? Because we have Storage and Writer class available +#if ELPP_VARIADIC_TEMPLATES_SUPPORTED +template +void Logger::log_(Level level, int vlevel, const char* s, const T& value, const Args&... args) { + base::MessageBuilder b; + b.initialize(this); + while (*s) { + if (*s == base::consts::kFormatSpecifierChar) { + if (*(s + 1) == base::consts::kFormatSpecifierChar) { + ++s; + } else { + if (*(s + 1) == base::consts::kFormatSpecifierCharValue) { + ++s; + b << value; + log_(level, vlevel, ++s, args...); + return; + } + } + } + b << *s++; + } + ELPP_INTERNAL_ERROR("Too many arguments provided. Unable to handle. Please provide more format specifiers", false); +} +template +void Logger::log_(Level level, int vlevel, const T& log) { + if (level == Level::Verbose) { + if (ELPP->vRegistry()->allowed(vlevel, __FILE__)) { + base::Writer(Level::Verbose, "FILE", 0, "FUNCTION", + base::DispatchAction::NormalLog, vlevel).construct(this, false) << log; + } else { + stream().str(ELPP_LITERAL("")); + releaseLock(); + } + } else { + base::Writer(level, "FILE", 0, "FUNCTION").construct(this, false) << log; + } +} +template +inline void Logger::log(Level level, const char* s, const T& value, const Args&... args) { + acquireLock(); // released in Writer! + log_(level, 0, s, value, args...); +} +template +inline void Logger::log(Level level, const T& log) { + acquireLock(); // released in Writer! + log_(level, 0, log); +} +# if ELPP_VERBOSE_LOG +template +inline void Logger::verbose(int vlevel, const char* s, const T& value, const Args&... args) { + acquireLock(); // released in Writer! + log_(el::Level::Verbose, vlevel, s, value, args...); +} +template +inline void Logger::verbose(int vlevel, const T& log) { + acquireLock(); // released in Writer! + log_(el::Level::Verbose, vlevel, log); +} +# else +template +inline void Logger::verbose(int, const char*, const T&, const Args&...) { + return; +} +template +inline void Logger::verbose(int, const T&) { + return; +} +# endif // ELPP_VERBOSE_LOG +# define LOGGER_LEVEL_WRITERS(FUNCTION_NAME, LOG_LEVEL)\ +template \ +inline void Logger::FUNCTION_NAME(const char* s, const T& value, const Args&... args) {\ +log(LOG_LEVEL, s, value, args...);\ +}\ +template \ +inline void Logger::FUNCTION_NAME(const T& value) {\ +log(LOG_LEVEL, value);\ +} +# define LOGGER_LEVEL_WRITERS_DISABLED(FUNCTION_NAME, LOG_LEVEL)\ +template \ +inline void Logger::FUNCTION_NAME(const char*, const T&, const Args&...) {\ +return;\ +}\ +template \ +inline void Logger::FUNCTION_NAME(const T&) {\ +return;\ +} + +# if ELPP_INFO_LOG +LOGGER_LEVEL_WRITERS(info, Level::Info) +# else +LOGGER_LEVEL_WRITERS_DISABLED(info, Level::Info) +# endif // ELPP_INFO_LOG +# if ELPP_DEBUG_LOG +LOGGER_LEVEL_WRITERS(debug, Level::Debug) +# else +LOGGER_LEVEL_WRITERS_DISABLED(debug, Level::Debug) +# endif // ELPP_DEBUG_LOG +# if ELPP_WARNING_LOG +LOGGER_LEVEL_WRITERS(warn, Level::Warning) +# else +LOGGER_LEVEL_WRITERS_DISABLED(warn, Level::Warning) +# endif // ELPP_WARNING_LOG +# if ELPP_ERROR_LOG +LOGGER_LEVEL_WRITERS(error, Level::Error) +# else +LOGGER_LEVEL_WRITERS_DISABLED(error, Level::Error) +# endif // ELPP_ERROR_LOG +# if ELPP_FATAL_LOG +LOGGER_LEVEL_WRITERS(fatal, Level::Fatal) +# else +LOGGER_LEVEL_WRITERS_DISABLED(fatal, Level::Fatal) +# endif // ELPP_FATAL_LOG +# if ELPP_TRACE_LOG +LOGGER_LEVEL_WRITERS(trace, Level::Trace) +# else +LOGGER_LEVEL_WRITERS_DISABLED(trace, Level::Trace) +# endif // ELPP_TRACE_LOG +# undef LOGGER_LEVEL_WRITERS +# undef LOGGER_LEVEL_WRITERS_DISABLED +#endif // ELPP_VARIADIC_TEMPLATES_SUPPORTED +#if ELPP_COMPILER_MSVC +# define ELPP_VARIADIC_FUNC_MSVC(variadicFunction, variadicArgs) variadicFunction variadicArgs +# define ELPP_VARIADIC_FUNC_MSVC_RUN(variadicFunction, ...) ELPP_VARIADIC_FUNC_MSVC(variadicFunction, (__VA_ARGS__)) +# define el_getVALength(...) ELPP_VARIADIC_FUNC_MSVC_RUN(el_resolveVALength, 0, ## __VA_ARGS__,\ +10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) +#else +# if ELPP_COMPILER_CLANG +# define el_getVALength(...) el_resolveVALength(0, __VA_ARGS__, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) +# else +# define el_getVALength(...) el_resolveVALength(0, ## __VA_ARGS__, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) +# endif // ELPP_COMPILER_CLANG +#endif // ELPP_COMPILER_MSVC +#define el_resolveVALength(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, N, ...) N +#define ELPP_WRITE_LOG(writer, level, dispatchAction, ...) \ +writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) +#define ELPP_WRITE_LOG_IF(writer, condition, level, dispatchAction, ...) if (condition) \ +writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) +#define ELPP_WRITE_LOG_EVERY_N(writer, occasion, level, dispatchAction, ...) \ +ELPP->validateEveryNCounter(__FILE__, __LINE__, occasion) && \ +writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) +#define ELPP_WRITE_LOG_AFTER_N(writer, n, level, dispatchAction, ...) \ +ELPP->validateAfterNCounter(__FILE__, __LINE__, n) && \ +writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) +#define ELPP_WRITE_LOG_N_TIMES(writer, n, level, dispatchAction, ...) \ +ELPP->validateNTimesCounter(__FILE__, __LINE__, n) && \ +writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) +class PerformanceTrackingData { + public: + enum class DataType : base::type::EnumType { + Checkpoint = 1, Complete = 2 + }; + // Do not use constructor, will run into multiple definition error, use init(PerformanceTracker*) + explicit PerformanceTrackingData(DataType dataType) : m_performanceTracker(nullptr), + m_dataType(dataType), m_firstCheckpoint(false), m_file(""), m_line(0), m_func("") {} + inline const std::string* blockName(void) const; + inline const struct timeval* startTime(void) const; + inline const struct timeval* endTime(void) const; + inline const struct timeval* lastCheckpointTime(void) const; + inline const base::PerformanceTracker* performanceTracker(void) const { + return m_performanceTracker; + } + inline PerformanceTrackingData::DataType dataType(void) const { + return m_dataType; + } + inline bool firstCheckpoint(void) const { + return m_firstCheckpoint; + } + inline std::string checkpointId(void) const { + return m_checkpointId; + } + inline const char* file(void) const { + return m_file; + } + inline base::type::LineNumber line(void) const { + return m_line; + } + inline const char* func(void) const { + return m_func; + } + inline const base::type::string_t* formattedTimeTaken() const { + return &m_formattedTimeTaken; + } + inline const std::string& loggerId(void) const; + private: + base::PerformanceTracker* m_performanceTracker; + base::type::string_t m_formattedTimeTaken; + PerformanceTrackingData::DataType m_dataType; + bool m_firstCheckpoint; + std::string m_checkpointId; + const char* m_file; + base::type::LineNumber m_line; + const char* m_func; + inline void init(base::PerformanceTracker* performanceTracker, bool firstCheckpoint = false) { + m_performanceTracker = performanceTracker; + m_firstCheckpoint = firstCheckpoint; + } + + friend class el::base::PerformanceTracker; +}; +namespace base { +/// @brief Represents performanceTracker block of code that conditionally adds performance status to log +/// either when goes outside the scope of when checkpoint() is called +class PerformanceTracker : public base::threading::ThreadSafe, public Loggable { + public: + PerformanceTracker(const std::string& blockName, + base::TimestampUnit timestampUnit = base::TimestampUnit::Millisecond, + const std::string& loggerId = std::string(el::base::consts::kPerformanceLoggerId), + bool scopedLog = true, Level level = base::consts::kPerformanceTrackerDefaultLevel); + /// @brief Copy constructor + PerformanceTracker(const PerformanceTracker& t) : + m_blockName(t.m_blockName), m_timestampUnit(t.m_timestampUnit), m_loggerId(t.m_loggerId), m_scopedLog(t.m_scopedLog), + m_level(t.m_level), m_hasChecked(t.m_hasChecked), m_lastCheckpointId(t.m_lastCheckpointId), m_enabled(t.m_enabled), + m_startTime(t.m_startTime), m_endTime(t.m_endTime), m_lastCheckpointTime(t.m_lastCheckpointTime) { + } + virtual ~PerformanceTracker(void); + /// @brief A checkpoint for current performanceTracker block. + void checkpoint(const std::string& id = std::string(), const char* file = __FILE__, + base::type::LineNumber line = __LINE__, + const char* func = ""); + inline Level level(void) const { + return m_level; + } + private: + std::string m_blockName; + base::TimestampUnit m_timestampUnit; + std::string m_loggerId; + bool m_scopedLog; + Level m_level; + bool m_hasChecked; + std::string m_lastCheckpointId; + bool m_enabled; + struct timeval m_startTime, m_endTime, m_lastCheckpointTime; + + PerformanceTracker(void); + + friend class el::PerformanceTrackingData; + friend class base::DefaultPerformanceTrackingCallback; + + const inline base::type::string_t getFormattedTimeTaken() const { + return getFormattedTimeTaken(m_startTime); + } + + const base::type::string_t getFormattedTimeTaken(struct timeval startTime) const; + + virtual inline void log(el::base::type::ostream_t& os) const { + os << getFormattedTimeTaken(); + } +}; +class DefaultPerformanceTrackingCallback : public PerformanceTrackingCallback { + protected: + void handle(const PerformanceTrackingData* data) { + m_data = data; + base::type::stringstream_t ss; + if (m_data->dataType() == PerformanceTrackingData::DataType::Complete) { + ss << ELPP_LITERAL("Executed [") << m_data->blockName()->c_str() << ELPP_LITERAL("] in [") << + *m_data->formattedTimeTaken() << ELPP_LITERAL("]"); + } else { + ss << ELPP_LITERAL("Performance checkpoint"); + if (!m_data->checkpointId().empty()) { + ss << ELPP_LITERAL(" [") << m_data->checkpointId().c_str() << ELPP_LITERAL("]"); + } + ss << ELPP_LITERAL(" for block [") << m_data->blockName()->c_str() << ELPP_LITERAL("] : [") << + *m_data->performanceTracker(); + if (!ELPP->hasFlag(LoggingFlag::DisablePerformanceTrackingCheckpointComparison) + && m_data->performanceTracker()->m_hasChecked) { + ss << ELPP_LITERAL(" ([") << *m_data->formattedTimeTaken() << ELPP_LITERAL("] from "); + if (m_data->performanceTracker()->m_lastCheckpointId.empty()) { + ss << ELPP_LITERAL("last checkpoint"); + } else { + ss << ELPP_LITERAL("checkpoint '") << m_data->performanceTracker()->m_lastCheckpointId.c_str() << ELPP_LITERAL("'"); + } + ss << ELPP_LITERAL(")]"); + } else { + ss << ELPP_LITERAL("]"); + } + } + el::base::Writer(m_data->performanceTracker()->level(), m_data->file(), m_data->line(), m_data->func()).construct(1, + m_data->loggerId().c_str()) << ss.str(); + } + private: + const PerformanceTrackingData* m_data; +}; +} // namespace base +inline const std::string* PerformanceTrackingData::blockName() const { + return const_cast(&m_performanceTracker->m_blockName); +} +inline const struct timeval* PerformanceTrackingData::startTime() const { + return const_cast(&m_performanceTracker->m_startTime); +} +inline const struct timeval* PerformanceTrackingData::endTime() const { + return const_cast(&m_performanceTracker->m_endTime); +} +inline const struct timeval* PerformanceTrackingData::lastCheckpointTime() const { + return const_cast(&m_performanceTracker->m_lastCheckpointTime); +} +inline const std::string& PerformanceTrackingData::loggerId(void) const { + return m_performanceTracker->m_loggerId; +} +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) +namespace base { +/// @brief Contains some internal debugging tools like crash handler and stack tracer +namespace debug { +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) +class StackTrace : base::NoCopy { + public: + static const unsigned int kMaxStack = 64; + static const unsigned int kStackStart = 2; // We want to skip c'tor and StackTrace::generateNew() + class StackTraceEntry { + public: + StackTraceEntry(std::size_t index, const std::string& loc, const std::string& demang, const std::string& hex, + const std::string& addr); + StackTraceEntry(std::size_t index, const std::string& loc) : + m_index(index), + m_location(loc) { + } + std::size_t m_index; + std::string m_location; + std::string m_demangled; + std::string m_hex; + std::string m_addr; + friend std::ostream& operator<<(std::ostream& ss, const StackTraceEntry& si); + + private: + StackTraceEntry(void); + }; + + StackTrace(void) { + generateNew(); + } + + virtual ~StackTrace(void) { + } + + inline std::vector& getLatestStack(void) { + return m_stack; + } + + friend std::ostream& operator<<(std::ostream& os, const StackTrace& st); + + private: + std::vector m_stack; + + void generateNew(void); +}; +/// @brief Handles unexpected crashes +class CrashHandler : base::NoCopy { + public: + typedef void (*Handler)(int); + + explicit CrashHandler(bool useDefault); + explicit CrashHandler(const Handler& cHandler) { + setHandler(cHandler); + } + void setHandler(const Handler& cHandler); + + private: + Handler m_handler; +}; +#else +class CrashHandler { + public: + explicit CrashHandler(bool) {} +}; +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) +} // namespace debug +} // namespace base +extern base::debug::CrashHandler elCrashHandler; +#define MAKE_LOGGABLE(ClassType, ClassInstance, OutputStreamInstance) \ +el::base::type::ostream_t& operator<<(el::base::type::ostream_t& OutputStreamInstance, const ClassType& ClassInstance) +/// @brief Initializes syslog with process ID, options and facility. calls closelog() on d'tor +class SysLogInitializer { + public: + SysLogInitializer(const char* processIdent, int options = 0, int facility = 0) { +#if defined(ELPP_SYSLOG) + openlog(processIdent, options, facility); +#else + ELPP_UNUSED(processIdent); + ELPP_UNUSED(options); + ELPP_UNUSED(facility); +#endif // defined(ELPP_SYSLOG) + } + virtual ~SysLogInitializer(void) { +#if defined(ELPP_SYSLOG) + closelog(); +#endif // defined(ELPP_SYSLOG) + } +}; +#define ELPP_INITIALIZE_SYSLOG(id, opt, fac) el::SysLogInitializer elSyslogInit(id, opt, fac) +/// @brief Static helpers for developers +class Helpers : base::StaticClass { + public: + /// @brief Shares logging repository (base::Storage) + static inline void setStorage(base::type::StoragePointer storage) { + ELPP = storage; + } + /// @return Main storage repository + static inline base::type::StoragePointer storage() { + return ELPP; + } + /// @brief Sets application arguments and figures out whats active for logging and whats not. + static inline void setArgs(int argc, char** argv) { + ELPP->setApplicationArguments(argc, argv); + } + /// @copydoc setArgs(int argc, char** argv) + static inline void setArgs(int argc, const char** argv) { + ELPP->setApplicationArguments(argc, const_cast(argv)); + } + /// @brief Sets thread name for current thread. Requires std::thread + static inline void setThreadName(const std::string& name) { + ELPP->setThreadName(name); + } + static inline std::string getThreadName() { + return ELPP->getThreadName(base::threading::getCurrentThreadId()); + } +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) + /// @brief Overrides default crash handler and installs custom handler. + /// @param crashHandler A functor with no return type that takes single int argument. + /// Handler is a typedef with specification: void (*Handler)(int) + static inline void setCrashHandler(const el::base::debug::CrashHandler::Handler& crashHandler) { + el::elCrashHandler.setHandler(crashHandler); + } + /// @brief Abort due to crash with signal in parameter + /// @param sig Crash signal + static void crashAbort(int sig, const char* sourceFile = "", unsigned int long line = 0); + /// @brief Logs reason of crash as per sig + /// @param sig Crash signal + /// @param stackTraceIfAvailable Includes stack trace if available + /// @param level Logging level + /// @param logger Logger to use for logging + static void logCrashReason(int sig, bool stackTraceIfAvailable = false, + Level level = Level::Fatal, const char* logger = base::consts::kDefaultLoggerId); +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) + /// @brief Installs pre rollout callback, this callback is triggered when log file is about to be rolled out + /// (can be useful for backing up) + static inline void installPreRollOutCallback(const PreRollOutCallback& callback) { + ELPP->setPreRollOutCallback(callback); + } + /// @brief Uninstalls pre rollout callback + static inline void uninstallPreRollOutCallback(void) { + ELPP->unsetPreRollOutCallback(); + } + /// @brief Installs post log dispatch callback, this callback is triggered when log is dispatched + template + static inline bool installLogDispatchCallback(const std::string& id) { + return ELPP->installLogDispatchCallback(id); + } + /// @brief Uninstalls log dispatch callback + template + static inline void uninstallLogDispatchCallback(const std::string& id) { + ELPP->uninstallLogDispatchCallback(id); + } + template + static inline T* logDispatchCallback(const std::string& id) { + return ELPP->logDispatchCallback(id); + } +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + /// @brief Installs post performance tracking callback, this callback is triggered when performance tracking is finished + template + static inline bool installPerformanceTrackingCallback(const std::string& id) { + return ELPP->installPerformanceTrackingCallback(id); + } + /// @brief Uninstalls post performance tracking handler + template + static inline void uninstallPerformanceTrackingCallback(const std::string& id) { + ELPP->uninstallPerformanceTrackingCallback(id); + } + template + static inline T* performanceTrackingCallback(const std::string& id) { + return ELPP->performanceTrackingCallback(id); + } +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + /// @brief Converts template to std::string - useful for loggable classes to log containers within log(std::ostream&) const + template + static std::string convertTemplateToStdString(const T& templ) { + el::Logger* logger = + ELPP->registeredLoggers()->get(el::base::consts::kDefaultLoggerId); + if (logger == nullptr) { + return std::string(); + } + base::MessageBuilder b; + b.initialize(logger); + logger->acquireLock(); + b << templ; +#if defined(ELPP_UNICODE) + std::string s = std::string(logger->stream().str().begin(), logger->stream().str().end()); +#else + std::string s = logger->stream().str(); +#endif // defined(ELPP_UNICODE) + logger->stream().str(ELPP_LITERAL("")); + logger->releaseLock(); + return s; + } + /// @brief Returns command line arguments (pointer) provided to easylogging++ + static inline const el::base::utils::CommandLineArgs* commandLineArgs(void) { + return ELPP->commandLineArgs(); + } + /// @brief Reserve space for custom format specifiers for performance + /// @see std::vector::reserve + static inline void reserveCustomFormatSpecifiers(std::size_t size) { + ELPP->m_customFormatSpecifiers.reserve(size); + } + /// @brief Installs user defined format specifier and handler + static inline void installCustomFormatSpecifier(const CustomFormatSpecifier& customFormatSpecifier) { + ELPP->installCustomFormatSpecifier(customFormatSpecifier); + } + /// @brief Uninstalls user defined format specifier and handler + static inline bool uninstallCustomFormatSpecifier(const char* formatSpecifier) { + return ELPP->uninstallCustomFormatSpecifier(formatSpecifier); + } + /// @brief Returns true if custom format specifier is installed + static inline bool hasCustomFormatSpecifier(const char* formatSpecifier) { + return ELPP->hasCustomFormatSpecifier(formatSpecifier); + } + static inline void validateFileRolling(Logger* logger, Level level) { + if (ELPP == nullptr || logger == nullptr) return; + logger->m_typedConfigurations->validateFileRolling(level, ELPP->preRollOutCallback()); + } +}; +/// @brief Static helpers to deal with loggers and their configurations +class Loggers : base::StaticClass { + public: + /// @brief Gets existing or registers new logger + static Logger* getLogger(const std::string& identity, bool registerIfNotAvailable = true); + /// @brief Changes default log builder for future loggers + static void setDefaultLogBuilder(el::LogBuilderPtr& logBuilderPtr); + /// @brief Installs logger registration callback, this callback is triggered when new logger is registered + template + static inline bool installLoggerRegistrationCallback(const std::string& id) { + return ELPP->registeredLoggers()->installLoggerRegistrationCallback(id); + } + /// @brief Uninstalls log dispatch callback + template + static inline void uninstallLoggerRegistrationCallback(const std::string& id) { + ELPP->registeredLoggers()->uninstallLoggerRegistrationCallback(id); + } + template + static inline T* loggerRegistrationCallback(const std::string& id) { + return ELPP->registeredLoggers()->loggerRegistrationCallback(id); + } + /// @brief Unregisters logger - use it only when you know what you are doing, you may unregister + /// loggers initialized / used by third-party libs. + static bool unregisterLogger(const std::string& identity); + /// @brief Whether or not logger with id is registered + static bool hasLogger(const std::string& identity); + /// @brief Reconfigures specified logger with new configurations + static Logger* reconfigureLogger(Logger* logger, const Configurations& configurations); + /// @brief Reconfigures logger with new configurations after looking it up using identity + static Logger* reconfigureLogger(const std::string& identity, const Configurations& configurations); + /// @brief Reconfigures logger's single configuration + static Logger* reconfigureLogger(const std::string& identity, ConfigurationType configurationType, + const std::string& value); + /// @brief Reconfigures all the existing loggers with new configurations + static void reconfigureAllLoggers(const Configurations& configurations); + /// @brief Reconfigures single configuration for all the loggers + static inline void reconfigureAllLoggers(ConfigurationType configurationType, const std::string& value) { + reconfigureAllLoggers(Level::Global, configurationType, value); + } + /// @brief Reconfigures single configuration for all the loggers for specified level + static void reconfigureAllLoggers(Level level, ConfigurationType configurationType, + const std::string& value); + /// @brief Sets default configurations. This configuration is used for future (and conditionally for existing) loggers + static void setDefaultConfigurations(const Configurations& configurations, + bool reconfigureExistingLoggers = false); + /// @brief Returns current default + static const Configurations* defaultConfigurations(void); + /// @brief Returns log stream reference pointer if needed by user + static const base::LogStreamsReferenceMap* logStreamsReference(void); + /// @brief Default typed configuration based on existing defaultConf + static base::TypedConfigurations defaultTypedConfigurations(void); + /// @brief Populates all logger IDs in current repository. + /// @param [out] targetList List of fill up. + static std::vector* populateAllLoggerIds(std::vector* targetList); + /// @brief Sets configurations from global configuration file. + static void configureFromGlobal(const char* globalConfigurationFilePath); + /// @brief Configures loggers using command line arg. Ensure you have already set command line args, + /// @return False if invalid argument or argument with no value provided, true if attempted to configure logger. + /// If true is returned that does not mean it has been configured successfully, it only means that it + /// has attempeted to configure logger using configuration file provided in argument + static bool configureFromArg(const char* argKey); + /// @brief Flushes all loggers for all levels - Be careful if you dont know how many loggers are registered + static void flushAll(void); + /// @brief Adds logging flag used internally. + static inline void addFlag(LoggingFlag flag) { + ELPP->addFlag(flag); + } + /// @brief Removes logging flag used internally. + static inline void removeFlag(LoggingFlag flag) { + ELPP->removeFlag(flag); + } + /// @brief Determines whether or not certain flag is active + static inline bool hasFlag(LoggingFlag flag) { + return ELPP->hasFlag(flag); + } + /// @brief Adds flag and removes it when scope goes out + class ScopedAddFlag { + public: + ScopedAddFlag(LoggingFlag flag) : m_flag(flag) { + Loggers::addFlag(m_flag); + } + ~ScopedAddFlag(void) { + Loggers::removeFlag(m_flag); + } + private: + LoggingFlag m_flag; + }; + /// @brief Removes flag and add it when scope goes out + class ScopedRemoveFlag { + public: + ScopedRemoveFlag(LoggingFlag flag) : m_flag(flag) { + Loggers::removeFlag(m_flag); + } + ~ScopedRemoveFlag(void) { + Loggers::addFlag(m_flag); + } + private: + LoggingFlag m_flag; + }; + /// @brief Sets hierarchy for logging. Needs to enable logging flag (HierarchicalLogging) + static void setLoggingLevel(Level level) { + ELPP->setLoggingLevel(level); + } + /// @brief Sets verbose level on the fly + static void setVerboseLevel(base::type::VerboseLevel level); + /// @brief Gets current verbose level + static base::type::VerboseLevel verboseLevel(void); + /// @brief Sets vmodules as specified (on the fly) + static void setVModules(const char* modules); + /// @brief Clears vmodules + static void clearVModules(void); +}; +class VersionInfo : base::StaticClass { + public: + /// @brief Current version number + static const std::string version(void); + + /// @brief Release date of current version + static const std::string releaseDate(void); +}; +} // namespace el +#undef VLOG_IS_ON +/// @brief Determines whether verbose logging is on for specified level current file. +#define VLOG_IS_ON(verboseLevel) (ELPP->vRegistry()->allowed(verboseLevel, __FILE__)) +#undef TIMED_BLOCK +#undef TIMED_SCOPE +#undef TIMED_SCOPE_IF +#undef TIMED_FUNC +#undef TIMED_FUNC_IF +#undef ELPP_MIN_UNIT +#if defined(ELPP_PERFORMANCE_MICROSECONDS) +# define ELPP_MIN_UNIT el::base::TimestampUnit::Microsecond +#else +# define ELPP_MIN_UNIT el::base::TimestampUnit::Millisecond +#endif // (defined(ELPP_PERFORMANCE_MICROSECONDS)) +/// @brief Performance tracked scope. Performance gets written when goes out of scope using +/// 'performance' logger. +/// +/// @detail Please note in order to check the performance at a certain time you can use obj->checkpoint(); +/// @see el::base::PerformanceTracker +/// @see el::base::PerformanceTracker::checkpoint +// Note: Do not surround this definition with null macro because of obj instance +#define TIMED_SCOPE_IF(obj, blockname, condition) el::base::type::PerformanceTrackerPtr obj( condition ? \ + new el::base::PerformanceTracker(blockname, ELPP_MIN_UNIT) : nullptr ) +#define TIMED_SCOPE(obj, blockname) TIMED_SCOPE_IF(obj, blockname, true) +#define TIMED_BLOCK(obj, blockName) for (struct { int i; el::base::type::PerformanceTrackerPtr timer; } obj = { 0, \ + el::base::type::PerformanceTrackerPtr(new el::base::PerformanceTracker(blockName, ELPP_MIN_UNIT)) }; obj.i < 1; ++obj.i) +/// @brief Performance tracked function. Performance gets written when goes out of scope using +/// 'performance' logger. +/// +/// @detail Please note in order to check the performance at a certain time you can use obj->checkpoint(); +/// @see el::base::PerformanceTracker +/// @see el::base::PerformanceTracker::checkpoint +#define TIMED_FUNC_IF(obj,condition) TIMED_SCOPE_IF(obj, ELPP_FUNC, condition) +#define TIMED_FUNC(obj) TIMED_SCOPE(obj, ELPP_FUNC) +#undef PERFORMANCE_CHECKPOINT +#undef PERFORMANCE_CHECKPOINT_WITH_ID +#define PERFORMANCE_CHECKPOINT(obj) obj->checkpoint(std::string(), __FILE__, __LINE__, ELPP_FUNC) +#define PERFORMANCE_CHECKPOINT_WITH_ID(obj, id) obj->checkpoint(id, __FILE__, __LINE__, ELPP_FUNC) +#undef ELPP_COUNTER +#undef ELPP_COUNTER_POS +/// @brief Gets hit counter for file/line +#define ELPP_COUNTER (ELPP->hitCounters()->getCounter(__FILE__, __LINE__)) +/// @brief Gets hit counter position for file/line, -1 if not registered yet +#define ELPP_COUNTER_POS (ELPP_COUNTER == nullptr ? -1 : ELPP_COUNTER->hitCounts()) +// Undef levels to support LOG(LEVEL) +#undef INFO +#undef WARNING +#undef DEBUG +#undef ERROR +#undef FATAL +#undef TRACE +#undef VERBOSE +// Undef existing +#undef CINFO +#undef CWARNING +#undef CDEBUG +#undef CFATAL +#undef CERROR +#undef CTRACE +#undef CVERBOSE +#undef CINFO_IF +#undef CWARNING_IF +#undef CDEBUG_IF +#undef CERROR_IF +#undef CFATAL_IF +#undef CTRACE_IF +#undef CVERBOSE_IF +#undef CINFO_EVERY_N +#undef CWARNING_EVERY_N +#undef CDEBUG_EVERY_N +#undef CERROR_EVERY_N +#undef CFATAL_EVERY_N +#undef CTRACE_EVERY_N +#undef CVERBOSE_EVERY_N +#undef CINFO_AFTER_N +#undef CWARNING_AFTER_N +#undef CDEBUG_AFTER_N +#undef CERROR_AFTER_N +#undef CFATAL_AFTER_N +#undef CTRACE_AFTER_N +#undef CVERBOSE_AFTER_N +#undef CINFO_N_TIMES +#undef CWARNING_N_TIMES +#undef CDEBUG_N_TIMES +#undef CERROR_N_TIMES +#undef CFATAL_N_TIMES +#undef CTRACE_N_TIMES +#undef CVERBOSE_N_TIMES +// Normal logs +#if ELPP_INFO_LOG +# define CINFO(writer, dispatchAction, ...) ELPP_WRITE_LOG(writer, el::Level::Info, dispatchAction, __VA_ARGS__) +#else +# define CINFO(writer, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_INFO_LOG +#if ELPP_WARNING_LOG +# define CWARNING(writer, dispatchAction, ...) ELPP_WRITE_LOG(writer, el::Level::Warning, dispatchAction, __VA_ARGS__) +#else +# define CWARNING(writer, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_WARNING_LOG +#if ELPP_DEBUG_LOG +# define CDEBUG(writer, dispatchAction, ...) ELPP_WRITE_LOG(writer, el::Level::Debug, dispatchAction, __VA_ARGS__) +#else +# define CDEBUG(writer, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_DEBUG_LOG +#if ELPP_ERROR_LOG +# define CERROR(writer, dispatchAction, ...) ELPP_WRITE_LOG(writer, el::Level::Error, dispatchAction, __VA_ARGS__) +#else +# define CERROR(writer, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_ERROR_LOG +#if ELPP_FATAL_LOG +# define CFATAL(writer, dispatchAction, ...) ELPP_WRITE_LOG(writer, el::Level::Fatal, dispatchAction, __VA_ARGS__) +#else +# define CFATAL(writer, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_FATAL_LOG +#if ELPP_TRACE_LOG +# define CTRACE(writer, dispatchAction, ...) ELPP_WRITE_LOG(writer, el::Level::Trace, dispatchAction, __VA_ARGS__) +#else +# define CTRACE(writer, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_TRACE_LOG +#if ELPP_VERBOSE_LOG +# define CVERBOSE(writer, vlevel, dispatchAction, ...) if (VLOG_IS_ON(vlevel)) writer(\ +el::Level::Verbose, __FILE__, __LINE__, ELPP_FUNC, dispatchAction, vlevel).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) +#else +# define CVERBOSE(writer, vlevel, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_VERBOSE_LOG +// Conditional logs +#if ELPP_INFO_LOG +# define CINFO_IF(writer, condition_, dispatchAction, ...) \ +ELPP_WRITE_LOG_IF(writer, (condition_), el::Level::Info, dispatchAction, __VA_ARGS__) +#else +# define CINFO_IF(writer, condition_, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_INFO_LOG +#if ELPP_WARNING_LOG +# define CWARNING_IF(writer, condition_, dispatchAction, ...)\ +ELPP_WRITE_LOG_IF(writer, (condition_), el::Level::Warning, dispatchAction, __VA_ARGS__) +#else +# define CWARNING_IF(writer, condition_, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_WARNING_LOG +#if ELPP_DEBUG_LOG +# define CDEBUG_IF(writer, condition_, dispatchAction, ...)\ +ELPP_WRITE_LOG_IF(writer, (condition_), el::Level::Debug, dispatchAction, __VA_ARGS__) +#else +# define CDEBUG_IF(writer, condition_, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_DEBUG_LOG +#if ELPP_ERROR_LOG +# define CERROR_IF(writer, condition_, dispatchAction, ...)\ +ELPP_WRITE_LOG_IF(writer, (condition_), el::Level::Error, dispatchAction, __VA_ARGS__) +#else +# define CERROR_IF(writer, condition_, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_ERROR_LOG +#if ELPP_FATAL_LOG +# define CFATAL_IF(writer, condition_, dispatchAction, ...)\ +ELPP_WRITE_LOG_IF(writer, (condition_), el::Level::Fatal, dispatchAction, __VA_ARGS__) +#else +# define CFATAL_IF(writer, condition_, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_FATAL_LOG +#if ELPP_TRACE_LOG +# define CTRACE_IF(writer, condition_, dispatchAction, ...)\ +ELPP_WRITE_LOG_IF(writer, (condition_), el::Level::Trace, dispatchAction, __VA_ARGS__) +#else +# define CTRACE_IF(writer, condition_, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_TRACE_LOG +#if ELPP_VERBOSE_LOG +# define CVERBOSE_IF(writer, condition_, vlevel, dispatchAction, ...) if (VLOG_IS_ON(vlevel) && (condition_)) writer( \ +el::Level::Verbose, __FILE__, __LINE__, ELPP_FUNC, dispatchAction, vlevel).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) +#else +# define CVERBOSE_IF(writer, condition_, vlevel, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_VERBOSE_LOG +// Occasional logs +#if ELPP_INFO_LOG +# define CINFO_EVERY_N(writer, occasion, dispatchAction, ...)\ +ELPP_WRITE_LOG_EVERY_N(writer, occasion, el::Level::Info, dispatchAction, __VA_ARGS__) +#else +# define CINFO_EVERY_N(writer, occasion, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_INFO_LOG +#if ELPP_WARNING_LOG +# define CWARNING_EVERY_N(writer, occasion, dispatchAction, ...)\ +ELPP_WRITE_LOG_EVERY_N(writer, occasion, el::Level::Warning, dispatchAction, __VA_ARGS__) +#else +# define CWARNING_EVERY_N(writer, occasion, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_WARNING_LOG +#if ELPP_DEBUG_LOG +# define CDEBUG_EVERY_N(writer, occasion, dispatchAction, ...)\ +ELPP_WRITE_LOG_EVERY_N(writer, occasion, el::Level::Debug, dispatchAction, __VA_ARGS__) +#else +# define CDEBUG_EVERY_N(writer, occasion, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_DEBUG_LOG +#if ELPP_ERROR_LOG +# define CERROR_EVERY_N(writer, occasion, dispatchAction, ...)\ +ELPP_WRITE_LOG_EVERY_N(writer, occasion, el::Level::Error, dispatchAction, __VA_ARGS__) +#else +# define CERROR_EVERY_N(writer, occasion, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_ERROR_LOG +#if ELPP_FATAL_LOG +# define CFATAL_EVERY_N(writer, occasion, dispatchAction, ...)\ +ELPP_WRITE_LOG_EVERY_N(writer, occasion, el::Level::Fatal, dispatchAction, __VA_ARGS__) +#else +# define CFATAL_EVERY_N(writer, occasion, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_FATAL_LOG +#if ELPP_TRACE_LOG +# define CTRACE_EVERY_N(writer, occasion, dispatchAction, ...)\ +ELPP_WRITE_LOG_EVERY_N(writer, occasion, el::Level::Trace, dispatchAction, __VA_ARGS__) +#else +# define CTRACE_EVERY_N(writer, occasion, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_TRACE_LOG +#if ELPP_VERBOSE_LOG +# define CVERBOSE_EVERY_N(writer, occasion, vlevel, dispatchAction, ...)\ +CVERBOSE_IF(writer, ELPP->validateEveryNCounter(__FILE__, __LINE__, occasion), vlevel, dispatchAction, __VA_ARGS__) +#else +# define CVERBOSE_EVERY_N(writer, occasion, vlevel, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_VERBOSE_LOG +// After N logs +#if ELPP_INFO_LOG +# define CINFO_AFTER_N(writer, n, dispatchAction, ...)\ +ELPP_WRITE_LOG_AFTER_N(writer, n, el::Level::Info, dispatchAction, __VA_ARGS__) +#else +# define CINFO_AFTER_N(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_INFO_LOG +#if ELPP_WARNING_LOG +# define CWARNING_AFTER_N(writer, n, dispatchAction, ...)\ +ELPP_WRITE_LOG_AFTER_N(writer, n, el::Level::Warning, dispatchAction, __VA_ARGS__) +#else +# define CWARNING_AFTER_N(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_WARNING_LOG +#if ELPP_DEBUG_LOG +# define CDEBUG_AFTER_N(writer, n, dispatchAction, ...)\ +ELPP_WRITE_LOG_AFTER_N(writer, n, el::Level::Debug, dispatchAction, __VA_ARGS__) +#else +# define CDEBUG_AFTER_N(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_DEBUG_LOG +#if ELPP_ERROR_LOG +# define CERROR_AFTER_N(writer, n, dispatchAction, ...)\ +ELPP_WRITE_LOG_AFTER_N(writer, n, el::Level::Error, dispatchAction, __VA_ARGS__) +#else +# define CERROR_AFTER_N(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_ERROR_LOG +#if ELPP_FATAL_LOG +# define CFATAL_AFTER_N(writer, n, dispatchAction, ...)\ +ELPP_WRITE_LOG_AFTER_N(writer, n, el::Level::Fatal, dispatchAction, __VA_ARGS__) +#else +# define CFATAL_AFTER_N(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_FATAL_LOG +#if ELPP_TRACE_LOG +# define CTRACE_AFTER_N(writer, n, dispatchAction, ...)\ +ELPP_WRITE_LOG_AFTER_N(writer, n, el::Level::Trace, dispatchAction, __VA_ARGS__) +#else +# define CTRACE_AFTER_N(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_TRACE_LOG +#if ELPP_VERBOSE_LOG +# define CVERBOSE_AFTER_N(writer, n, vlevel, dispatchAction, ...)\ +CVERBOSE_IF(writer, ELPP->validateAfterNCounter(__FILE__, __LINE__, n), vlevel, dispatchAction, __VA_ARGS__) +#else +# define CVERBOSE_AFTER_N(writer, n, vlevel, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_VERBOSE_LOG +// N Times logs +#if ELPP_INFO_LOG +# define CINFO_N_TIMES(writer, n, dispatchAction, ...)\ +ELPP_WRITE_LOG_N_TIMES(writer, n, el::Level::Info, dispatchAction, __VA_ARGS__) +#else +# define CINFO_N_TIMES(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_INFO_LOG +#if ELPP_WARNING_LOG +# define CWARNING_N_TIMES(writer, n, dispatchAction, ...)\ +ELPP_WRITE_LOG_N_TIMES(writer, n, el::Level::Warning, dispatchAction, __VA_ARGS__) +#else +# define CWARNING_N_TIMES(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_WARNING_LOG +#if ELPP_DEBUG_LOG +# define CDEBUG_N_TIMES(writer, n, dispatchAction, ...)\ +ELPP_WRITE_LOG_N_TIMES(writer, n, el::Level::Debug, dispatchAction, __VA_ARGS__) +#else +# define CDEBUG_N_TIMES(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_DEBUG_LOG +#if ELPP_ERROR_LOG +# define CERROR_N_TIMES(writer, n, dispatchAction, ...)\ +ELPP_WRITE_LOG_N_TIMES(writer, n, el::Level::Error, dispatchAction, __VA_ARGS__) +#else +# define CERROR_N_TIMES(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_ERROR_LOG +#if ELPP_FATAL_LOG +# define CFATAL_N_TIMES(writer, n, dispatchAction, ...)\ +ELPP_WRITE_LOG_N_TIMES(writer, n, el::Level::Fatal, dispatchAction, __VA_ARGS__) +#else +# define CFATAL_N_TIMES(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_FATAL_LOG +#if ELPP_TRACE_LOG +# define CTRACE_N_TIMES(writer, n, dispatchAction, ...)\ +ELPP_WRITE_LOG_N_TIMES(writer, n, el::Level::Trace, dispatchAction, __VA_ARGS__) +#else +# define CTRACE_N_TIMES(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_TRACE_LOG +#if ELPP_VERBOSE_LOG +# define CVERBOSE_N_TIMES(writer, n, vlevel, dispatchAction, ...)\ +CVERBOSE_IF(writer, ELPP->validateNTimesCounter(__FILE__, __LINE__, n), vlevel, dispatchAction, __VA_ARGS__) +#else +# define CVERBOSE_N_TIMES(writer, n, vlevel, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_VERBOSE_LOG +// +// Custom Loggers - Requires (level, dispatchAction, loggerId/s) +// +// undef existing +#undef CLOG +#undef CLOG_VERBOSE +#undef CVLOG +#undef CLOG_IF +#undef CLOG_VERBOSE_IF +#undef CVLOG_IF +#undef CLOG_EVERY_N +#undef CVLOG_EVERY_N +#undef CLOG_AFTER_N +#undef CVLOG_AFTER_N +#undef CLOG_N_TIMES +#undef CVLOG_N_TIMES +// Normal logs +#define CLOG(LEVEL, ...)\ +C##LEVEL(el::base::Writer, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define CVLOG(vlevel, ...) CVERBOSE(el::base::Writer, vlevel, el::base::DispatchAction::NormalLog, __VA_ARGS__) +// Conditional logs +#define CLOG_IF(condition, LEVEL, ...)\ +C##LEVEL##_IF(el::base::Writer, condition, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define CVLOG_IF(condition, vlevel, ...)\ +CVERBOSE_IF(el::base::Writer, condition, vlevel, el::base::DispatchAction::NormalLog, __VA_ARGS__) +// Hit counts based logs +#define CLOG_EVERY_N(n, LEVEL, ...)\ +C##LEVEL##_EVERY_N(el::base::Writer, n, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define CVLOG_EVERY_N(n, vlevel, ...)\ +CVERBOSE_EVERY_N(el::base::Writer, n, vlevel, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define CLOG_AFTER_N(n, LEVEL, ...)\ +C##LEVEL##_AFTER_N(el::base::Writer, n, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define CVLOG_AFTER_N(n, vlevel, ...)\ +CVERBOSE_AFTER_N(el::base::Writer, n, vlevel, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define CLOG_N_TIMES(n, LEVEL, ...)\ +C##LEVEL##_N_TIMES(el::base::Writer, n, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define CVLOG_N_TIMES(n, vlevel, ...)\ +CVERBOSE_N_TIMES(el::base::Writer, n, vlevel, el::base::DispatchAction::NormalLog, __VA_ARGS__) +// +// Default Loggers macro using CLOG(), CLOG_VERBOSE() and CVLOG() macros +// +// undef existing +#undef LOG +#undef VLOG +#undef LOG_IF +#undef VLOG_IF +#undef LOG_EVERY_N +#undef VLOG_EVERY_N +#undef LOG_AFTER_N +#undef VLOG_AFTER_N +#undef LOG_N_TIMES +#undef VLOG_N_TIMES +#undef ELPP_CURR_FILE_LOGGER_ID +#if defined(ELPP_DEFAULT_LOGGER) +# define ELPP_CURR_FILE_LOGGER_ID ELPP_DEFAULT_LOGGER +#else +# define ELPP_CURR_FILE_LOGGER_ID el::base::consts::kDefaultLoggerId +#endif +#undef ELPP_TRACE +#define ELPP_TRACE CLOG(TRACE, ELPP_CURR_FILE_LOGGER_ID) +// Normal logs +#define LOG(LEVEL) CLOG(LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define VLOG(vlevel) CVLOG(vlevel, ELPP_CURR_FILE_LOGGER_ID) +// Conditional logs +#define LOG_IF(condition, LEVEL) CLOG_IF(condition, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define VLOG_IF(condition, vlevel) CVLOG_IF(condition, vlevel, ELPP_CURR_FILE_LOGGER_ID) +// Hit counts based logs +#define LOG_EVERY_N(n, LEVEL) CLOG_EVERY_N(n, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define VLOG_EVERY_N(n, vlevel) CVLOG_EVERY_N(n, vlevel, ELPP_CURR_FILE_LOGGER_ID) +#define LOG_AFTER_N(n, LEVEL) CLOG_AFTER_N(n, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define VLOG_AFTER_N(n, vlevel) CVLOG_AFTER_N(n, vlevel, ELPP_CURR_FILE_LOGGER_ID) +#define LOG_N_TIMES(n, LEVEL) CLOG_N_TIMES(n, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define VLOG_N_TIMES(n, vlevel) CVLOG_N_TIMES(n, vlevel, ELPP_CURR_FILE_LOGGER_ID) +// Generic PLOG() +#undef CPLOG +#undef CPLOG_IF +#undef PLOG +#undef PLOG_IF +#undef DCPLOG +#undef DCPLOG_IF +#undef DPLOG +#undef DPLOG_IF +#define CPLOG(LEVEL, ...)\ +C##LEVEL(el::base::PErrorWriter, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define CPLOG_IF(condition, LEVEL, ...)\ +C##LEVEL##_IF(el::base::PErrorWriter, condition, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define DCPLOG(LEVEL, ...)\ +if (ELPP_DEBUG_LOG) C##LEVEL(el::base::PErrorWriter, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define DCPLOG_IF(condition, LEVEL, ...)\ +C##LEVEL##_IF(el::base::PErrorWriter, (ELPP_DEBUG_LOG) && (condition), el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define PLOG(LEVEL) CPLOG(LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define PLOG_IF(condition, LEVEL) CPLOG_IF(condition, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define DPLOG(LEVEL) DCPLOG(LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define DPLOG_IF(condition, LEVEL) DCPLOG_IF(condition, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +// Generic SYSLOG() +#undef CSYSLOG +#undef CSYSLOG_IF +#undef CSYSLOG_EVERY_N +#undef CSYSLOG_AFTER_N +#undef CSYSLOG_N_TIMES +#undef SYSLOG +#undef SYSLOG_IF +#undef SYSLOG_EVERY_N +#undef SYSLOG_AFTER_N +#undef SYSLOG_N_TIMES +#undef DCSYSLOG +#undef DCSYSLOG_IF +#undef DCSYSLOG_EVERY_N +#undef DCSYSLOG_AFTER_N +#undef DCSYSLOG_N_TIMES +#undef DSYSLOG +#undef DSYSLOG_IF +#undef DSYSLOG_EVERY_N +#undef DSYSLOG_AFTER_N +#undef DSYSLOG_N_TIMES +#if defined(ELPP_SYSLOG) +# define CSYSLOG(LEVEL, ...)\ +C##LEVEL(el::base::Writer, el::base::DispatchAction::SysLog, __VA_ARGS__) +# define CSYSLOG_IF(condition, LEVEL, ...)\ +C##LEVEL##_IF(el::base::Writer, condition, el::base::DispatchAction::SysLog, __VA_ARGS__) +# define CSYSLOG_EVERY_N(n, LEVEL, ...) C##LEVEL##_EVERY_N(el::base::Writer, n, el::base::DispatchAction::SysLog, __VA_ARGS__) +# define CSYSLOG_AFTER_N(n, LEVEL, ...) C##LEVEL##_AFTER_N(el::base::Writer, n, el::base::DispatchAction::SysLog, __VA_ARGS__) +# define CSYSLOG_N_TIMES(n, LEVEL, ...) C##LEVEL##_N_TIMES(el::base::Writer, n, el::base::DispatchAction::SysLog, __VA_ARGS__) +# define SYSLOG(LEVEL) CSYSLOG(LEVEL, el::base::consts::kSysLogLoggerId) +# define SYSLOG_IF(condition, LEVEL) CSYSLOG_IF(condition, LEVEL, el::base::consts::kSysLogLoggerId) +# define SYSLOG_EVERY_N(n, LEVEL) CSYSLOG_EVERY_N(n, LEVEL, el::base::consts::kSysLogLoggerId) +# define SYSLOG_AFTER_N(n, LEVEL) CSYSLOG_AFTER_N(n, LEVEL, el::base::consts::kSysLogLoggerId) +# define SYSLOG_N_TIMES(n, LEVEL) CSYSLOG_N_TIMES(n, LEVEL, el::base::consts::kSysLogLoggerId) +# define DCSYSLOG(LEVEL, ...) if (ELPP_DEBUG_LOG) C##LEVEL(el::base::Writer, el::base::DispatchAction::SysLog, __VA_ARGS__) +# define DCSYSLOG_IF(condition, LEVEL, ...)\ +C##LEVEL##_IF(el::base::Writer, (ELPP_DEBUG_LOG) && (condition), el::base::DispatchAction::SysLog, __VA_ARGS__) +# define DCSYSLOG_EVERY_N(n, LEVEL, ...)\ +if (ELPP_DEBUG_LOG) C##LEVEL##_EVERY_N(el::base::Writer, n, el::base::DispatchAction::SysLog, __VA_ARGS__) +# define DCSYSLOG_AFTER_N(n, LEVEL, ...)\ +if (ELPP_DEBUG_LOG) C##LEVEL##_AFTER_N(el::base::Writer, n, el::base::DispatchAction::SysLog, __VA_ARGS__) +# define DCSYSLOG_N_TIMES(n, LEVEL, ...)\ +if (ELPP_DEBUG_LOG) C##LEVEL##_EVERY_N(el::base::Writer, n, el::base::DispatchAction::SysLog, __VA_ARGS__) +# define DSYSLOG(LEVEL) DCSYSLOG(LEVEL, el::base::consts::kSysLogLoggerId) +# define DSYSLOG_IF(condition, LEVEL) DCSYSLOG_IF(condition, LEVEL, el::base::consts::kSysLogLoggerId) +# define DSYSLOG_EVERY_N(n, LEVEL) DCSYSLOG_EVERY_N(n, LEVEL, el::base::consts::kSysLogLoggerId) +# define DSYSLOG_AFTER_N(n, LEVEL) DCSYSLOG_AFTER_N(n, LEVEL, el::base::consts::kSysLogLoggerId) +# define DSYSLOG_N_TIMES(n, LEVEL) DCSYSLOG_N_TIMES(n, LEVEL, el::base::consts::kSysLogLoggerId) +#else +# define CSYSLOG(LEVEL, ...) el::base::NullWriter() +# define CSYSLOG_IF(condition, LEVEL, ...) el::base::NullWriter() +# define CSYSLOG_EVERY_N(n, LEVEL, ...) el::base::NullWriter() +# define CSYSLOG_AFTER_N(n, LEVEL, ...) el::base::NullWriter() +# define CSYSLOG_N_TIMES(n, LEVEL, ...) el::base::NullWriter() +# define SYSLOG(LEVEL) el::base::NullWriter() +# define SYSLOG_IF(condition, LEVEL) el::base::NullWriter() +# define SYSLOG_EVERY_N(n, LEVEL) el::base::NullWriter() +# define SYSLOG_AFTER_N(n, LEVEL) el::base::NullWriter() +# define SYSLOG_N_TIMES(n, LEVEL) el::base::NullWriter() +# define DCSYSLOG(LEVEL, ...) el::base::NullWriter() +# define DCSYSLOG_IF(condition, LEVEL, ...) el::base::NullWriter() +# define DCSYSLOG_EVERY_N(n, LEVEL, ...) el::base::NullWriter() +# define DCSYSLOG_AFTER_N(n, LEVEL, ...) el::base::NullWriter() +# define DCSYSLOG_N_TIMES(n, LEVEL, ...) el::base::NullWriter() +# define DSYSLOG(LEVEL) el::base::NullWriter() +# define DSYSLOG_IF(condition, LEVEL) el::base::NullWriter() +# define DSYSLOG_EVERY_N(n, LEVEL) el::base::NullWriter() +# define DSYSLOG_AFTER_N(n, LEVEL) el::base::NullWriter() +# define DSYSLOG_N_TIMES(n, LEVEL) el::base::NullWriter() +#endif // defined(ELPP_SYSLOG) +// +// Custom Debug Only Loggers - Requires (level, loggerId/s) +// +// undef existing +#undef DCLOG +#undef DCVLOG +#undef DCLOG_IF +#undef DCVLOG_IF +#undef DCLOG_EVERY_N +#undef DCVLOG_EVERY_N +#undef DCLOG_AFTER_N +#undef DCVLOG_AFTER_N +#undef DCLOG_N_TIMES +#undef DCVLOG_N_TIMES +// Normal logs +#define DCLOG(LEVEL, ...) if (ELPP_DEBUG_LOG) CLOG(LEVEL, __VA_ARGS__) +#define DCLOG_VERBOSE(vlevel, ...) if (ELPP_DEBUG_LOG) CLOG_VERBOSE(vlevel, __VA_ARGS__) +#define DCVLOG(vlevel, ...) if (ELPP_DEBUG_LOG) CVLOG(vlevel, __VA_ARGS__) +// Conditional logs +#define DCLOG_IF(condition, LEVEL, ...) if (ELPP_DEBUG_LOG) CLOG_IF(condition, LEVEL, __VA_ARGS__) +#define DCVLOG_IF(condition, vlevel, ...) if (ELPP_DEBUG_LOG) CVLOG_IF(condition, vlevel, __VA_ARGS__) +// Hit counts based logs +#define DCLOG_EVERY_N(n, LEVEL, ...) if (ELPP_DEBUG_LOG) CLOG_EVERY_N(n, LEVEL, __VA_ARGS__) +#define DCVLOG_EVERY_N(n, vlevel, ...) if (ELPP_DEBUG_LOG) CVLOG_EVERY_N(n, vlevel, __VA_ARGS__) +#define DCLOG_AFTER_N(n, LEVEL, ...) if (ELPP_DEBUG_LOG) CLOG_AFTER_N(n, LEVEL, __VA_ARGS__) +#define DCVLOG_AFTER_N(n, vlevel, ...) if (ELPP_DEBUG_LOG) CVLOG_AFTER_N(n, vlevel, __VA_ARGS__) +#define DCLOG_N_TIMES(n, LEVEL, ...) if (ELPP_DEBUG_LOG) CLOG_N_TIMES(n, LEVEL, __VA_ARGS__) +#define DCVLOG_N_TIMES(n, vlevel, ...) if (ELPP_DEBUG_LOG) CVLOG_N_TIMES(n, vlevel, __VA_ARGS__) +// +// Default Debug Only Loggers macro using CLOG(), CLOG_VERBOSE() and CVLOG() macros +// +#if !defined(ELPP_NO_DEBUG_MACROS) +// undef existing +#undef DLOG +#undef DVLOG +#undef DLOG_IF +#undef DVLOG_IF +#undef DLOG_EVERY_N +#undef DVLOG_EVERY_N +#undef DLOG_AFTER_N +#undef DVLOG_AFTER_N +#undef DLOG_N_TIMES +#undef DVLOG_N_TIMES +// Normal logs +#define DLOG(LEVEL) DCLOG(LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define DVLOG(vlevel) DCVLOG(vlevel, ELPP_CURR_FILE_LOGGER_ID) +// Conditional logs +#define DLOG_IF(condition, LEVEL) DCLOG_IF(condition, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define DVLOG_IF(condition, vlevel) DCVLOG_IF(condition, vlevel, ELPP_CURR_FILE_LOGGER_ID) +// Hit counts based logs +#define DLOG_EVERY_N(n, LEVEL) DCLOG_EVERY_N(n, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define DVLOG_EVERY_N(n, vlevel) DCVLOG_EVERY_N(n, vlevel, ELPP_CURR_FILE_LOGGER_ID) +#define DLOG_AFTER_N(n, LEVEL) DCLOG_AFTER_N(n, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define DVLOG_AFTER_N(n, vlevel) DCVLOG_AFTER_N(n, vlevel, ELPP_CURR_FILE_LOGGER_ID) +#define DLOG_N_TIMES(n, LEVEL) DCLOG_N_TIMES(n, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define DVLOG_N_TIMES(n, vlevel) DCVLOG_N_TIMES(n, vlevel, ELPP_CURR_FILE_LOGGER_ID) +#endif // defined(ELPP_NO_DEBUG_MACROS) +#if !defined(ELPP_NO_CHECK_MACROS) +// Check macros +#undef CCHECK +#undef CPCHECK +#undef CCHECK_EQ +#undef CCHECK_NE +#undef CCHECK_LT +#undef CCHECK_GT +#undef CCHECK_LE +#undef CCHECK_GE +#undef CCHECK_BOUNDS +#undef CCHECK_NOTNULL +#undef CCHECK_STRCASEEQ +#undef CCHECK_STRCASENE +#undef CHECK +#undef PCHECK +#undef CHECK_EQ +#undef CHECK_NE +#undef CHECK_LT +#undef CHECK_GT +#undef CHECK_LE +#undef CHECK_GE +#undef CHECK_BOUNDS +#undef CHECK_NOTNULL +#undef CHECK_STRCASEEQ +#undef CHECK_STRCASENE +#define CCHECK(condition, ...) CLOG_IF(!(condition), FATAL, __VA_ARGS__) << "Check failed: [" << #condition << "] " +#define CPCHECK(condition, ...) CPLOG_IF(!(condition), FATAL, __VA_ARGS__) << "Check failed: [" << #condition << "] " +#define CHECK(condition) CCHECK(condition, ELPP_CURR_FILE_LOGGER_ID) +#define PCHECK(condition) CPCHECK(condition, ELPP_CURR_FILE_LOGGER_ID) +#define CCHECK_EQ(a, b, ...) CCHECK(a == b, __VA_ARGS__) +#define CCHECK_NE(a, b, ...) CCHECK(a != b, __VA_ARGS__) +#define CCHECK_LT(a, b, ...) CCHECK(a < b, __VA_ARGS__) +#define CCHECK_GT(a, b, ...) CCHECK(a > b, __VA_ARGS__) +#define CCHECK_LE(a, b, ...) CCHECK(a <= b, __VA_ARGS__) +#define CCHECK_GE(a, b, ...) CCHECK(a >= b, __VA_ARGS__) +#define CCHECK_BOUNDS(val, min, max, ...) CCHECK(val >= min && val <= max, __VA_ARGS__) +#define CHECK_EQ(a, b) CCHECK_EQ(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_NE(a, b) CCHECK_NE(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_LT(a, b) CCHECK_LT(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_GT(a, b) CCHECK_GT(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_LE(a, b) CCHECK_LE(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_GE(a, b) CCHECK_GE(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_BOUNDS(val, min, max) CCHECK_BOUNDS(val, min, max, ELPP_CURR_FILE_LOGGER_ID) +#define CCHECK_NOTNULL(ptr, ...) CCHECK((ptr) != nullptr, __VA_ARGS__) +#define CCHECK_STREQ(str1, str2, ...) CLOG_IF(!el::base::utils::Str::cStringEq(str1, str2), FATAL, __VA_ARGS__) \ +<< "Check failed: [" << #str1 << " == " << #str2 << "] " +#define CCHECK_STRNE(str1, str2, ...) CLOG_IF(el::base::utils::Str::cStringEq(str1, str2), FATAL, __VA_ARGS__) \ +<< "Check failed: [" << #str1 << " != " << #str2 << "] " +#define CCHECK_STRCASEEQ(str1, str2, ...) CLOG_IF(!el::base::utils::Str::cStringCaseEq(str1, str2), FATAL, __VA_ARGS__) \ +<< "Check failed: [" << #str1 << " == " << #str2 << "] " +#define CCHECK_STRCASENE(str1, str2, ...) CLOG_IF(el::base::utils::Str::cStringCaseEq(str1, str2), FATAL, __VA_ARGS__) \ +<< "Check failed: [" << #str1 << " != " << #str2 << "] " +#define CHECK_NOTNULL(ptr) CCHECK_NOTNULL((ptr), ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_STREQ(str1, str2) CCHECK_STREQ(str1, str2, ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_STRNE(str1, str2) CCHECK_STRNE(str1, str2, ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_STRCASEEQ(str1, str2) CCHECK_STRCASEEQ(str1, str2, ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_STRCASENE(str1, str2) CCHECK_STRCASENE(str1, str2, ELPP_CURR_FILE_LOGGER_ID) +#undef DCCHECK +#undef DCCHECK_EQ +#undef DCCHECK_NE +#undef DCCHECK_LT +#undef DCCHECK_GT +#undef DCCHECK_LE +#undef DCCHECK_GE +#undef DCCHECK_BOUNDS +#undef DCCHECK_NOTNULL +#undef DCCHECK_STRCASEEQ +#undef DCCHECK_STRCASENE +#undef DCPCHECK +#undef DCHECK +#undef DCHECK_EQ +#undef DCHECK_NE +#undef DCHECK_LT +#undef DCHECK_GT +#undef DCHECK_LE +#undef DCHECK_GE +#undef DCHECK_BOUNDS_ +#undef DCHECK_NOTNULL +#undef DCHECK_STRCASEEQ +#undef DCHECK_STRCASENE +#undef DPCHECK +#define DCCHECK(condition, ...) if (ELPP_DEBUG_LOG) CCHECK(condition, __VA_ARGS__) +#define DCCHECK_EQ(a, b, ...) if (ELPP_DEBUG_LOG) CCHECK_EQ(a, b, __VA_ARGS__) +#define DCCHECK_NE(a, b, ...) if (ELPP_DEBUG_LOG) CCHECK_NE(a, b, __VA_ARGS__) +#define DCCHECK_LT(a, b, ...) if (ELPP_DEBUG_LOG) CCHECK_LT(a, b, __VA_ARGS__) +#define DCCHECK_GT(a, b, ...) if (ELPP_DEBUG_LOG) CCHECK_GT(a, b, __VA_ARGS__) +#define DCCHECK_LE(a, b, ...) if (ELPP_DEBUG_LOG) CCHECK_LE(a, b, __VA_ARGS__) +#define DCCHECK_GE(a, b, ...) if (ELPP_DEBUG_LOG) CCHECK_GE(a, b, __VA_ARGS__) +#define DCCHECK_BOUNDS(val, min, max, ...) if (ELPP_DEBUG_LOG) CCHECK_BOUNDS(val, min, max, __VA_ARGS__) +#define DCCHECK_NOTNULL(ptr, ...) if (ELPP_DEBUG_LOG) CCHECK_NOTNULL((ptr), __VA_ARGS__) +#define DCCHECK_STREQ(str1, str2, ...) if (ELPP_DEBUG_LOG) CCHECK_STREQ(str1, str2, __VA_ARGS__) +#define DCCHECK_STRNE(str1, str2, ...) if (ELPP_DEBUG_LOG) CCHECK_STRNE(str1, str2, __VA_ARGS__) +#define DCCHECK_STRCASEEQ(str1, str2, ...) if (ELPP_DEBUG_LOG) CCHECK_STRCASEEQ(str1, str2, __VA_ARGS__) +#define DCCHECK_STRCASENE(str1, str2, ...) if (ELPP_DEBUG_LOG) CCHECK_STRCASENE(str1, str2, __VA_ARGS__) +#define DCPCHECK(condition, ...) if (ELPP_DEBUG_LOG) CPCHECK(condition, __VA_ARGS__) +#define DCHECK(condition) DCCHECK(condition, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_EQ(a, b) DCCHECK_EQ(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_NE(a, b) DCCHECK_NE(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_LT(a, b) DCCHECK_LT(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_GT(a, b) DCCHECK_GT(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_LE(a, b) DCCHECK_LE(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_GE(a, b) DCCHECK_GE(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_BOUNDS(val, min, max) DCCHECK_BOUNDS(val, min, max, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_NOTNULL(ptr) DCCHECK_NOTNULL((ptr), ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_STREQ(str1, str2) DCCHECK_STREQ(str1, str2, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_STRNE(str1, str2) DCCHECK_STRNE(str1, str2, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_STRCASEEQ(str1, str2) DCCHECK_STRCASEEQ(str1, str2, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_STRCASENE(str1, str2) DCCHECK_STRCASENE(str1, str2, ELPP_CURR_FILE_LOGGER_ID) +#define DPCHECK(condition) DCPCHECK(condition, ELPP_CURR_FILE_LOGGER_ID) +#endif // defined(ELPP_NO_CHECK_MACROS) +#if defined(ELPP_DISABLE_DEFAULT_CRASH_HANDLING) +# define ELPP_USE_DEF_CRASH_HANDLER false +#else +# define ELPP_USE_DEF_CRASH_HANDLER true +#endif // defined(ELPP_DISABLE_DEFAULT_CRASH_HANDLING) +#define ELPP_CRASH_HANDLER_INIT +#define ELPP_INIT_EASYLOGGINGPP(val) \ +namespace el { \ +namespace base { \ +el::base::type::StoragePointer elStorage(val); \ +} \ +el::base::debug::CrashHandler elCrashHandler(ELPP_USE_DEF_CRASH_HANDLER); \ +} + +#if ELPP_ASYNC_LOGGING +# define INITIALIZE_EASYLOGGINGPP ELPP_INIT_EASYLOGGINGPP(new el::base::Storage(el::LogBuilderPtr(new el::base::DefaultLogBuilder()),\ +new el::base::AsyncDispatchWorker())) +#else +# define INITIALIZE_EASYLOGGINGPP ELPP_INIT_EASYLOGGINGPP(new el::base::Storage(el::LogBuilderPtr(new el::base::DefaultLogBuilder()))) +#endif // ELPP_ASYNC_LOGGING +#define INITIALIZE_NULL_EASYLOGGINGPP \ +namespace el {\ +namespace base {\ +el::base::type::StoragePointer elStorage;\ +}\ +el::base::debug::CrashHandler elCrashHandler(ELPP_USE_DEF_CRASH_HANDLER);\ +} +#define SHARE_EASYLOGGINGPP(initializedStorage)\ +namespace el {\ +namespace base {\ +el::base::type::StoragePointer elStorage(initializedStorage);\ +}\ +el::base::debug::CrashHandler elCrashHandler(ELPP_USE_DEF_CRASH_HANDLER);\ +} + +#if defined(ELPP_UNICODE) +# define START_EASYLOGGINGPP(argc, argv) el::Helpers::setArgs(argc, argv); std::locale::global(std::locale("")) +#else +# define START_EASYLOGGINGPP(argc, argv) el::Helpers::setArgs(argc, argv) +#endif // defined(ELPP_UNICODE) +#endif // EASYLOGGINGPP_H diff --git a/cpp/src/wrapper/knowhere/wrapper_log.h b/cpp/src/wrapper/knowhere/wrapper_log.h index 39ca78092b..7cde925292 100644 --- a/cpp/src/wrapper/knowhere/wrapper_log.h +++ b/cpp/src/wrapper/knowhere/wrapper_log.h @@ -6,7 +6,7 @@ #pragma once -#include +#include "utils/easylogging++.h" namespace zilliz { namespace milvus { diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 253ea2d794..978ca69ff6 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -25,7 +25,7 @@ set(unittest_libs gmock gtest_main gmock_main - easyloggingpp +# easyloggingpp pthread metrics gfortran diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index eae0b9bb83..8c203f329f 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -19,7 +19,9 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs) aux_source_directory(./ test_srcs) set(util_files - ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp) + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp + ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc + ${MILVUS_ENGINE_SRC}/utils/easylogging++.h) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index bf177475a7..05e7ba4def 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -12,7 +12,7 @@ #include "utils/CommonUtil.h" #include -#include +#include "utils/easylogging++.h" #include diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index b23a634b7b..2f7a8394e5 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -5,7 +5,7 @@ //////////////////////////////////////////////////////////////////////////////// #include #include -#include +#include "utils/easylogging++.h" #include #include diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index 6e3fc4691d..d9c2cbf03e 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -5,7 +5,7 @@ //////////////////////////////////////////////////////////////////////////////// #include #include -#include +#include "utils/easylogging++.h" #include #include "db/Exception.h" diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp index 79925c9235..396377ab4d 100644 --- a/cpp/unittest/db/mysql_db_test.cpp +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -9,7 +9,7 @@ #include "db/meta/MetaConsts.h" #include -#include +#include "utils/easylogging++.h" #include #include diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index 1db24652db..5d6da9c3fb 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -5,7 +5,7 @@ //////////////////////////////////////////////////////////////////////////////// #include #include -#include +#include "utils/easylogging++.h" #include #include diff --git a/cpp/unittest/db/scheduler_test.cpp b/cpp/unittest/db/scheduler_test.cpp index dc23623dc1..fa941cbcb5 100644 --- a/cpp/unittest/db/scheduler_test.cpp +++ b/cpp/unittest/db/scheduler_test.cpp @@ -5,7 +5,7 @@ //////////////////////////////////////////////////////////////////////////////// #include #include -#include +#include "utils/easylogging++.h" #include #include "db/scheduler/TaskScheduler.h" diff --git a/cpp/unittest/knowhere/CMakeLists.txt b/cpp/unittest/knowhere/CMakeLists.txt index ddb8eb2d9e..59fff95509 100644 --- a/cpp/unittest/knowhere/CMakeLists.txt +++ b/cpp/unittest/knowhere/CMakeLists.txt @@ -7,7 +7,9 @@ set(knowhere_src ${MILVUS_ENGINE_SRC}/wrapper/knowhere/vec_index.cpp) set(helper - utils.cpp) + utils.cpp + ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc + ${MILVUS_ENGINE_SRC}/utils/easylogging++.h) set(knowhere_libs knowhere diff --git a/cpp/unittest/knowhere/knowhere_test.cpp b/cpp/unittest/knowhere/knowhere_test.cpp index 8096b2821a..6e5f0fc686 100644 --- a/cpp/unittest/knowhere/knowhere_test.cpp +++ b/cpp/unittest/knowhere/knowhere_test.cpp @@ -5,7 +5,7 @@ //////////////////////////////////////////////////////////////////////////////// #include -#include +#include "utils/easylogging++.h" #include #include "knowhere/index/vector_index/gpu_ivf.h" diff --git a/cpp/unittest/main.cpp b/cpp/unittest/main.cpp index dac5bc7013..086b715aa9 100644 --- a/cpp/unittest/main.cpp +++ b/cpp/unittest/main.cpp @@ -5,7 +5,7 @@ //////////////////////////////////////////////////////////////////////////////// #include #include -#include +#include "utils/easylogging++.h" #include "server/ServerConfig.h" #include "utils/CommonUtil.h" diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 1f0f5bd023..b3ac068757 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -18,7 +18,9 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/src/metrics metrics_src) aux_source_directory(./ test_srcs) set(util_files - ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp) + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp + ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc + ${MILVUS_ENGINE_SRC}/utils/easylogging++.h) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) diff --git a/cpp/unittest/scheduler/CMakeLists.txt b/cpp/unittest/scheduler/CMakeLists.txt index feb7d6178f..f09f8639a1 100644 --- a/cpp/unittest/scheduler/CMakeLists.txt +++ b/cpp/unittest/scheduler/CMakeLists.txt @@ -21,7 +21,10 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs) aux_source_directory(./ test_srcs) -set(util_files ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp) +set(util_files + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp + ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc + ${MILVUS_ENGINE_SRC}/utils/easylogging++.h) set(db_scheduler_srcs ${scheduler_files} diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index 7e67711f3d..48e37357b1 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -39,6 +39,11 @@ set(db_scheduler_srcs ${scheduler_task_files} ) +set(util_files + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp + ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc + ${MILVUS_ENGINE_SRC}/utils/easylogging++.h) + set(db_src ${config_files} ${cache_srcs} diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index 5896cfb6ca..a600ace499 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -5,7 +5,7 @@ //////////////////////////////////////////////////////////////////////////////// #include #include -#include +#include "utils/easylogging++.h" #include #include #include @@ -290,6 +290,7 @@ TEST(UtilTest, ROLLOUTHANDLER_TEST){ std::string dir1 = "/tmp/milvus_test"; std::string dir2 = "/tmp/milvus_test/log_test"; std::string filename[6] = {"log_global.log", "log_debug.log", "log_warning.log", "log_trace.log", "log_error.log", "log_fatal.log"}; + el::Level list[6] = {el::Level::Global, el::Level::Debug, el::Level::Warning, el::Level::Trace, el::Level::Error, el::Level::Fatal}; mkdir(dir1.c_str(), S_IRWXU); mkdir(dir2.c_str(), S_IRWXU); @@ -300,7 +301,7 @@ TEST(UtilTest, ROLLOUTHANDLER_TEST){ file.open(tmp.c_str()); file << "zilliz" << std::endl; - server::RolloutHandler(tmp.c_str(), 0); + server::RolloutHandler(tmp.c_str(), 0, list[i]); tmp.append(".1"); std::ifstream file2; From 8fdef361b13db3c34f6a71bd0d78734279a443a2 Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Mon, 16 Sep 2019 20:18:18 +0800 Subject: [PATCH 003/217] change cmakelist Former-commit-id: 9a60b9fb71226b5b15e7bb2b755ff2ad53e6d922 --- cpp/cmake/ThirdPartyPackages.cmake | 88 ------------------------------ cpp/src/CMakeLists.txt | 2 - cpp/src/utils/LogUtil.cpp | 20 +++---- cpp/unittest/CMakeLists.txt | 1 - 4 files changed, 7 insertions(+), 104 deletions(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 933d470247..fa61da180a 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -19,7 +19,6 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES ARROW BOOST BZip2 -# Easylogging++ FAISS GTest Knowhere @@ -56,8 +55,6 @@ macro(build_dependency DEPENDENCY_NAME) build_arrow() elseif("${DEPENDENCY_NAME}" STREQUAL "BZip2") build_bzip2() -# elseif("${DEPENDENCY_NAME}" STREQUAL "Easylogging++") -# build_easyloggingpp() elseif("${DEPENDENCY_NAME}" STREQUAL "FAISS") build_faiss() elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest") @@ -295,13 +292,6 @@ else() endif() set(BZIP2_MD5 "00b516f4704d4a7cb50a1d97e6e8e15b") -#if(DEFINED ENV{MILVUS_EASYLOGGINGPP_URL}) -# set(EASYLOGGINGPP_SOURCE_URL "$ENV{MILVUS_EASYLOGGINGPP_URL}") -#else() -# set(EASYLOGGINGPP_SOURCE_URL "https://github.com/zuhd-org/easyloggingpp/archive/${EASYLOGGINGPP_VERSION}.tar.gz") -#endif() -#set(EASYLOGGINGPP_MD5 "b78cd319db4be9b639927657b8aa7732") - if(DEFINED ENV{MILVUS_FAISS_URL}) set(FAISS_SOURCE_URL "$ENV{MILVUS_FAISS_URL}") else() @@ -814,84 +804,6 @@ if(MILVUS_WITH_KNOWHERE) include_directories(SYSTEM "${KNOWHERE_INCLUDE_DIR}/SPTAG/AnnService") endif() -# ---------------------------------------------------------------------- -# Easylogging++ - -#macro(build_easyloggingpp) -# message(STATUS "Building Easylogging++-${EASYLOGGINGPP_VERSION} from source") -# set(EASYLOGGINGPP_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/easyloggingpp_ep-prefix/src/easyloggingpp_ep") -# set(EASYLOGGINGPP_INCLUDE_DIR "${EASYLOGGINGPP_PREFIX}/include") -# set(EASYLOGGINGPP_STATIC_LIB -# "${EASYLOGGINGPP_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}easyloggingpp${CMAKE_STATIC_LIBRARY_SUFFIX}") -# -# set(EASYLOGGINGPP_CMAKE_ARGS -# ${EP_COMMON_CMAKE_ARGS} -# "-DCMAKE_INSTALL_PREFIX=${EASYLOGGINGPP_PREFIX}" -# -DCMAKE_INSTALL_LIBDIR=lib -# -Dtest=OFF -# -Dbuild_static_lib=ON) -# -# if(USE_JFROG_CACHE STREQUAL "ON") -# set(EASYLOGGINGPP_CACHE_PACKAGE_NAME "easyloggingpp_${EASYLOGGINGPP_MD5}.tar.gz") -# set(EASYLOGGINGPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${EASYLOGGINGPP_CACHE_PACKAGE_NAME}") -# set(EASYLOGGINGPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${EASYLOGGINGPP_CACHE_PACKAGE_NAME}") -# -# execute_process(COMMAND wget -q --method HEAD ${EASYLOGGINGPP_CACHE_URL} RESULT_VARIABLE return_code) -# message(STATUS "Check the remote cache file ${EASYLOGGINGPP_CACHE_URL}. return code = ${return_code}") -# if (NOT return_code EQUAL 0) -# externalproject_add(easyloggingpp_ep -# URL -# ${EASYLOGGINGPP_SOURCE_URL} -# ${EP_LOG_OPTIONS} -# CMAKE_ARGS -# ${EASYLOGGINGPP_CMAKE_ARGS} -# BUILD_COMMAND -# ${MAKE} -# ${MAKE_BUILD_ARGS} -# BUILD_BYPRODUCTS -# ${EASYLOGGINGPP_STATIC_LIB}) -# -# ExternalProject_Create_Cache(easyloggingpp_ep ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/easyloggingpp_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${EASYLOGGINGPP_CACHE_URL}) -# else() -# file(DOWNLOAD ${EASYLOGGINGPP_CACHE_URL} ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} STATUS status) -# list(GET status 0 status_code) -# message(STATUS "DOWNLOADING FROM ${EASYLOGGINGPP_CACHE_URL} TO ${EASYLOGGINGPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") -# if (status_code EQUAL 0) -# ExternalProject_Use_Cache(easyloggingpp_ep ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) -# endif() -# endif() -# else() -# externalproject_add(easyloggingpp_ep -# URL -# ${EASYLOGGINGPP_SOURCE_URL} -# ${EP_LOG_OPTIONS} -# CMAKE_ARGS -# ${EASYLOGGINGPP_CMAKE_ARGS} -# BUILD_COMMAND -# ${MAKE} -# ${MAKE_BUILD_ARGS} -# BUILD_BYPRODUCTS -# ${EASYLOGGINGPP_STATIC_LIB}) -# endif() -# -# file(MAKE_DIRECTORY "${EASYLOGGINGPP_INCLUDE_DIR}") -# add_library(easyloggingpp STATIC IMPORTED) -# set_target_properties( -# easyloggingpp -# PROPERTIES IMPORTED_LOCATION "${EASYLOGGINGPP_STATIC_LIB}" -# INTERFACE_INCLUDE_DIRECTORIES "${EASYLOGGINGPP_INCLUDE_DIR}") -# -# add_dependencies(easyloggingpp easyloggingpp_ep) -#endmacro() -# -#if(MILVUS_WITH_EASYLOGGINGPP) -# resolve_dependency(Easylogging++) -# -# get_target_property(EASYLOGGINGPP_INCLUDE_DIR easyloggingpp INTERFACE_INCLUDE_DIRECTORIES) -# link_directories(SYSTEM "${EASYLOGGINGPP_PREFIX}/lib") -# include_directories(SYSTEM "${EASYLOGGINGPP_INCLUDE_DIR}") -#endif() - # ---------------------------------------------------------------------- # OpenBLAS diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 733c493b59..3dc838c19e 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -95,7 +95,6 @@ set(client_grpc_lib set(third_party_libs knowhere -# easyloggingpp sqlite ${client_grpc_lib} yaml-cpp @@ -146,7 +145,6 @@ target_link_libraries(milvus_engine ${engine_libs} ${third_party_libs}) add_library(metrics STATIC ${metrics_files}) set(metrics_lib -# easyloggingpp yaml-cpp prometheus-cpp-push prometheus-cpp-pull diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp index 5f0573cce8..35ff862d50 100644 --- a/cpp/src/utils/LogUtil.cpp +++ b/cpp/src/utils/LogUtil.cpp @@ -48,31 +48,25 @@ void RolloutHandler(const char *filename, std::size_t size, el::Level level) { int ret; std::string m(std::string(dir) + "/" + s); s = m; - if(level == el::Level::Global){ + if (level == el::Level::Global) { s.append("." + std::to_string(++global_idx)); ret = rename(m.c_str(), s.c_str()); - } - else if(level == el::Level::Debug){ + } else if (level == el::Level::Debug) { s.append("." + std::to_string(++debug_idx)); ret = rename(m.c_str(), s.c_str()); - } - else if(level == el::Level::Warning){ + } else if (level == el::Level::Warning) { s.append("." + std::to_string(++warning_idx)); ret = rename(m.c_str(), s.c_str()); - } - else if(level == el::Level::Trace){ + } else if (level == el::Level::Trace) { s.append("." + std::to_string(++trace_idx)); ret = rename(m.c_str(), s.c_str()); - } - else if(level == el::Level::Error){ + } else if (level == el::Level::Error) { s.append("." + std::to_string(++error_idx)); ret = rename(m.c_str(), s.c_str()); - } - else if(level == el::Level::Fatal){ + } else if (level == el::Level::Fatal) { s.append("." + std::to_string(++fatal_idx)); ret = rename(m.c_str(), s.c_str()); - } - else{ + } else { s.append("." + std::to_string(++global_idx)); ret = rename(m.c_str(), s.c_str()); } diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 978ca69ff6..69ea8f4ea1 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -25,7 +25,6 @@ set(unittest_libs gmock gtest_main gmock_main -# easyloggingpp pthread metrics gfortran From 49c3b8e9f6a42bc29d386d665ff9f376ff45cdb3 Mon Sep 17 00:00:00 2001 From: starlord Date: Tue, 17 Sep 2019 09:55:25 +0800 Subject: [PATCH 004/217] refine cache code Former-commit-id: 9d74c7f97e3bca9bb43963ed6f05491e258b712c --- cpp/src/cache/Cache.inl | 10 +++++----- cpp/unittest/server/cache_test.cpp | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/src/cache/Cache.inl b/cpp/src/cache/Cache.inl index be01c11c77..5ca1028318 100644 --- a/cpp/src/cache/Cache.inl +++ b/cpp/src/cache/Cache.inl @@ -56,11 +56,11 @@ void Cache::insert(const std::string& key, const ItemObj& item) { return; } - if(item->size() > capacity_) { - SERVER_LOG_ERROR << "Item size " << item->size() - << " is too large to insert into cache, capacity " << capacity_; - return; - } +// if(item->size() > capacity_) { +// SERVER_LOG_ERROR << "Item size " << item->size() +// << " is too large to insert into cache, capacity " << capacity_; +// return; +// } //calculate usage { diff --git a/cpp/unittest/server/cache_test.cpp b/cpp/unittest/server/cache_test.cpp index 8599098b55..8293805209 100644 --- a/cpp/unittest/server/cache_test.cpp +++ b/cpp/unittest/server/cache_test.cpp @@ -162,7 +162,7 @@ TEST(CacheTest, CPU_CACHE_TEST) { engine::VecIndexPtr mock_index = std::make_shared(256, 6000000); cache::DataObjPtr data_obj = std::make_shared(mock_index); cpu_mgr->InsertItem("index_6g", data_obj); - ASSERT_FALSE(cpu_mgr->ItemExists("index_6g"));//data greater than capacity can not be inserted sucessfully + ASSERT_TRUE(cpu_mgr->ItemExists("index_6g")); } cpu_mgr->PrintInfo(); From 650d43dd4ecce34720d4bb86be8a0ad7f62d85d1 Mon Sep 17 00:00:00 2001 From: starlord Date: Tue, 17 Sep 2019 10:16:37 +0800 Subject: [PATCH 005/217] update changelog Former-commit-id: 9aefaa8682dbe383b7fdf75b4edeb6332bf36019 --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index c0b8c1a293..be8ddcb664 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -8,6 +8,7 @@ Please mark all change in change log and use the ticket from JIRA. ## Improvement - MS-552 - Add and change the easylogging library +- MS-553 - Refine cache code ## New Feature From 9433bf81cc4a9e3f46e8f32dc12ecf0843eac414 Mon Sep 17 00:00:00 2001 From: jinhai Date: Tue, 17 Sep 2019 10:51:48 +0800 Subject: [PATCH 006/217] MS-554 Change license to apache 2.0 Former-commit-id: e09b7dd7872372fa0ece9f5d18ec4c316b89cbe8 --- cpp/CHANGELOG.md | 1 + cpp/CMakeLists.txt | 20 ++++++++++++++--- cpp/src/CMakeLists.txt | 20 ++++++++++++++--- cpp/src/cache/Cache.h | 22 ++++++++++++++----- cpp/src/cache/Cache.inl | 22 ++++++++++++++----- cpp/src/cache/CacheMgr.h | 22 ++++++++++++++----- cpp/src/cache/CacheMgr.inl | 22 ++++++++++++++----- cpp/src/cache/CpuCacheMgr.cpp | 22 ++++++++++++++----- cpp/src/cache/CpuCacheMgr.h | 22 ++++++++++++++----- cpp/src/cache/DataObj.h | 22 ++++++++++++++----- cpp/src/cache/GpuCacheMgr.cpp | 22 ++++++++++++++----- cpp/src/cache/GpuCacheMgr.h | 22 ++++++++++++++----- cpp/src/cache/LRU.h | 22 ++++++++++++++----- cpp/src/config/ConfigMgr.cpp | 22 ++++++++++++++----- cpp/src/config/ConfigMgr.h | 22 ++++++++++++++----- cpp/src/config/ConfigNode.cpp | 22 ++++++++++++++----- cpp/src/config/ConfigNode.h | 22 ++++++++++++++----- cpp/src/config/YamlConfigMgr.cpp | 22 ++++++++++++++----- cpp/src/config/YamlConfigMgr.h | 22 ++++++++++++++----- cpp/src/core/CMakeLists.txt | 20 ++++++++++++++--- .../include/knowhere/adapter/faiss_adopt.h | 22 ++++++++++++++----- .../core/include/knowhere/adapter/structure.h | 22 ++++++++++++++----- .../core/include/knowhere/common/exception.h | 22 ++++++++++++++----- cpp/src/core/include/knowhere/common/timer.h | 22 ++++++++++++++----- .../knowhere/index/vector_index/cloner.h | 22 ++++++++++++++----- .../knowhere/index/vector_index/idmap.h | 22 ++++++++++++++----- .../include/knowhere/index/vector_index/ivf.h | 22 ++++++++++++++----- .../index/vector_index/nsg/neighbor.h | 22 ++++++++++++++----- .../knowhere/index/vector_index/nsg/nsg.h | 22 ++++++++++++++----- .../knowhere/index/vector_index/nsg/nsg_io.h | 22 ++++++++++++++----- .../knowhere/index/vector_index/nsg_index.h | 22 ++++++++++++++----- .../core/src/knowhere/adapter/structure.cpp | 22 ++++++++++++++----- .../core/src/knowhere/common/exception.cpp | 22 ++++++++++++++----- cpp/src/core/src/knowhere/common/timer.cpp | 22 ++++++++++++++----- .../knowhere/index/vector_index/cloner.cpp | 22 ++++++++++++++----- .../knowhere/index/vector_index/gpu_ivf.cpp | 22 ++++++++++++++----- .../src/knowhere/index/vector_index/idmap.cpp | 22 ++++++++++++++----- .../src/knowhere/index/vector_index/ivf.cpp | 22 ++++++++++++++----- .../index/vector_index/nsg/index_io.h | 22 ++++++++++++++----- .../knowhere/index/vector_index/nsg/nsg.cpp | 22 ++++++++++++++----- .../index/vector_index/nsg/nsg_io.cpp | 22 ++++++++++++++----- .../knowhere/index/vector_index/nsg/utils.cpp | 22 ++++++++++++++----- .../knowhere/index/vector_index/nsg/utils.h | 22 ++++++++++++++----- .../knowhere/index/vector_index/nsg_index.cpp | 22 ++++++++++++++----- .../core/test/faiss_ori/gpuresource_test.cpp | 22 ++++++++++++++----- cpp/src/core/test/test_idmap.cpp | 22 ++++++++++++++----- cpp/src/core/test/test_ivf.cpp | 22 ++++++++++++++----- cpp/src/core/test/test_json.cpp | 22 ++++++++++++++----- cpp/src/core/test/test_kdt.cpp | 22 ++++++++++++++----- cpp/src/core/test/test_nsg.cpp | 22 ++++++++++++++----- cpp/src/core/test/test_nsg/test_nsg.cpp | 22 ++++++++++++++----- cpp/src/core/test/utils.cpp | 22 ++++++++++++++----- cpp/src/core/test/utils.h | 22 ++++++++++++++----- cpp/src/db/Constants.h | 22 ++++++++++++++----- cpp/src/db/DB.h | 22 ++++++++++++++----- cpp/src/db/DBFactory.cpp | 22 ++++++++++++++----- cpp/src/db/DBFactory.h | 22 ++++++++++++++----- cpp/src/db/DBImpl.cpp | 22 ++++++++++++++----- cpp/src/db/DBImpl.h | 22 ++++++++++++++----- cpp/src/db/Exception.h | 22 ++++++++++++++----- cpp/src/db/IDGenerator.cpp | 22 ++++++++++++++----- cpp/src/db/IDGenerator.h | 22 ++++++++++++++----- cpp/src/db/Log.h | 22 ++++++++++++++----- cpp/src/db/Options.cpp | 22 ++++++++++++++----- cpp/src/db/Options.h | 22 ++++++++++++++----- cpp/src/db/Status.cpp | 22 ++++++++++++++----- cpp/src/db/Status.h | 22 ++++++++++++++----- cpp/src/db/Types.h | 22 ++++++++++++++----- cpp/src/db/Utils.cpp | 22 ++++++++++++++----- cpp/src/db/Utils.h | 22 ++++++++++++++----- cpp/src/db/engine/EngineFactory.cpp | 22 ++++++++++++++----- cpp/src/db/engine/EngineFactory.h | 22 ++++++++++++++----- cpp/src/db/engine/ExecutionEngine.h | 22 ++++++++++++++----- cpp/src/db/engine/ExecutionEngineImpl.cpp | 22 ++++++++++++++----- cpp/src/db/engine/ExecutionEngineImpl.h | 22 ++++++++++++++----- cpp/src/db/insert/MemManager.h | 18 +++++++++++++++ cpp/src/db/insert/MemManagerImpl.cpp | 18 +++++++++++++++ cpp/src/db/insert/MemManagerImpl.h | 18 +++++++++++++++ cpp/src/db/insert/MemMenagerFactory.cpp | 22 ++++++++++++++----- cpp/src/db/insert/MemMenagerFactory.h | 22 ++++++++++++++----- cpp/src/db/insert/MemTable.cpp | 18 +++++++++++++++ cpp/src/db/insert/MemTable.h | 18 +++++++++++++++ cpp/src/db/insert/MemTableFile.cpp | 18 +++++++++++++++ cpp/src/db/insert/MemTableFile.h | 18 +++++++++++++++ cpp/src/db/insert/VectorSource.cpp | 18 +++++++++++++++ cpp/src/db/insert/VectorSource.h | 18 +++++++++++++++ cpp/src/db/meta/Meta.h | 22 ++++++++++++++----- cpp/src/db/meta/MetaConsts.h | 22 ++++++++++++++----- cpp/src/db/meta/MetaFactory.cpp | 22 ++++++++++++++----- cpp/src/db/meta/MetaFactory.h | 22 ++++++++++++++----- cpp/src/db/meta/MetaTypes.h | 22 ++++++++++++++----- cpp/src/db/meta/MySQLConnectionPool.cpp | 18 +++++++++++++++ cpp/src/db/meta/MySQLConnectionPool.h | 18 +++++++++++++++ cpp/src/db/meta/MySQLMetaImpl.cpp | 22 ++++++++++++++----- cpp/src/db/meta/MySQLMetaImpl.h | 22 ++++++++++++++----- cpp/src/db/meta/SqliteMetaImpl.cpp | 22 ++++++++++++++----- cpp/src/db/meta/SqliteMetaImpl.h | 22 ++++++++++++++----- cpp/src/db/scheduler/TaskDispatchQueue.cpp | 22 ++++++++++++++----- cpp/src/db/scheduler/TaskDispatchQueue.h | 22 ++++++++++++++----- cpp/src/db/scheduler/TaskDispatchStrategy.cpp | 22 ++++++++++++++----- cpp/src/db/scheduler/TaskDispatchStrategy.h | 22 ++++++++++++++----- cpp/src/db/scheduler/TaskScheduler.cpp | 22 ++++++++++++++----- cpp/src/db/scheduler/TaskScheduler.h | 22 ++++++++++++++----- .../db/scheduler/context/DeleteContext.cpp | 22 ++++++++++++++----- cpp/src/db/scheduler/context/DeleteContext.h | 22 ++++++++++++++----- .../db/scheduler/context/IScheduleContext.h | 22 ++++++++++++++----- .../db/scheduler/context/SearchContext.cpp | 22 ++++++++++++++----- cpp/src/db/scheduler/context/SearchContext.h | 22 ++++++++++++++----- cpp/src/db/scheduler/task/DeleteTask.cpp | 22 ++++++++++++++----- cpp/src/db/scheduler/task/DeleteTask.h | 22 ++++++++++++++----- cpp/src/db/scheduler/task/IScheduleTask.h | 22 ++++++++++++++----- cpp/src/db/scheduler/task/IndexLoadTask.cpp | 22 ++++++++++++++----- cpp/src/db/scheduler/task/IndexLoadTask.h | 22 ++++++++++++++----- cpp/src/db/scheduler/task/SearchTask.cpp | 22 ++++++++++++++----- cpp/src/db/scheduler/task/SearchTask.h | 22 ++++++++++++++----- cpp/src/main.cpp | 22 ++++++++++++++----- cpp/src/metrics/MetricBase.h | 22 ++++++++++++++----- cpp/src/metrics/Metrics.cpp | 22 ++++++++++++++----- cpp/src/metrics/Metrics.h | 22 ++++++++++++++----- cpp/src/metrics/PrometheusMetrics.cpp | 22 ++++++++++++++----- cpp/src/metrics/PrometheusMetrics.h | 22 ++++++++++++++----- cpp/src/metrics/SystemInfo.cpp | 22 ++++++++++++++----- cpp/src/metrics/SystemInfo.h | 22 ++++++++++++++----- cpp/src/scheduler/Algorithm.cpp | 22 ++++++++++++++----- cpp/src/scheduler/Algorithm.h | 22 ++++++++++++++----- cpp/src/scheduler/ResourceFactory.cpp | 22 ++++++++++++++----- cpp/src/scheduler/ResourceFactory.h | 22 ++++++++++++++----- cpp/src/scheduler/ResourceMgr.cpp | 22 ++++++++++++++----- cpp/src/scheduler/ResourceMgr.h | 22 ++++++++++++++----- cpp/src/scheduler/SchedInst.cpp | 22 ++++++++++++++----- cpp/src/scheduler/SchedInst.h | 22 ++++++++++++++----- cpp/src/scheduler/Scheduler.cpp | 22 ++++++++++++++----- cpp/src/scheduler/Scheduler.h | 22 ++++++++++++++----- cpp/src/scheduler/TaskTable.cpp | 22 ++++++++++++++----- cpp/src/scheduler/TaskTable.h | 22 ++++++++++++++----- cpp/src/scheduler/Utils.cpp | 22 ++++++++++++++----- cpp/src/scheduler/Utils.h | 22 ++++++++++++++----- cpp/src/scheduler/action/Action.h | 22 ++++++++++++++----- .../scheduler/action/PushTaskToNeighbour.cpp | 22 ++++++++++++++----- cpp/src/scheduler/event/Event.h | 22 ++++++++++++++----- cpp/src/scheduler/event/EventDump.cpp | 22 ++++++++++++++----- cpp/src/scheduler/event/FinishTaskEvent.h | 22 ++++++++++++++----- cpp/src/scheduler/event/LoadCompletedEvent.h | 22 ++++++++++++++----- cpp/src/scheduler/event/StartUpEvent.h | 22 ++++++++++++++----- .../scheduler/event/TaskTableUpdatedEvent.h | 22 ++++++++++++++----- cpp/src/scheduler/resource/Connection.h | 22 ++++++++++++++----- cpp/src/scheduler/resource/CpuResource.cpp | 22 ++++++++++++++----- cpp/src/scheduler/resource/CpuResource.h | 22 ++++++++++++++----- cpp/src/scheduler/resource/DiskResource.cpp | 22 ++++++++++++++----- cpp/src/scheduler/resource/DiskResource.h | 22 ++++++++++++++----- cpp/src/scheduler/resource/GpuResource.cpp | 22 ++++++++++++++----- cpp/src/scheduler/resource/GpuResource.h | 22 ++++++++++++++----- cpp/src/scheduler/resource/Node.cpp | 22 ++++++++++++++----- cpp/src/scheduler/resource/Node.h | 22 ++++++++++++++----- cpp/src/scheduler/resource/Resource.cpp | 22 ++++++++++++++----- cpp/src/scheduler/resource/Resource.h | 22 ++++++++++++++----- cpp/src/scheduler/resource/TestResource.cpp | 22 ++++++++++++++----- cpp/src/scheduler/resource/TestResource.h | 22 ++++++++++++++----- cpp/src/scheduler/task/DeleteTask.cpp | 22 ++++++++++++++----- cpp/src/scheduler/task/DeleteTask.h | 22 ++++++++++++++----- cpp/src/scheduler/task/Path.h | 22 ++++++++++++++----- cpp/src/scheduler/task/SearchTask.cpp | 22 ++++++++++++++----- cpp/src/scheduler/task/SearchTask.h | 22 ++++++++++++++----- cpp/src/scheduler/task/Task.h | 22 ++++++++++++++----- cpp/src/scheduler/task/TaskConvert.cpp | 22 ++++++++++++++----- cpp/src/scheduler/task/TaskConvert.h | 22 ++++++++++++++----- cpp/src/scheduler/task/TestTask.cpp | 22 ++++++++++++++----- cpp/src/scheduler/task/TestTask.h | 22 ++++++++++++++----- cpp/src/scheduler/tasklabel/BroadcastLabel.h | 22 ++++++++++++++----- cpp/src/scheduler/tasklabel/DefaultLabel.h | 22 ++++++++++++++----- cpp/src/scheduler/tasklabel/SpecResLabel.h | 22 ++++++++++++++----- cpp/src/scheduler/tasklabel/TaskLabel.h | 22 ++++++++++++++----- cpp/src/sdk/CMakeLists.txt | 20 ++++++++++++++--- cpp/src/sdk/examples/CMakeLists.txt | 20 ++++++++++++++--- .../sdk/examples/grpcsimple/CMakeLists.txt | 20 ++++++++++++++--- cpp/src/sdk/examples/grpcsimple/main.cpp | 22 ++++++++++++++----- .../examples/grpcsimple/src/ClientTest.cpp | 22 ++++++++++++++----- .../sdk/examples/grpcsimple/src/ClientTest.h | 22 ++++++++++++++----- cpp/src/sdk/grpc/ClientProxy.cpp | 22 ++++++++++++++----- cpp/src/sdk/grpc/ClientProxy.h | 22 ++++++++++++++----- cpp/src/sdk/grpc/GrpcClient.cpp | 22 ++++++++++++++----- cpp/src/sdk/grpc/GrpcClient.h | 22 ++++++++++++++----- cpp/src/sdk/include/MilvusApi.h | 18 +++++++++++++++ cpp/src/sdk/include/Status.h | 18 +++++++++++++++ cpp/src/sdk/interface/ConnectionImpl.cpp | 22 ++++++++++++++----- cpp/src/sdk/interface/ConnectionImpl.h | 22 ++++++++++++++----- cpp/src/sdk/interface/Status.cpp | 22 ++++++++++++++----- cpp/src/sdk/util/Exception.h | 22 ++++++++++++++----- cpp/src/server/DBWrapper.cpp | 22 ++++++++++++++----- cpp/src/server/DBWrapper.h | 22 ++++++++++++++----- cpp/src/server/Server.cpp | 22 ++++++++++++++----- cpp/src/server/Server.h | 22 ++++++++++++++----- cpp/src/server/ServerConfig.cpp | 22 ++++++++++++++----- cpp/src/server/ServerConfig.h | 22 ++++++++++++++----- cpp/src/server/grpc_impl/GrpcMilvusServer.cpp | 22 ++++++++++++++----- cpp/src/server/grpc_impl/GrpcMilvusServer.h | 22 ++++++++++++++----- .../server/grpc_impl/GrpcRequestHandler.cpp | 22 ++++++++++++++----- cpp/src/server/grpc_impl/GrpcRequestHandler.h | 22 ++++++++++++++----- .../server/grpc_impl/GrpcRequestScheduler.cpp | 22 ++++++++++++++----- .../server/grpc_impl/GrpcRequestScheduler.h | 22 ++++++++++++++----- cpp/src/server/grpc_impl/GrpcRequestTask.cpp | 22 ++++++++++++++----- cpp/src/server/grpc_impl/GrpcRequestTask.h | 22 ++++++++++++++----- cpp/src/storage/IStorage.h | 18 +++++++++++++++ cpp/src/storage/s3/S3ClientWrapper.cpp | 18 +++++++++++++++ cpp/src/storage/s3/S3ClientWrapper.h | 18 +++++++++++++++ cpp/src/utils/BlockingQueue.h | 22 ++++++++++++++----- cpp/src/utils/BlockingQueue.inl | 18 +++++++++++++++ cpp/src/utils/CommonUtil.cpp | 22 ++++++++++++++----- cpp/src/utils/CommonUtil.h | 22 ++++++++++++++----- cpp/src/utils/Error.h | 22 ++++++++++++++----- cpp/src/utils/Log.h | 22 ++++++++++++++----- cpp/src/utils/LogUtil.cpp | 22 ++++++++++++++----- cpp/src/utils/LogUtil.h | 22 ++++++++++++++----- cpp/src/utils/SignalUtil.cpp | 22 ++++++++++++++----- cpp/src/utils/SignalUtil.h | 22 ++++++++++++++----- cpp/src/utils/StringHelpFunctions.cpp | 22 ++++++++++++++----- cpp/src/utils/StringHelpFunctions.h | 22 ++++++++++++++----- cpp/src/utils/ThreadPool.h | 22 ++++++++++++++----- cpp/src/utils/TimeRecorder.cpp | 22 ++++++++++++++----- cpp/src/utils/TimeRecorder.h | 22 ++++++++++++++----- cpp/src/utils/ValidationUtil.cpp | 18 +++++++++++++++ cpp/src/utils/ValidationUtil.h | 18 +++++++++++++++ cpp/src/wrapper/gpu/Arithmetic.h | 22 ++++++++++++++----- cpp/src/wrapper/gpu/Topk.cu | 22 ++++++++++++++----- cpp/src/wrapper/gpu/Topk.h | 22 ++++++++++++++----- cpp/src/wrapper/knowhere/KnowhereResource.cpp | 22 ++++++++++++++----- cpp/src/wrapper/knowhere/KnowhereResource.h | 22 ++++++++++++++----- cpp/src/wrapper/knowhere/data_transfer.cpp | 22 ++++++++++++++----- cpp/src/wrapper/knowhere/data_transfer.h | 22 ++++++++++++++----- cpp/src/wrapper/knowhere/vec_impl.cpp | 22 ++++++++++++++----- cpp/src/wrapper/knowhere/vec_impl.h | 22 ++++++++++++++----- cpp/src/wrapper/knowhere/vec_index.cpp | 22 ++++++++++++++----- cpp/src/wrapper/knowhere/vec_index.h | 22 ++++++++++++++----- cpp/src/wrapper/knowhere/wrapper_log.h | 22 ++++++++++++++----- cpp/unittest/CMakeLists.txt | 20 ++++++++++++++--- cpp/unittest/db/CMakeLists.txt | 20 ++++++++++++++--- cpp/unittest/db/db_tests.cpp | 22 ++++++++++++++----- cpp/unittest/db/engine_test.cpp | 22 ++++++++++++++----- cpp/unittest/db/mem_test.cpp | 18 +++++++++++++++ cpp/unittest/db/meta_tests.cpp | 22 ++++++++++++++----- cpp/unittest/db/misc_test.cpp | 22 ++++++++++++++----- cpp/unittest/db/mysql_db_test.cpp | 22 ++++++++++++++----- cpp/unittest/db/mysql_meta_test.cpp | 22 ++++++++++++++----- cpp/unittest/db/scheduler_test.cpp | 22 ++++++++++++++----- cpp/unittest/db/search_test.cpp | 22 ++++++++++++++----- cpp/unittest/db/utils.cpp | 22 ++++++++++++++----- cpp/unittest/db/utils.h | 22 ++++++++++++++----- cpp/unittest/knowhere/CMakeLists.txt | 19 ++++++++++++++++ cpp/unittest/knowhere/knowhere_test.cpp | 22 ++++++++++++++----- cpp/unittest/knowhere/utils.cpp | 22 ++++++++++++++----- cpp/unittest/knowhere/utils.h | 22 ++++++++++++++----- cpp/unittest/main.cpp | 22 ++++++++++++++----- cpp/unittest/metrics/CMakeLists.txt | 19 ++++++++++++++++ cpp/unittest/metrics/metricbase_test.cpp | 22 ++++++++++++++----- cpp/unittest/metrics/metrics_test.cpp | 22 ++++++++++++++----- cpp/unittest/metrics/prometheus_test.cpp | 22 ++++++++++++++----- cpp/unittest/metrics/utils.cpp | 22 ++++++++++++++----- cpp/unittest/metrics/utils.h | 22 ++++++++++++++----- cpp/unittest/scheduler/CMakeLists.txt | 20 ++++++++++++++--- cpp/unittest/scheduler/algorithm_test.cpp | 22 ++++++++++++++----- cpp/unittest/scheduler/event_test.cpp | 22 ++++++++++++++----- cpp/unittest/scheduler/node_test.cpp | 18 +++++++++++++++ cpp/unittest/scheduler/normal_test.cpp | 18 +++++++++++++++ .../scheduler/resource_factory_test.cpp | 18 +++++++++++++++ cpp/unittest/scheduler/resource_mgr_test.cpp | 22 ++++++++++++++----- cpp/unittest/scheduler/resource_test.cpp | 22 ++++++++++++++----- cpp/unittest/scheduler/schedinst_test.cpp | 22 ++++++++++++++----- cpp/unittest/scheduler/scheduler_test.cpp | 22 ++++++++++++++----- cpp/unittest/scheduler/task_test.cpp | 22 ++++++++++++++----- cpp/unittest/scheduler/tasktable_test.cpp | 18 +++++++++++++++ cpp/unittest/server/CMakeLists.txt | 20 ++++++++++++++--- cpp/unittest/server/cache_test.cpp | 22 ++++++++++++++----- cpp/unittest/server/config_test.cpp | 22 ++++++++++++++----- cpp/unittest/server/rpc_test.cpp | 22 ++++++++++++++----- cpp/unittest/server/util_test.cpp | 22 ++++++++++++++----- cpp/unittest/storage/CMakeLists.txt | 20 ++++++++++++++--- cpp/unittest/storage/S3ClientTest.cpp | 22 ++++++++++++++----- 277 files changed, 4721 insertions(+), 1228 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index be8ddcb664..a14997bb75 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -13,6 +13,7 @@ Please mark all change in change log and use the ticket from JIRA. ## New Feature ## Task +- MS-554 - Change license to Apache 2.0 # Milvus 0.4.0 (2019-09-12) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 1754bfeff3..fa65e15bf9 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1,9 +1,23 @@ #------------------------------------------------------------------------------- -# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -# Unauthorized copying of this file, via any medium is strictly prohibited. -# Proprietary and confidential. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. #------------------------------------------------------------------------------- + cmake_minimum_required(VERSION 3.14) message(STATUS "Building using CMake version: ${CMAKE_VERSION}") diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 3dc838c19e..5c6485b0ee 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -1,9 +1,23 @@ #------------------------------------------------------------------------------- -# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -# Unauthorized copying of this file, via any medium is strictly prohibited. -# Proprietary and confidential. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. #------------------------------------------------------------------------------- + add_subdirectory(core) set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) diff --git a/cpp/src/cache/Cache.h b/cpp/src/cache/Cache.h index d0df4bdcab..b2a57ebdfb 100644 --- a/cpp/src/cache/Cache.h +++ b/cpp/src/cache/Cache.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/cache/Cache.inl b/cpp/src/cache/Cache.inl index 5ca1028318..3eaffc4556 100644 --- a/cpp/src/cache/Cache.inl +++ b/cpp/src/cache/Cache.inl @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + namespace zilliz { diff --git a/cpp/src/cache/CacheMgr.h b/cpp/src/cache/CacheMgr.h index 8c5647a83a..1dd630e14d 100644 --- a/cpp/src/cache/CacheMgr.h +++ b/cpp/src/cache/CacheMgr.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/cache/CacheMgr.inl b/cpp/src/cache/CacheMgr.inl index 05bb1eb08a..36c3d94642 100644 --- a/cpp/src/cache/CacheMgr.inl +++ b/cpp/src/cache/CacheMgr.inl @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + namespace zilliz { namespace milvus { diff --git a/cpp/src/cache/CpuCacheMgr.cpp b/cpp/src/cache/CpuCacheMgr.cpp index 69962919fa..e47d768f34 100644 --- a/cpp/src/cache/CpuCacheMgr.cpp +++ b/cpp/src/cache/CpuCacheMgr.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "CpuCacheMgr.h" #include "server/ServerConfig.h" diff --git a/cpp/src/cache/CpuCacheMgr.h b/cpp/src/cache/CpuCacheMgr.h index 2aee7501d0..5a2cdb599a 100644 --- a/cpp/src/cache/CpuCacheMgr.h +++ b/cpp/src/cache/CpuCacheMgr.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "CacheMgr.h" diff --git a/cpp/src/cache/DataObj.h b/cpp/src/cache/DataObj.h index d9c14f4d1b..8ba59104cd 100644 --- a/cpp/src/cache/DataObj.h +++ b/cpp/src/cache/DataObj.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/cache/GpuCacheMgr.cpp b/cpp/src/cache/GpuCacheMgr.cpp index 6567f997ad..bd6f2adfe1 100644 --- a/cpp/src/cache/GpuCacheMgr.cpp +++ b/cpp/src/cache/GpuCacheMgr.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include "utils/Log.h" diff --git a/cpp/src/cache/GpuCacheMgr.h b/cpp/src/cache/GpuCacheMgr.h index ee0ac68660..57539b6981 100644 --- a/cpp/src/cache/GpuCacheMgr.h +++ b/cpp/src/cache/GpuCacheMgr.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "CacheMgr.h" #include "DataObj.h" diff --git a/cpp/src/cache/LRU.h b/cpp/src/cache/LRU.h index 5d8e8881c9..f117425d2b 100644 --- a/cpp/src/cache/LRU.h +++ b/cpp/src/cache/LRU.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/config/ConfigMgr.cpp b/cpp/src/config/ConfigMgr.cpp index 06819bd6dd..f39679fcdd 100644 --- a/cpp/src/config/ConfigMgr.cpp +++ b/cpp/src/config/ConfigMgr.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "ConfigMgr.h" #include "YamlConfigMgr.h" diff --git a/cpp/src/config/ConfigMgr.h b/cpp/src/config/ConfigMgr.h index e14ea43b36..3e546103b6 100644 --- a/cpp/src/config/ConfigMgr.h +++ b/cpp/src/config/ConfigMgr.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "utils/Error.h" diff --git a/cpp/src/config/ConfigNode.cpp b/cpp/src/config/ConfigNode.cpp index a6b91bf67d..88954c1bdf 100644 --- a/cpp/src/config/ConfigNode.cpp +++ b/cpp/src/config/ConfigNode.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "ConfigNode.h" #include "utils/Error.h" #include "utils/Log.h" diff --git a/cpp/src/config/ConfigNode.h b/cpp/src/config/ConfigNode.h index 6cb3f8f808..606e090d4b 100644 --- a/cpp/src/config/ConfigNode.h +++ b/cpp/src/config/ConfigNode.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/config/YamlConfigMgr.cpp b/cpp/src/config/YamlConfigMgr.cpp index bb395a5003..bddbc32245 100644 --- a/cpp/src/config/YamlConfigMgr.cpp +++ b/cpp/src/config/YamlConfigMgr.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "YamlConfigMgr.h" #include "utils/Log.h" diff --git a/cpp/src/config/YamlConfigMgr.h b/cpp/src/config/YamlConfigMgr.h index 412694dab1..ef4d02c3ce 100644 --- a/cpp/src/config/YamlConfigMgr.h +++ b/cpp/src/config/YamlConfigMgr.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "ConfigMgr.h" diff --git a/cpp/src/core/CMakeLists.txt b/cpp/src/core/CMakeLists.txt index d406f41ce3..b18af23f1e 100644 --- a/cpp/src/core/CMakeLists.txt +++ b/cpp/src/core/CMakeLists.txt @@ -1,9 +1,23 @@ #------------------------------------------------------------------------------- -# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -# Unauthorized copying of this file, via any medium is strictly prohibited. -# Proprietary and confidential. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. #------------------------------------------------------------------------------- + cmake_minimum_required(VERSION 3.14) message(STATUS "---------------core--------------") message(STATUS "Building using CMake version: ${CMAKE_VERSION}") diff --git a/cpp/src/core/include/knowhere/adapter/faiss_adopt.h b/cpp/src/core/include/knowhere/adapter/faiss_adopt.h index 26a005122e..bdace40a74 100644 --- a/cpp/src/core/include/knowhere/adapter/faiss_adopt.h +++ b/cpp/src/core/include/knowhere/adapter/faiss_adopt.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/core/include/knowhere/adapter/structure.h b/cpp/src/core/include/knowhere/adapter/structure.h index e0478ed38e..9ffac4fdac 100644 --- a/cpp/src/core/include/knowhere/adapter/structure.h +++ b/cpp/src/core/include/knowhere/adapter/structure.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/core/include/knowhere/common/exception.h b/cpp/src/core/include/knowhere/common/exception.h index 786c322584..d357f0501e 100644 --- a/cpp/src/core/include/knowhere/common/exception.h +++ b/cpp/src/core/include/knowhere/common/exception.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/core/include/knowhere/common/timer.h b/cpp/src/core/include/knowhere/common/timer.h index 680d984d19..8ecd60df99 100644 --- a/cpp/src/core/include/knowhere/common/timer.h +++ b/cpp/src/core/include/knowhere/common/timer.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/core/include/knowhere/index/vector_index/cloner.h b/cpp/src/core/include/knowhere/index/vector_index/cloner.h index 008954c34a..f546ecc7d9 100644 --- a/cpp/src/core/include/knowhere/index/vector_index/cloner.h +++ b/cpp/src/core/include/knowhere/index/vector_index/cloner.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/core/include/knowhere/index/vector_index/idmap.h b/cpp/src/core/include/knowhere/index/vector_index/idmap.h index 9582249057..b54a589423 100644 --- a/cpp/src/core/include/knowhere/index/vector_index/idmap.h +++ b/cpp/src/core/include/knowhere/index/vector_index/idmap.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/core/include/knowhere/index/vector_index/ivf.h b/cpp/src/core/include/knowhere/index/vector_index/ivf.h index 36f292d9e5..e8ac6cae99 100644 --- a/cpp/src/core/include/knowhere/index/vector_index/ivf.h +++ b/cpp/src/core/include/knowhere/index/vector_index/ivf.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/core/include/knowhere/index/vector_index/nsg/neighbor.h b/cpp/src/core/include/knowhere/index/vector_index/nsg/neighbor.h index 40281d2d6d..9aceb62692 100644 --- a/cpp/src/core/include/knowhere/index/vector_index/nsg/neighbor.h +++ b/cpp/src/core/include/knowhere/index/vector_index/nsg/neighbor.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/core/include/knowhere/index/vector_index/nsg/nsg.h b/cpp/src/core/include/knowhere/index/vector_index/nsg/nsg.h index 5783a8796e..247d78b97b 100644 --- a/cpp/src/core/include/knowhere/index/vector_index/nsg/nsg.h +++ b/cpp/src/core/include/knowhere/index/vector_index/nsg/nsg.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/core/include/knowhere/index/vector_index/nsg/nsg_io.h b/cpp/src/core/include/knowhere/index/vector_index/nsg/nsg_io.h index 5f20e715c7..e84d2a9851 100644 --- a/cpp/src/core/include/knowhere/index/vector_index/nsg/nsg_io.h +++ b/cpp/src/core/include/knowhere/index/vector_index/nsg/nsg_io.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/core/include/knowhere/index/vector_index/nsg_index.h b/cpp/src/core/include/knowhere/index/vector_index/nsg_index.h index 21896fac49..a0829f381c 100644 --- a/cpp/src/core/include/knowhere/index/vector_index/nsg_index.h +++ b/cpp/src/core/include/knowhere/index/vector_index/nsg_index.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/core/src/knowhere/adapter/structure.cpp b/cpp/src/core/src/knowhere/adapter/structure.cpp index c78b1f125d..f30479dd5a 100644 --- a/cpp/src/core/src/knowhere/adapter/structure.cpp +++ b/cpp/src/core/src/knowhere/adapter/structure.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "knowhere/adapter/structure.h" diff --git a/cpp/src/core/src/knowhere/common/exception.cpp b/cpp/src/core/src/knowhere/common/exception.cpp index 1697f14910..f46cbfc35a 100644 --- a/cpp/src/core/src/knowhere/common/exception.cpp +++ b/cpp/src/core/src/knowhere/common/exception.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "knowhere/common/exception.h" #include diff --git a/cpp/src/core/src/knowhere/common/timer.cpp b/cpp/src/core/src/knowhere/common/timer.cpp index 01e62b2425..08654413ff 100644 --- a/cpp/src/core/src/knowhere/common/timer.cpp +++ b/cpp/src/core/src/knowhere/common/timer.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include // TODO(linxj): using Log instead diff --git a/cpp/src/core/src/knowhere/index/vector_index/cloner.cpp b/cpp/src/core/src/knowhere/index/vector_index/cloner.cpp index 4229bf0a88..f5bcfb896d 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/cloner.cpp +++ b/cpp/src/core/src/knowhere/index/vector_index/cloner.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "knowhere/common/exception.h" #include "knowhere/index/vector_index/cloner.h" diff --git a/cpp/src/core/src/knowhere/index/vector_index/gpu_ivf.cpp b/cpp/src/core/src/knowhere/index/vector_index/gpu_ivf.cpp index fcea597251..590ded16bf 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/gpu_ivf.cpp +++ b/cpp/src/core/src/knowhere/index/vector_index/gpu_ivf.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include diff --git a/cpp/src/core/src/knowhere/index/vector_index/idmap.cpp b/cpp/src/core/src/knowhere/index/vector_index/idmap.cpp index e6157b5a1e..142d06b5d8 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/idmap.cpp +++ b/cpp/src/core/src/knowhere/index/vector_index/idmap.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include diff --git a/cpp/src/core/src/knowhere/index/vector_index/ivf.cpp b/cpp/src/core/src/knowhere/index/vector_index/ivf.cpp index 7d3e9e54fa..acfbb17651 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/ivf.cpp +++ b/cpp/src/core/src/knowhere/index/vector_index/ivf.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include #include diff --git a/cpp/src/core/src/knowhere/index/vector_index/nsg/index_io.h b/cpp/src/core/src/knowhere/index/vector_index/nsg/index_io.h index 0e32622f15..fd8fc9e7ad 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/nsg/index_io.h +++ b/cpp/src/core/src/knowhere/index/vector_index/nsg/index_io.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/core/src/knowhere/index/vector_index/nsg/nsg.cpp b/cpp/src/core/src/knowhere/index/vector_index/nsg/nsg.cpp index 5e8acbf344..d3efc5e38a 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/nsg/nsg.cpp +++ b/cpp/src/core/src/knowhere/index/vector_index/nsg/nsg.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include #include diff --git a/cpp/src/core/src/knowhere/index/vector_index/nsg/nsg_io.cpp b/cpp/src/core/src/knowhere/index/vector_index/nsg/nsg_io.cpp index e9854236bc..283e8cc7e2 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/nsg/nsg_io.cpp +++ b/cpp/src/core/src/knowhere/index/vector_index/nsg/nsg_io.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include diff --git a/cpp/src/core/src/knowhere/index/vector_index/nsg/utils.cpp b/cpp/src/core/src/knowhere/index/vector_index/nsg/utils.cpp index 4be8aece02..96a78bb63f 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/nsg/utils.cpp +++ b/cpp/src/core/src/knowhere/index/vector_index/nsg/utils.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include diff --git a/cpp/src/core/src/knowhere/index/vector_index/nsg/utils.h b/cpp/src/core/src/knowhere/index/vector_index/nsg/utils.h index 4d9dbbbda3..82c8a00dc8 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/nsg/utils.h +++ b/cpp/src/core/src/knowhere/index/vector_index/nsg/utils.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/core/src/knowhere/index/vector_index/nsg_index.cpp b/cpp/src/core/src/knowhere/index/vector_index/nsg_index.cpp index aa56b81382..550efef70d 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/nsg_index.cpp +++ b/cpp/src/core/src/knowhere/index/vector_index/nsg_index.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "knowhere/index/vector_index/nsg_index.h" #include "knowhere/index/vector_index/nsg/nsg.h" diff --git a/cpp/src/core/test/faiss_ori/gpuresource_test.cpp b/cpp/src/core/test/faiss_ori/gpuresource_test.cpp index f88ad3e111..0dcc2766cc 100644 --- a/cpp/src/core/test/faiss_ori/gpuresource_test.cpp +++ b/cpp/src/core/test/faiss_ori/gpuresource_test.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include diff --git a/cpp/src/core/test/test_idmap.cpp b/cpp/src/core/test/test_idmap.cpp index 9a8001be2b..f85210b574 100644 --- a/cpp/src/core/test/test_idmap.cpp +++ b/cpp/src/core/test/test_idmap.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include diff --git a/cpp/src/core/test/test_ivf.cpp b/cpp/src/core/test/test_ivf.cpp index efa7ef17bd..f1ae3f44a6 100644 --- a/cpp/src/core/test/test_ivf.cpp +++ b/cpp/src/core/test/test_ivf.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include diff --git a/cpp/src/core/test/test_json.cpp b/cpp/src/core/test/test_json.cpp index bc4c1666ff..b3c4fd4993 100644 --- a/cpp/src/core/test/test_json.cpp +++ b/cpp/src/core/test/test_json.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "knowhere/common/config.h" diff --git a/cpp/src/core/test/test_kdt.cpp b/cpp/src/core/test/test_kdt.cpp index 34eeb9933b..9581cc8ee0 100644 --- a/cpp/src/core/test/test_kdt.cpp +++ b/cpp/src/core/test/test_kdt.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include diff --git a/cpp/src/core/test/test_nsg.cpp b/cpp/src/core/test/test_nsg.cpp index d7e472b13e..f156925027 100644 --- a/cpp/src/core/test/test_nsg.cpp +++ b/cpp/src/core/test/test_nsg.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include diff --git a/cpp/src/core/test/test_nsg/test_nsg.cpp b/cpp/src/core/test/test_nsg/test_nsg.cpp index 6d378c41a5..b1b1edef46 100644 --- a/cpp/src/core/test/test_nsg/test_nsg.cpp +++ b/cpp/src/core/test/test_nsg/test_nsg.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include diff --git a/cpp/src/core/test/utils.cpp b/cpp/src/core/test/utils.cpp index 5c007aac28..97f4600ca7 100644 --- a/cpp/src/core/test/utils.cpp +++ b/cpp/src/core/test/utils.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "utils.h" diff --git a/cpp/src/core/test/utils.h b/cpp/src/core/test/utils.h index ca56fedb2c..b9db01de51 100644 --- a/cpp/src/core/test/utils.h +++ b/cpp/src/core/test/utils.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/db/Constants.h b/cpp/src/db/Constants.h index 479f670563..aa254b0bd6 100644 --- a/cpp/src/db/Constants.h +++ b/cpp/src/db/Constants.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index 4dc4e562c8..1de424ed78 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "Options.h" diff --git a/cpp/src/db/DBFactory.cpp b/cpp/src/db/DBFactory.cpp index b04067c38a..e44e3f4d97 100644 --- a/cpp/src/db/DBFactory.cpp +++ b/cpp/src/db/DBFactory.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "DBFactory.h" #include "DBImpl.h" diff --git a/cpp/src/db/DBFactory.h b/cpp/src/db/DBFactory.h index a9404ccd1b..7de4396846 100644 --- a/cpp/src/db/DBFactory.h +++ b/cpp/src/db/DBFactory.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "DB.h" diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 80b1994778..205d27ad21 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "DBImpl.h" #include "src/db/meta/SqliteMetaImpl.h" #include "Log.h" diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index e4bfbd5d34..1d171b163f 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "DB.h" diff --git a/cpp/src/db/Exception.h b/cpp/src/db/Exception.h index 2474b70283..9132996b5a 100644 --- a/cpp/src/db/Exception.h +++ b/cpp/src/db/Exception.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/db/IDGenerator.cpp b/cpp/src/db/IDGenerator.cpp index 74d3a1433a..a82ccee629 100644 --- a/cpp/src/db/IDGenerator.cpp +++ b/cpp/src/db/IDGenerator.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "IDGenerator.h" #include diff --git a/cpp/src/db/IDGenerator.h b/cpp/src/db/IDGenerator.h index eb39221734..fc408697dd 100644 --- a/cpp/src/db/IDGenerator.h +++ b/cpp/src/db/IDGenerator.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "Types.h" diff --git a/cpp/src/db/Log.h b/cpp/src/db/Log.h index f36d861c3e..0a7bb7dd7f 100644 --- a/cpp/src/db/Log.h +++ b/cpp/src/db/Log.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "utils/easylogging++.h" diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index ecebd30bdb..fdb6731d1e 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include #include "utils/easylogging++.h" diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 4cd262ccfa..c5b4648c6d 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "Constants.h" diff --git a/cpp/src/db/Status.cpp b/cpp/src/db/Status.cpp index d19b1e66ff..0bc51ca59e 100644 --- a/cpp/src/db/Status.cpp +++ b/cpp/src/db/Status.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include #include diff --git a/cpp/src/db/Status.h b/cpp/src/db/Status.h index 60733a5548..e3dae4c0ad 100644 --- a/cpp/src/db/Status.h +++ b/cpp/src/db/Status.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "utils/Error.h" diff --git a/cpp/src/db/Types.h b/cpp/src/db/Types.h index 322b22048a..d927ecab5f 100644 --- a/cpp/src/db/Types.h +++ b/cpp/src/db/Types.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "db/engine/ExecutionEngine.h" diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index 2c9173f6a7..d197d85525 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "Utils.h" #include "utils/CommonUtil.h" #include "Log.h" diff --git a/cpp/src/db/Utils.h b/cpp/src/db/Utils.h index 2094250a1f..f27273c6fe 100644 --- a/cpp/src/db/Utils.h +++ b/cpp/src/db/Utils.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "Options.h" diff --git a/cpp/src/db/engine/EngineFactory.cpp b/cpp/src/db/engine/EngineFactory.cpp index a326d6a2c6..a975ca88c4 100644 --- a/cpp/src/db/engine/EngineFactory.cpp +++ b/cpp/src/db/engine/EngineFactory.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "EngineFactory.h" #include "ExecutionEngineImpl.h" #include "db/Log.h" diff --git a/cpp/src/db/engine/EngineFactory.h b/cpp/src/db/engine/EngineFactory.h index 7f2047af9b..906260b966 100644 --- a/cpp/src/db/engine/EngineFactory.h +++ b/cpp/src/db/engine/EngineFactory.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "db/Status.h" diff --git a/cpp/src/db/engine/ExecutionEngine.h b/cpp/src/db/engine/ExecutionEngine.h index 4b59baf28e..417d002a21 100644 --- a/cpp/src/db/engine/ExecutionEngine.h +++ b/cpp/src/db/engine/ExecutionEngine.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "db/Status.h" diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 8a738347d2..a062a5994a 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include "src/cache/GpuCacheMgr.h" diff --git a/cpp/src/db/engine/ExecutionEngineImpl.h b/cpp/src/db/engine/ExecutionEngineImpl.h index e5480322a2..611a247a08 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.h +++ b/cpp/src/db/engine/ExecutionEngineImpl.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "ExecutionEngine.h" diff --git a/cpp/src/db/insert/MemManager.h b/cpp/src/db/insert/MemManager.h index 81fd3c1108..de6ee85ed1 100644 --- a/cpp/src/db/insert/MemManager.h +++ b/cpp/src/db/insert/MemManager.h @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include "db/Status.h" diff --git a/cpp/src/db/insert/MemManagerImpl.cpp b/cpp/src/db/insert/MemManagerImpl.cpp index 240b2f2bb0..1c7f49a168 100644 --- a/cpp/src/db/insert/MemManagerImpl.cpp +++ b/cpp/src/db/insert/MemManagerImpl.cpp @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #include "MemManagerImpl.h" #include "VectorSource.h" #include "db/Log.h" diff --git a/cpp/src/db/insert/MemManagerImpl.h b/cpp/src/db/insert/MemManagerImpl.h index a17a90c2cd..9b503216a7 100644 --- a/cpp/src/db/insert/MemManagerImpl.h +++ b/cpp/src/db/insert/MemManagerImpl.h @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include "db/meta/Meta.h" diff --git a/cpp/src/db/insert/MemMenagerFactory.cpp b/cpp/src/db/insert/MemMenagerFactory.cpp index 965ba596bd..1f39ea3256 100644 --- a/cpp/src/db/insert/MemMenagerFactory.cpp +++ b/cpp/src/db/insert/MemMenagerFactory.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "MemMenagerFactory.h" #include "MemManagerImpl.h" diff --git a/cpp/src/db/insert/MemMenagerFactory.h b/cpp/src/db/insert/MemMenagerFactory.h index f20103a0eb..4bf4267c87 100644 --- a/cpp/src/db/insert/MemMenagerFactory.h +++ b/cpp/src/db/insert/MemMenagerFactory.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "MemManager.h" diff --git a/cpp/src/db/insert/MemTable.cpp b/cpp/src/db/insert/MemTable.cpp index fe6cdf0967..19814e3aad 100644 --- a/cpp/src/db/insert/MemTable.cpp +++ b/cpp/src/db/insert/MemTable.cpp @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #include "MemTable.h" #include "db/Log.h" diff --git a/cpp/src/db/insert/MemTable.h b/cpp/src/db/insert/MemTable.h index 54744e5214..a443186900 100644 --- a/cpp/src/db/insert/MemTable.h +++ b/cpp/src/db/insert/MemTable.h @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include "db/Status.h" diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index 946b658b4b..e991b7900c 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #include "MemTableFile.h" #include "db/Constants.h" #include "db/Log.h" diff --git a/cpp/src/db/insert/MemTableFile.h b/cpp/src/db/insert/MemTableFile.h index b816e9cddd..d4b11cda7a 100644 --- a/cpp/src/db/insert/MemTableFile.h +++ b/cpp/src/db/insert/MemTableFile.h @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include "db/Status.h" diff --git a/cpp/src/db/insert/VectorSource.cpp b/cpp/src/db/insert/VectorSource.cpp index f7d6a48297..e80667e31f 100644 --- a/cpp/src/db/insert/VectorSource.cpp +++ b/cpp/src/db/insert/VectorSource.cpp @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #include "VectorSource.h" #include "db/engine/ExecutionEngine.h" #include "db/engine/EngineFactory.h" diff --git a/cpp/src/db/insert/VectorSource.h b/cpp/src/db/insert/VectorSource.h index 23cce9bc2c..5c44f580d2 100644 --- a/cpp/src/db/insert/VectorSource.h +++ b/cpp/src/db/insert/VectorSource.h @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include "db/Status.h" diff --git a/cpp/src/db/meta/Meta.h b/cpp/src/db/meta/Meta.h index 35f5e938a0..755e0170f2 100644 --- a/cpp/src/db/meta/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "MetaTypes.h" diff --git a/cpp/src/db/meta/MetaConsts.h b/cpp/src/db/meta/MetaConsts.h index ef0a207eae..04b548009b 100644 --- a/cpp/src/db/meta/MetaConsts.h +++ b/cpp/src/db/meta/MetaConsts.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once namespace zilliz { diff --git a/cpp/src/db/meta/MetaFactory.cpp b/cpp/src/db/meta/MetaFactory.cpp index 9e025658af..ef9c4f13fd 100644 --- a/cpp/src/db/meta/MetaFactory.cpp +++ b/cpp/src/db/meta/MetaFactory.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "MetaFactory.h" #include "SqliteMetaImpl.h" diff --git a/cpp/src/db/meta/MetaFactory.h b/cpp/src/db/meta/MetaFactory.h index fa964c3094..edc9706649 100644 --- a/cpp/src/db/meta/MetaFactory.h +++ b/cpp/src/db/meta/MetaFactory.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "Meta.h" diff --git a/cpp/src/db/meta/MetaTypes.h b/cpp/src/db/meta/MetaTypes.h index e31be40ddc..2444e46bf4 100644 --- a/cpp/src/db/meta/MetaTypes.h +++ b/cpp/src/db/meta/MetaTypes.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "db/engine/ExecutionEngine.h" diff --git a/cpp/src/db/meta/MySQLConnectionPool.cpp b/cpp/src/db/meta/MySQLConnectionPool.cpp index 8e82dc5ae7..87998122f4 100644 --- a/cpp/src/db/meta/MySQLConnectionPool.cpp +++ b/cpp/src/db/meta/MySQLConnectionPool.cpp @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #include "MySQLConnectionPool.h" namespace zilliz { diff --git a/cpp/src/db/meta/MySQLConnectionPool.h b/cpp/src/db/meta/MySQLConnectionPool.h index 9cde818b45..98695f5739 100644 --- a/cpp/src/db/meta/MySQLConnectionPool.h +++ b/cpp/src/db/meta/MySQLConnectionPool.h @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #include "mysql++/mysql++.h" #include diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 6c89471741..20ec9648c5 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "MySQLMetaImpl.h" #include "db/IDGenerator.h" #include "db/Utils.h" diff --git a/cpp/src/db/meta/MySQLMetaImpl.h b/cpp/src/db/meta/MySQLMetaImpl.h index 97604ef3ea..dca109b231 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.h +++ b/cpp/src/db/meta/MySQLMetaImpl.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "Meta.h" diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 38941abf38..0c3e2aa3e5 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "SqliteMetaImpl.h" #include "db/IDGenerator.h" #include "db/Utils.h" diff --git a/cpp/src/db/meta/SqliteMetaImpl.h b/cpp/src/db/meta/SqliteMetaImpl.h index 641dcc40ec..b1465dd95a 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.h +++ b/cpp/src/db/meta/SqliteMetaImpl.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "Meta.h" diff --git a/cpp/src/db/scheduler/TaskDispatchQueue.cpp b/cpp/src/db/scheduler/TaskDispatchQueue.cpp index e1bf7b75f5..41a95060a8 100644 --- a/cpp/src/db/scheduler/TaskDispatchQueue.cpp +++ b/cpp/src/db/scheduler/TaskDispatchQueue.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "TaskDispatchQueue.h" #include "TaskDispatchStrategy.h" diff --git a/cpp/src/db/scheduler/TaskDispatchQueue.h b/cpp/src/db/scheduler/TaskDispatchQueue.h index 435be4f597..b8a0dd611e 100644 --- a/cpp/src/db/scheduler/TaskDispatchQueue.h +++ b/cpp/src/db/scheduler/TaskDispatchQueue.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "context/IScheduleContext.h" diff --git a/cpp/src/db/scheduler/TaskDispatchStrategy.cpp b/cpp/src/db/scheduler/TaskDispatchStrategy.cpp index 38c9463117..ce6d10c147 100644 --- a/cpp/src/db/scheduler/TaskDispatchStrategy.cpp +++ b/cpp/src/db/scheduler/TaskDispatchStrategy.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "TaskDispatchStrategy.h" #include "context/SearchContext.h" #include "context/DeleteContext.h" diff --git a/cpp/src/db/scheduler/TaskDispatchStrategy.h b/cpp/src/db/scheduler/TaskDispatchStrategy.h index d835e3ce86..8d1b51605e 100644 --- a/cpp/src/db/scheduler/TaskDispatchStrategy.h +++ b/cpp/src/db/scheduler/TaskDispatchStrategy.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "context/IScheduleContext.h" diff --git a/cpp/src/db/scheduler/TaskScheduler.cpp b/cpp/src/db/scheduler/TaskScheduler.cpp index 3e1807b8c5..a546dbabf1 100644 --- a/cpp/src/db/scheduler/TaskScheduler.cpp +++ b/cpp/src/db/scheduler/TaskScheduler.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "server/ServerConfig.h" #include "TaskScheduler.h" diff --git a/cpp/src/db/scheduler/TaskScheduler.h b/cpp/src/db/scheduler/TaskScheduler.h index f4556696ec..86feb37f8f 100644 --- a/cpp/src/db/scheduler/TaskScheduler.h +++ b/cpp/src/db/scheduler/TaskScheduler.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "context/IScheduleContext.h" diff --git a/cpp/src/db/scheduler/context/DeleteContext.cpp b/cpp/src/db/scheduler/context/DeleteContext.cpp index 9207697c5f..32ec2f8f09 100644 --- a/cpp/src/db/scheduler/context/DeleteContext.cpp +++ b/cpp/src/db/scheduler/context/DeleteContext.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "DeleteContext.h" diff --git a/cpp/src/db/scheduler/context/DeleteContext.h b/cpp/src/db/scheduler/context/DeleteContext.h index 6c21d1e65a..7132acc827 100644 --- a/cpp/src/db/scheduler/context/DeleteContext.h +++ b/cpp/src/db/scheduler/context/DeleteContext.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "IScheduleContext.h" diff --git a/cpp/src/db/scheduler/context/IScheduleContext.h b/cpp/src/db/scheduler/context/IScheduleContext.h index 6ae56e364e..1ac347526c 100644 --- a/cpp/src/db/scheduler/context/IScheduleContext.h +++ b/cpp/src/db/scheduler/context/IScheduleContext.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/db/scheduler/context/SearchContext.cpp b/cpp/src/db/scheduler/context/SearchContext.cpp index 4043f04159..ae9ced5938 100644 --- a/cpp/src/db/scheduler/context/SearchContext.cpp +++ b/cpp/src/db/scheduler/context/SearchContext.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "SearchContext.h" #include "utils/Log.h" diff --git a/cpp/src/db/scheduler/context/SearchContext.h b/cpp/src/db/scheduler/context/SearchContext.h index a950962b6e..3aee2c436e 100644 --- a/cpp/src/db/scheduler/context/SearchContext.h +++ b/cpp/src/db/scheduler/context/SearchContext.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "IScheduleContext.h" diff --git a/cpp/src/db/scheduler/task/DeleteTask.cpp b/cpp/src/db/scheduler/task/DeleteTask.cpp index 975a595e6f..310b844ff9 100644 --- a/cpp/src/db/scheduler/task/DeleteTask.cpp +++ b/cpp/src/db/scheduler/task/DeleteTask.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "DeleteTask.h" diff --git a/cpp/src/db/scheduler/task/DeleteTask.h b/cpp/src/db/scheduler/task/DeleteTask.h index 866f0a7789..362b125c92 100644 --- a/cpp/src/db/scheduler/task/DeleteTask.h +++ b/cpp/src/db/scheduler/task/DeleteTask.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "IScheduleTask.h" diff --git a/cpp/src/db/scheduler/task/IScheduleTask.h b/cpp/src/db/scheduler/task/IScheduleTask.h index 652a1739fe..07025a7d72 100644 --- a/cpp/src/db/scheduler/task/IScheduleTask.h +++ b/cpp/src/db/scheduler/task/IScheduleTask.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/db/scheduler/task/IndexLoadTask.cpp b/cpp/src/db/scheduler/task/IndexLoadTask.cpp index 4efb041d61..3dc91a279c 100644 --- a/cpp/src/db/scheduler/task/IndexLoadTask.cpp +++ b/cpp/src/db/scheduler/task/IndexLoadTask.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "IndexLoadTask.h" #include "SearchTask.h" diff --git a/cpp/src/db/scheduler/task/IndexLoadTask.h b/cpp/src/db/scheduler/task/IndexLoadTask.h index c3548d7311..29356e5f58 100644 --- a/cpp/src/db/scheduler/task/IndexLoadTask.h +++ b/cpp/src/db/scheduler/task/IndexLoadTask.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "IScheduleTask.h" diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index 5117dc948c..e79ddbcb8c 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "SearchTask.h" #include "metrics/Metrics.h" #include "db/Log.h" diff --git a/cpp/src/db/scheduler/task/SearchTask.h b/cpp/src/db/scheduler/task/SearchTask.h index 88a51c2c84..47c4813fd2 100644 --- a/cpp/src/db/scheduler/task/SearchTask.h +++ b/cpp/src/db/scheduler/task/SearchTask.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "IScheduleTask.h" diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index e31e286652..d9b160d6e0 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "server/Server.h" #include "version.h" diff --git a/cpp/src/metrics/MetricBase.h b/cpp/src/metrics/MetricBase.h index 315eaa0f88..aa181fb3f3 100644 --- a/cpp/src/metrics/MetricBase.h +++ b/cpp/src/metrics/MetricBase.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/metrics/Metrics.cpp b/cpp/src/metrics/Metrics.cpp index 23fb0a15b8..10ddd7ee3c 100644 --- a/cpp/src/metrics/Metrics.cpp +++ b/cpp/src/metrics/Metrics.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "Metrics.h" #include "PrometheusMetrics.h" diff --git a/cpp/src/metrics/Metrics.h b/cpp/src/metrics/Metrics.h index 83b082fe46..e298af7acf 100644 --- a/cpp/src/metrics/Metrics.h +++ b/cpp/src/metrics/Metrics.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "MetricBase.h" diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index 07ca9aa301..f4d70a21c3 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include "PrometheusMetrics.h" diff --git a/cpp/src/metrics/PrometheusMetrics.h b/cpp/src/metrics/PrometheusMetrics.h index 70443d6331..5a3379305a 100644 --- a/cpp/src/metrics/PrometheusMetrics.h +++ b/cpp/src/metrics/PrometheusMetrics.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "utils/Error.h" diff --git a/cpp/src/metrics/SystemInfo.cpp b/cpp/src/metrics/SystemInfo.cpp index 3b6698d42b..61ae42c438 100644 --- a/cpp/src/metrics/SystemInfo.cpp +++ b/cpp/src/metrics/SystemInfo.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "SystemInfo.h" diff --git a/cpp/src/metrics/SystemInfo.h b/cpp/src/metrics/SystemInfo.h index ab27375c73..9089c7cddb 100644 --- a/cpp/src/metrics/SystemInfo.h +++ b/cpp/src/metrics/SystemInfo.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/scheduler/Algorithm.cpp b/cpp/src/scheduler/Algorithm.cpp index f8ad0212d4..0a9aefbce0 100644 --- a/cpp/src/scheduler/Algorithm.cpp +++ b/cpp/src/scheduler/Algorithm.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "Algorithm.h" diff --git a/cpp/src/scheduler/Algorithm.h b/cpp/src/scheduler/Algorithm.h index 05d9ad71d8..763ef8ad5d 100644 --- a/cpp/src/scheduler/Algorithm.h +++ b/cpp/src/scheduler/Algorithm.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "resource/Resource.h" #include "ResourceMgr.h" diff --git a/cpp/src/scheduler/ResourceFactory.cpp b/cpp/src/scheduler/ResourceFactory.cpp index 71137c361a..40cd00f206 100644 --- a/cpp/src/scheduler/ResourceFactory.cpp +++ b/cpp/src/scheduler/ResourceFactory.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "ResourceFactory.h" diff --git a/cpp/src/scheduler/ResourceFactory.h b/cpp/src/scheduler/ResourceFactory.h index 84549c62f1..58a29497b0 100644 --- a/cpp/src/scheduler/ResourceFactory.h +++ b/cpp/src/scheduler/ResourceFactory.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/scheduler/ResourceMgr.cpp b/cpp/src/scheduler/ResourceMgr.cpp index cbbf449763..5ce3d572bc 100644 --- a/cpp/src/scheduler/ResourceMgr.cpp +++ b/cpp/src/scheduler/ResourceMgr.cpp @@ -1,9 +1,21 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "ResourceMgr.h" #include "db/Log.h" diff --git a/cpp/src/scheduler/ResourceMgr.h b/cpp/src/scheduler/ResourceMgr.h index 114cc778e1..ebe0894770 100644 --- a/cpp/src/scheduler/ResourceMgr.h +++ b/cpp/src/scheduler/ResourceMgr.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index df71e69af3..5432227523 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "SchedInst.h" #include "server/ServerConfig.h" diff --git a/cpp/src/scheduler/SchedInst.h b/cpp/src/scheduler/SchedInst.h index 92f3575ebc..fe73aae362 100644 --- a/cpp/src/scheduler/SchedInst.h +++ b/cpp/src/scheduler/SchedInst.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "ResourceMgr.h" diff --git a/cpp/src/scheduler/Scheduler.cpp b/cpp/src/scheduler/Scheduler.cpp index e70eecc00d..6aa4501e29 100644 --- a/cpp/src/scheduler/Scheduler.cpp +++ b/cpp/src/scheduler/Scheduler.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include "event/LoadCompletedEvent.h" diff --git a/cpp/src/scheduler/Scheduler.h b/cpp/src/scheduler/Scheduler.h index c2a36069b9..0de80d5cc5 100644 --- a/cpp/src/scheduler/Scheduler.h +++ b/cpp/src/scheduler/Scheduler.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/scheduler/TaskTable.cpp b/cpp/src/scheduler/TaskTable.cpp index 91d0bd7052..1678665593 100644 --- a/cpp/src/scheduler/TaskTable.cpp +++ b/cpp/src/scheduler/TaskTable.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "TaskTable.h" #include "event/TaskTableUpdatedEvent.h" diff --git a/cpp/src/scheduler/TaskTable.h b/cpp/src/scheduler/TaskTable.h index 7b064f20d4..c4be442d11 100644 --- a/cpp/src/scheduler/TaskTable.h +++ b/cpp/src/scheduler/TaskTable.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/scheduler/Utils.cpp b/cpp/src/scheduler/Utils.cpp index 6c1f6372be..b011751105 100644 --- a/cpp/src/scheduler/Utils.cpp +++ b/cpp/src/scheduler/Utils.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "Utils.h" diff --git a/cpp/src/scheduler/Utils.h b/cpp/src/scheduler/Utils.h index c00325db17..066695b35b 100644 --- a/cpp/src/scheduler/Utils.h +++ b/cpp/src/scheduler/Utils.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include diff --git a/cpp/src/scheduler/action/Action.h b/cpp/src/scheduler/action/Action.h index 7c7b4c3bec..4af8e1cfee 100644 --- a/cpp/src/scheduler/action/Action.h +++ b/cpp/src/scheduler/action/Action.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * copyright 上海赜睿信息科技有限公司(zilliz) - all rights reserved - * unauthorized copying of this file, via any medium is strictly prohibited. - * proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "../resource/Resource.h" diff --git a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp index 3de0165b0f..05f997e96d 100644 --- a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include diff --git a/cpp/src/scheduler/event/Event.h b/cpp/src/scheduler/event/Event.h index c022f96788..af8be5ecf0 100644 --- a/cpp/src/scheduler/event/Event.h +++ b/cpp/src/scheduler/event/Event.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * copyright 上海赜睿信息科技有限公司(zilliz) - all rights reserved - * unauthorized copying of this file, via any medium is strictly prohibited. - * proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/scheduler/event/EventDump.cpp b/cpp/src/scheduler/event/EventDump.cpp index ceee63ba9d..384fd98e8d 100644 --- a/cpp/src/scheduler/event/EventDump.cpp +++ b/cpp/src/scheduler/event/EventDump.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "Event.h" #include "StartUpEvent.h" diff --git a/cpp/src/scheduler/event/FinishTaskEvent.h b/cpp/src/scheduler/event/FinishTaskEvent.h index 14daa9b532..3c88a62693 100644 --- a/cpp/src/scheduler/event/FinishTaskEvent.h +++ b/cpp/src/scheduler/event/FinishTaskEvent.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * copyright 上海赜睿信息科技有限公司(zilliz) - all rights reserved - * unauthorized copying of this file, via any medium is strictly prohibited. - * proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "Event.h" diff --git a/cpp/src/scheduler/event/LoadCompletedEvent.h b/cpp/src/scheduler/event/LoadCompletedEvent.h index c59b6256cb..7e7abaa697 100644 --- a/cpp/src/scheduler/event/LoadCompletedEvent.h +++ b/cpp/src/scheduler/event/LoadCompletedEvent.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * copyright 上海赜睿信息科技有限公司(zilliz) - all rights reserved - * unauthorized copying of this file, via any medium is strictly prohibited. - * proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "Event.h" diff --git a/cpp/src/scheduler/event/StartUpEvent.h b/cpp/src/scheduler/event/StartUpEvent.h index 4b5ec78cd6..0ae4db5ce0 100644 --- a/cpp/src/scheduler/event/StartUpEvent.h +++ b/cpp/src/scheduler/event/StartUpEvent.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * copyright 上海赜睿信息科技有限公司(zilliz) - all rights reserved - * unauthorized copying of this file, via any medium is strictly prohibited. - * proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "Event.h" diff --git a/cpp/src/scheduler/event/TaskTableUpdatedEvent.h b/cpp/src/scheduler/event/TaskTableUpdatedEvent.h index f96c30674c..766df9254d 100644 --- a/cpp/src/scheduler/event/TaskTableUpdatedEvent.h +++ b/cpp/src/scheduler/event/TaskTableUpdatedEvent.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * copyright 上海赜睿信息科技有限公司(zilliz) - all rights reserved - * unauthorized copying of this file, via any medium is strictly prohibited. - * proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "Event.h" diff --git a/cpp/src/scheduler/resource/Connection.h b/cpp/src/scheduler/resource/Connection.h index 922cf2b437..ae7bf88e86 100644 --- a/cpp/src/scheduler/resource/Connection.h +++ b/cpp/src/scheduler/resource/Connection.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/scheduler/resource/CpuResource.cpp b/cpp/src/scheduler/resource/CpuResource.cpp index 9428e1816b..863a427096 100644 --- a/cpp/src/scheduler/resource/CpuResource.cpp +++ b/cpp/src/scheduler/resource/CpuResource.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "CpuResource.h" diff --git a/cpp/src/scheduler/resource/CpuResource.h b/cpp/src/scheduler/resource/CpuResource.h index 01822f8d9f..19bac6090d 100644 --- a/cpp/src/scheduler/resource/CpuResource.h +++ b/cpp/src/scheduler/resource/CpuResource.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/scheduler/resource/DiskResource.cpp b/cpp/src/scheduler/resource/DiskResource.cpp index 943e1e5efb..bb07f96b96 100644 --- a/cpp/src/scheduler/resource/DiskResource.cpp +++ b/cpp/src/scheduler/resource/DiskResource.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "DiskResource.h" diff --git a/cpp/src/scheduler/resource/DiskResource.h b/cpp/src/scheduler/resource/DiskResource.h index 8c70404e83..8c8a6ae46a 100644 --- a/cpp/src/scheduler/resource/DiskResource.h +++ b/cpp/src/scheduler/resource/DiskResource.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/scheduler/resource/GpuResource.cpp b/cpp/src/scheduler/resource/GpuResource.cpp index 38f1d624a6..c8959cf393 100644 --- a/cpp/src/scheduler/resource/GpuResource.cpp +++ b/cpp/src/scheduler/resource/GpuResource.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "GpuResource.h" diff --git a/cpp/src/scheduler/resource/GpuResource.h b/cpp/src/scheduler/resource/GpuResource.h index 5ef47a127d..12947a35fa 100644 --- a/cpp/src/scheduler/resource/GpuResource.h +++ b/cpp/src/scheduler/resource/GpuResource.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/scheduler/resource/Node.cpp b/cpp/src/scheduler/resource/Node.cpp index 0b322abb57..d4c42a5a97 100644 --- a/cpp/src/scheduler/resource/Node.cpp +++ b/cpp/src/scheduler/resource/Node.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include "Node.h" diff --git a/cpp/src/scheduler/resource/Node.h b/cpp/src/scheduler/resource/Node.h index 568aaf93e7..311ec8156a 100644 --- a/cpp/src/scheduler/resource/Node.h +++ b/cpp/src/scheduler/resource/Node.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/scheduler/resource/Resource.cpp b/cpp/src/scheduler/resource/Resource.cpp index 9bb9b57209..b094bb1c06 100644 --- a/cpp/src/scheduler/resource/Resource.cpp +++ b/cpp/src/scheduler/resource/Resource.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include "../Utils.h" #include "Resource.h" diff --git a/cpp/src/scheduler/resource/Resource.h b/cpp/src/scheduler/resource/Resource.h index a47bef83b1..f49ea71b5c 100644 --- a/cpp/src/scheduler/resource/Resource.h +++ b/cpp/src/scheduler/resource/Resource.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/scheduler/resource/TestResource.cpp b/cpp/src/scheduler/resource/TestResource.cpp index 8c35481695..ebdcb403bd 100644 --- a/cpp/src/scheduler/resource/TestResource.cpp +++ b/cpp/src/scheduler/resource/TestResource.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "TestResource.h" diff --git a/cpp/src/scheduler/resource/TestResource.h b/cpp/src/scheduler/resource/TestResource.h index 0468b2fa12..b8f749ba58 100644 --- a/cpp/src/scheduler/resource/TestResource.h +++ b/cpp/src/scheduler/resource/TestResource.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/scheduler/task/DeleteTask.cpp b/cpp/src/scheduler/task/DeleteTask.cpp index 69b5ed7f05..ded1ad0462 100644 --- a/cpp/src/scheduler/task/DeleteTask.cpp +++ b/cpp/src/scheduler/task/DeleteTask.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "DeleteTask.h" diff --git a/cpp/src/scheduler/task/DeleteTask.h b/cpp/src/scheduler/task/DeleteTask.h index 4f9909081f..b49107cc2a 100644 --- a/cpp/src/scheduler/task/DeleteTask.h +++ b/cpp/src/scheduler/task/DeleteTask.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/scheduler/task/Path.h b/cpp/src/scheduler/task/Path.h index 388a7b9c82..64df58b46c 100644 --- a/cpp/src/scheduler/task/Path.h +++ b/cpp/src/scheduler/task/Path.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/scheduler/task/SearchTask.cpp b/cpp/src/scheduler/task/SearchTask.cpp index 46d429ab56..6da63964ce 100644 --- a/cpp/src/scheduler/task/SearchTask.cpp +++ b/cpp/src/scheduler/task/SearchTask.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "src/metrics/Metrics.h" #include "src/utils/TimeRecorder.h" diff --git a/cpp/src/scheduler/task/SearchTask.h b/cpp/src/scheduler/task/SearchTask.h index 6adda7c1fb..b92e5bd851 100644 --- a/cpp/src/scheduler/task/SearchTask.h +++ b/cpp/src/scheduler/task/SearchTask.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "Task.h" diff --git a/cpp/src/scheduler/task/Task.h b/cpp/src/scheduler/task/Task.h index 0523edfaa0..bfaa1fe68d 100644 --- a/cpp/src/scheduler/task/Task.h +++ b/cpp/src/scheduler/task/Task.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "db/scheduler/context/SearchContext.h" diff --git a/cpp/src/scheduler/task/TaskConvert.cpp b/cpp/src/scheduler/task/TaskConvert.cpp index e55f84f51f..6be6309093 100644 --- a/cpp/src/scheduler/task/TaskConvert.cpp +++ b/cpp/src/scheduler/task/TaskConvert.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "TaskConvert.h" #include "scheduler/tasklabel/DefaultLabel.h" diff --git a/cpp/src/scheduler/task/TaskConvert.h b/cpp/src/scheduler/task/TaskConvert.h index bfc2df760b..2420c48b23 100644 --- a/cpp/src/scheduler/task/TaskConvert.h +++ b/cpp/src/scheduler/task/TaskConvert.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "db/scheduler/task/DeleteTask.h" #include "db/scheduler/task/IndexLoadTask.h" diff --git a/cpp/src/scheduler/task/TestTask.cpp b/cpp/src/scheduler/task/TestTask.cpp index 1078da583f..f1f9c29477 100644 --- a/cpp/src/scheduler/task/TestTask.cpp +++ b/cpp/src/scheduler/task/TestTask.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include "TestTask.h" diff --git a/cpp/src/scheduler/task/TestTask.h b/cpp/src/scheduler/task/TestTask.h index 6e969e9084..66fd864e1d 100644 --- a/cpp/src/scheduler/task/TestTask.h +++ b/cpp/src/scheduler/task/TestTask.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "SearchTask.h" diff --git a/cpp/src/scheduler/tasklabel/BroadcastLabel.h b/cpp/src/scheduler/tasklabel/BroadcastLabel.h index 406add5122..2faa49acd1 100644 --- a/cpp/src/scheduler/tasklabel/BroadcastLabel.h +++ b/cpp/src/scheduler/tasklabel/BroadcastLabel.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "TaskLabel.h" diff --git a/cpp/src/scheduler/tasklabel/DefaultLabel.h b/cpp/src/scheduler/tasklabel/DefaultLabel.h index ada34cd679..d1619684b9 100644 --- a/cpp/src/scheduler/tasklabel/DefaultLabel.h +++ b/cpp/src/scheduler/tasklabel/DefaultLabel.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "TaskLabel.h" diff --git a/cpp/src/scheduler/tasklabel/SpecResLabel.h b/cpp/src/scheduler/tasklabel/SpecResLabel.h index 51468bf28b..3955625412 100644 --- a/cpp/src/scheduler/tasklabel/SpecResLabel.h +++ b/cpp/src/scheduler/tasklabel/SpecResLabel.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "TaskLabel.h" diff --git a/cpp/src/scheduler/tasklabel/TaskLabel.h b/cpp/src/scheduler/tasklabel/TaskLabel.h index 5160248768..da8049ebf9 100644 --- a/cpp/src/scheduler/tasklabel/TaskLabel.h +++ b/cpp/src/scheduler/tasklabel/TaskLabel.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/sdk/CMakeLists.txt b/cpp/src/sdk/CMakeLists.txt index 150a1fdf16..6c3a68b8b8 100644 --- a/cpp/src/sdk/CMakeLists.txt +++ b/cpp/src/sdk/CMakeLists.txt @@ -1,9 +1,23 @@ #------------------------------------------------------------------------------- -# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -# Unauthorized copying of this file, via any medium is strictly prohibited. -# Proprietary and confidential. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. #------------------------------------------------------------------------------- + aux_source_directory(interface interface_files) aux_source_directory(util util_files) diff --git a/cpp/src/sdk/examples/CMakeLists.txt b/cpp/src/sdk/examples/CMakeLists.txt index 9e0437f157..aa15190178 100644 --- a/cpp/src/sdk/examples/CMakeLists.txt +++ b/cpp/src/sdk/examples/CMakeLists.txt @@ -1,7 +1,21 @@ #------------------------------------------------------------------------------- -# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -# Unauthorized copying of this file, via any medium is strictly prohibited. -# Proprietary and confidential. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. #------------------------------------------------------------------------------- + add_subdirectory(grpcsimple) diff --git a/cpp/src/sdk/examples/grpcsimple/CMakeLists.txt b/cpp/src/sdk/examples/grpcsimple/CMakeLists.txt index 36f6364780..b6c67061be 100644 --- a/cpp/src/sdk/examples/grpcsimple/CMakeLists.txt +++ b/cpp/src/sdk/examples/grpcsimple/CMakeLists.txt @@ -1,9 +1,23 @@ #------------------------------------------------------------------------------- -# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -# Unauthorized copying of this file, via any medium is strictly prohibited. -# Proprietary and confidential. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. #------------------------------------------------------------------------------- + aux_source_directory(src src_files) include_directories(src) diff --git a/cpp/src/sdk/examples/grpcsimple/main.cpp b/cpp/src/sdk/examples/grpcsimple/main.cpp index 499b8a9935..4de060590c 100644 --- a/cpp/src/sdk/examples/grpcsimple/main.cpp +++ b/cpp/src/sdk/examples/grpcsimple/main.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include diff --git a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp index 2a5b767137..abfc92cc0b 100644 --- a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "ClientTest.h" #include "MilvusApi.h" #include "cache/CpuCacheMgr.h" diff --git a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h index 0f42577ad1..d95897927e 100644 --- a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h +++ b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/sdk/grpc/ClientProxy.cpp b/cpp/src/sdk/grpc/ClientProxy.cpp index 2163e78c66..56ba1e2de6 100644 --- a/cpp/src/sdk/grpc/ClientProxy.cpp +++ b/cpp/src/sdk/grpc/ClientProxy.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* -* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -* Unauthorized copying of this file, via any medium is strictly prohibited. -* Proprietary and confidential. -******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "ClientProxy.h" #include "version.h" #include "milvus.grpc.pb.h" diff --git a/cpp/src/sdk/grpc/ClientProxy.h b/cpp/src/sdk/grpc/ClientProxy.h index 300d316c36..c3f79d7209 100644 --- a/cpp/src/sdk/grpc/ClientProxy.h +++ b/cpp/src/sdk/grpc/ClientProxy.h @@ -1,8 +1,20 @@ -/******************************************************************************* -* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -* Unauthorized copying of this file, via any medium is strictly prohibited. -* Proprietary and confidential. -******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "MilvusApi.h" diff --git a/cpp/src/sdk/grpc/GrpcClient.cpp b/cpp/src/sdk/grpc/GrpcClient.cpp index 4e0283318a..146a96ec29 100644 --- a/cpp/src/sdk/grpc/GrpcClient.cpp +++ b/cpp/src/sdk/grpc/GrpcClient.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* -* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -* Unauthorized copying of this file, via any medium is strictly prohibited. -* Proprietary and confidential. -******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include #include diff --git a/cpp/src/sdk/grpc/GrpcClient.h b/cpp/src/sdk/grpc/GrpcClient.h index fb64fdb50c..ba9ba05f19 100644 --- a/cpp/src/sdk/grpc/GrpcClient.h +++ b/cpp/src/sdk/grpc/GrpcClient.h @@ -1,8 +1,20 @@ -/******************************************************************************* -* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -* Unauthorized copying of this file, via any medium is strictly prohibited. -* Proprietary and confidential. -******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include #include diff --git a/cpp/src/sdk/include/MilvusApi.h b/cpp/src/sdk/include/MilvusApi.h index 01147d36fb..05de495fbc 100644 --- a/cpp/src/sdk/include/MilvusApi.h +++ b/cpp/src/sdk/include/MilvusApi.h @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include "Status.h" diff --git a/cpp/src/sdk/include/Status.h b/cpp/src/sdk/include/Status.h index 7804f681f5..6dbc762c0a 100644 --- a/cpp/src/sdk/include/Status.h +++ b/cpp/src/sdk/include/Status.h @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include diff --git a/cpp/src/sdk/interface/ConnectionImpl.cpp b/cpp/src/sdk/interface/ConnectionImpl.cpp index 6fb955594d..3730cfd191 100644 --- a/cpp/src/sdk/interface/ConnectionImpl.cpp +++ b/cpp/src/sdk/interface/ConnectionImpl.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* -* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -* Unauthorized copying of this file, via any medium is strictly prohibited. -* Proprietary and confidential. -******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "ConnectionImpl.h" namespace milvus { diff --git a/cpp/src/sdk/interface/ConnectionImpl.h b/cpp/src/sdk/interface/ConnectionImpl.h index c8e834b008..6a402fcb0d 100644 --- a/cpp/src/sdk/interface/ConnectionImpl.h +++ b/cpp/src/sdk/interface/ConnectionImpl.h @@ -1,8 +1,20 @@ -/******************************************************************************* -* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -* Unauthorized copying of this file, via any medium is strictly prohibited. -* Proprietary and confidential. -******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "MilvusApi.h" diff --git a/cpp/src/sdk/interface/Status.cpp b/cpp/src/sdk/interface/Status.cpp index 14c43ce084..344d86c12f 100644 --- a/cpp/src/sdk/interface/Status.cpp +++ b/cpp/src/sdk/interface/Status.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* -* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -* Unauthorized copying of this file, via any medium is strictly prohibited. -* Proprietary and confidential. -******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "Status.h" namespace milvus { diff --git a/cpp/src/sdk/util/Exception.h b/cpp/src/sdk/util/Exception.h index 0ad08305f8..f55b8f0632 100644 --- a/cpp/src/sdk/util/Exception.h +++ b/cpp/src/sdk/util/Exception.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "Status.h" diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 25f57419b8..d4a145ff37 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "DBWrapper.h" #include "ServerConfig.h" diff --git a/cpp/src/server/DBWrapper.h b/cpp/src/server/DBWrapper.h index b47b18b1f7..a321e3cdae 100644 --- a/cpp/src/server/DBWrapper.h +++ b/cpp/src/server/DBWrapper.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "utils/Error.h" diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index b9e6494eb8..f64ec009b9 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include "Server.h" #include "server/grpc_impl/GrpcMilvusServer.h" diff --git a/cpp/src/server/Server.h b/cpp/src/server/Server.h index c5ea076fb9..f66c8a314b 100644 --- a/cpp/src/server/Server.h +++ b/cpp/src/server/Server.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "utils/Error.h" diff --git a/cpp/src/server/ServerConfig.cpp b/cpp/src/server/ServerConfig.cpp index 13217e9377..988ee707b6 100644 --- a/cpp/src/server/ServerConfig.cpp +++ b/cpp/src/server/ServerConfig.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "ServerConfig.h" #include diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index a8d906f710..d1f414b47b 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "utils/Error.h" diff --git a/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp b/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp index 3b0927b438..7b17cd6e56 100644 --- a/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* -* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -* Unauthorized copying of this file, via any medium is strictly prohibited. -* Proprietary and confidential. -******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "milvus.grpc.pb.h" #include "GrpcMilvusServer.h" #include "../ServerConfig.h" diff --git a/cpp/src/server/grpc_impl/GrpcMilvusServer.h b/cpp/src/server/grpc_impl/GrpcMilvusServer.h index fe62563bdf..2ddc5ee2fb 100644 --- a/cpp/src/server/grpc_impl/GrpcMilvusServer.h +++ b/cpp/src/server/grpc_impl/GrpcMilvusServer.h @@ -1,8 +1,20 @@ -/******************************************************************************* -* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -* Unauthorized copying of this file, via any medium is strictly prohibited. -* Proprietary and confidential. -******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp b/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp index b1ee3ad073..4fe86c3d1d 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* -* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -* Unauthorized copying of this file, via any medium is strictly prohibited. -* Proprietary and confidential. -******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "GrpcRequestHandler.h" #include "GrpcRequestTask.h" diff --git a/cpp/src/server/grpc_impl/GrpcRequestHandler.h b/cpp/src/server/grpc_impl/GrpcRequestHandler.h index 40537c5e67..b855cdebcb 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestHandler.h +++ b/cpp/src/server/grpc_impl/GrpcRequestHandler.h @@ -1,8 +1,20 @@ -/******************************************************************************* -* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -* Unauthorized copying of this file, via any medium is strictly prohibited. -* Proprietary and confidential. -******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp index 9f6fe4d409..81ef8d81ad 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* -* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -* Unauthorized copying of this file, via any medium is strictly prohibited. -* Proprietary and confidential. -******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "GrpcRequestScheduler.h" #include "utils/Log.h" diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.h b/cpp/src/server/grpc_impl/GrpcRequestScheduler.h index b873904cec..f8f287b1e6 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.h +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.h @@ -1,8 +1,20 @@ -/******************************************************************************* -* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -* Unauthorized copying of this file, via any medium is strictly prohibited. -* Proprietary and confidential. -******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "utils/BlockingQueue.h" diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index 9800b3e754..8c7697e1dc 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* -* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -* Unauthorized copying of this file, via any medium is strictly prohibited. -* Proprietary and confidential. -******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "GrpcRequestTask.h" #include "../ServerConfig.h" #include "utils/CommonUtil.h" diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.h b/cpp/src/server/grpc_impl/GrpcRequestTask.h index ab43bdef70..42a1048124 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.h +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.h @@ -1,8 +1,20 @@ -/******************************************************************************* -* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -* Unauthorized copying of this file, via any medium is strictly prohibited. -* Proprietary and confidential. -******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "GrpcRequestScheduler.h" #include "utils/Error.h" diff --git a/cpp/src/storage/IStorage.h b/cpp/src/storage/IStorage.h index 918834b106..431dd45caf 100644 --- a/cpp/src/storage/IStorage.h +++ b/cpp/src/storage/IStorage.h @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include "db/Status.h" diff --git a/cpp/src/storage/s3/S3ClientWrapper.cpp b/cpp/src/storage/s3/S3ClientWrapper.cpp index b739bc1e28..2dca5b3eca 100644 --- a/cpp/src/storage/s3/S3ClientWrapper.cpp +++ b/cpp/src/storage/s3/S3ClientWrapper.cpp @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + //#include "S3ClientWrapper.h" // //#include diff --git a/cpp/src/storage/s3/S3ClientWrapper.h b/cpp/src/storage/s3/S3ClientWrapper.h index fc66304a5a..95a940655e 100644 --- a/cpp/src/storage/s3/S3ClientWrapper.h +++ b/cpp/src/storage/s3/S3ClientWrapper.h @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + //#pragma once // //#include "storage/IStorage.h" diff --git a/cpp/src/utils/BlockingQueue.h b/cpp/src/utils/BlockingQueue.h index a77afd7f96..6e46ed1bb1 100644 --- a/cpp/src/utils/BlockingQueue.h +++ b/cpp/src/utils/BlockingQueue.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/utils/BlockingQueue.inl b/cpp/src/utils/BlockingQueue.inl index 21146cdfbb..8df38dd547 100644 --- a/cpp/src/utils/BlockingQueue.inl +++ b/cpp/src/utils/BlockingQueue.inl @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once //#include "Log.h" diff --git a/cpp/src/utils/CommonUtil.cpp b/cpp/src/utils/CommonUtil.cpp index c8fdb62e15..0f3f7a229e 100644 --- a/cpp/src/utils/CommonUtil.cpp +++ b/cpp/src/utils/CommonUtil.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "CommonUtil.h" #include "utils/Log.h" diff --git a/cpp/src/utils/CommonUtil.h b/cpp/src/utils/CommonUtil.h index 99122142ad..8c825f70fe 100755 --- a/cpp/src/utils/CommonUtil.h +++ b/cpp/src/utils/CommonUtil.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index f49e4b36af..d7008115f9 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/utils/Log.h b/cpp/src/utils/Log.h index 204d5812b6..f7c7b27f87 100644 --- a/cpp/src/utils/Log.h +++ b/cpp/src/utils/Log.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "Error.h" diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp index 35ff862d50..6224c854a5 100644 --- a/cpp/src/utils/LogUtil.cpp +++ b/cpp/src/utils/LogUtil.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "LogUtil.h" #include "server/ServerConfig.h" #include "easylogging++.h" diff --git a/cpp/src/utils/LogUtil.h b/cpp/src/utils/LogUtil.h index e2cb011ce7..ebeb117aa2 100644 --- a/cpp/src/utils/LogUtil.h +++ b/cpp/src/utils/LogUtil.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/utils/SignalUtil.cpp b/cpp/src/utils/SignalUtil.cpp index 2bc62b2364..07c1001044 100644 --- a/cpp/src/utils/SignalUtil.cpp +++ b/cpp/src/utils/SignalUtil.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "SignalUtil.h" #include "src/server/Server.h" #include "utils/Log.h" diff --git a/cpp/src/utils/SignalUtil.h b/cpp/src/utils/SignalUtil.h index a8fbf54d48..39645cf989 100644 --- a/cpp/src/utils/SignalUtil.h +++ b/cpp/src/utils/SignalUtil.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once namespace zilliz { diff --git a/cpp/src/utils/StringHelpFunctions.cpp b/cpp/src/utils/StringHelpFunctions.cpp index d45f515761..9c4a6d3a3a 100644 --- a/cpp/src/utils/StringHelpFunctions.cpp +++ b/cpp/src/utils/StringHelpFunctions.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "StringHelpFunctions.h" namespace zilliz { diff --git a/cpp/src/utils/StringHelpFunctions.h b/cpp/src/utils/StringHelpFunctions.h index f9101ab1e2..1fa45322da 100644 --- a/cpp/src/utils/StringHelpFunctions.h +++ b/cpp/src/utils/StringHelpFunctions.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include "./Error.h" diff --git a/cpp/src/utils/ThreadPool.h b/cpp/src/utils/ThreadPool.h index cf7cc56c65..a4df85cc14 100644 --- a/cpp/src/utils/ThreadPool.h +++ b/cpp/src/utils/ThreadPool.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/utils/TimeRecorder.cpp b/cpp/src/utils/TimeRecorder.cpp index ee33680f54..2aef8c9a67 100644 --- a/cpp/src/utils/TimeRecorder.cpp +++ b/cpp/src/utils/TimeRecorder.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "TimeRecorder.h" #include "utils/Log.h" diff --git a/cpp/src/utils/TimeRecorder.h b/cpp/src/utils/TimeRecorder.h index 1656e0d77f..7f77b0665c 100644 --- a/cpp/src/utils/TimeRecorder.h +++ b/cpp/src/utils/TimeRecorder.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index f5196a1840..2572cbd8bd 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #include "db/engine/ExecutionEngine.h" #include "ValidationUtil.h" #include "Log.h" diff --git a/cpp/src/utils/ValidationUtil.h b/cpp/src/utils/ValidationUtil.h index 2817716c10..ed443e9165 100644 --- a/cpp/src/utils/ValidationUtil.h +++ b/cpp/src/utils/ValidationUtil.h @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include "db/meta/MetaTypes.h" diff --git a/cpp/src/wrapper/gpu/Arithmetic.h b/cpp/src/wrapper/gpu/Arithmetic.h index 1b6d7a98c1..c6ab7a2bc8 100644 --- a/cpp/src/wrapper/gpu/Arithmetic.h +++ b/cpp/src/wrapper/gpu/Arithmetic.h @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #include diff --git a/cpp/src/wrapper/gpu/Topk.cu b/cpp/src/wrapper/gpu/Topk.cu index 0c857ff683..423c8806c4 100644 --- a/cpp/src/wrapper/gpu/Topk.cu +++ b/cpp/src/wrapper/gpu/Topk.cu @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "faiss/FaissAssert.h" #include "faiss/gpu/utils/Limits.cuh" diff --git a/cpp/src/wrapper/gpu/Topk.h b/cpp/src/wrapper/gpu/Topk.h index 62d5756b4c..d8dc69fd09 100644 --- a/cpp/src/wrapper/gpu/Topk.h +++ b/cpp/src/wrapper/gpu/Topk.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/wrapper/knowhere/KnowhereResource.cpp b/cpp/src/wrapper/knowhere/KnowhereResource.cpp index d5a9c2f6dc..afbeb81816 100644 --- a/cpp/src/wrapper/knowhere/KnowhereResource.cpp +++ b/cpp/src/wrapper/knowhere/KnowhereResource.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "KnowhereResource.h" #include "server/ServerConfig.h" diff --git a/cpp/src/wrapper/knowhere/KnowhereResource.h b/cpp/src/wrapper/knowhere/KnowhereResource.h index b7b320d9cc..c14234eaed 100644 --- a/cpp/src/wrapper/knowhere/KnowhereResource.h +++ b/cpp/src/wrapper/knowhere/KnowhereResource.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/wrapper/knowhere/data_transfer.cpp b/cpp/src/wrapper/knowhere/data_transfer.cpp index 583a44ee29..ad29e6cd7d 100644 --- a/cpp/src/wrapper/knowhere/data_transfer.cpp +++ b/cpp/src/wrapper/knowhere/data_transfer.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "data_transfer.h" diff --git a/cpp/src/wrapper/knowhere/data_transfer.h b/cpp/src/wrapper/knowhere/data_transfer.h index 46de4ff21f..7f41af98db 100644 --- a/cpp/src/wrapper/knowhere/data_transfer.h +++ b/cpp/src/wrapper/knowhere/data_transfer.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/wrapper/knowhere/vec_impl.cpp b/cpp/src/wrapper/knowhere/vec_impl.cpp index c012ea60fb..8fd48d2928 100644 --- a/cpp/src/wrapper/knowhere/vec_impl.cpp +++ b/cpp/src/wrapper/knowhere/vec_impl.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include "knowhere/index/vector_index/idmap.h" diff --git a/cpp/src/wrapper/knowhere/vec_impl.h b/cpp/src/wrapper/knowhere/vec_impl.h index e98d030a56..b10dc6b711 100644 --- a/cpp/src/wrapper/knowhere/vec_impl.h +++ b/cpp/src/wrapper/knowhere/vec_impl.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/wrapper/knowhere/vec_index.cpp b/cpp/src/wrapper/knowhere/vec_index.cpp index 2c368541cc..d5d14d29de 100644 --- a/cpp/src/wrapper/knowhere/vec_index.cpp +++ b/cpp/src/wrapper/knowhere/vec_index.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "knowhere/index/vector_index/ivf.h" #include "knowhere/index/vector_index/idmap.h" #include "knowhere/index/vector_index/gpu_ivf.h" diff --git a/cpp/src/wrapper/knowhere/vec_index.h b/cpp/src/wrapper/knowhere/vec_index.h index a5cdcabf04..3ffa0eeec8 100644 --- a/cpp/src/wrapper/knowhere/vec_index.h +++ b/cpp/src/wrapper/knowhere/vec_index.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/src/wrapper/knowhere/wrapper_log.h b/cpp/src/wrapper/knowhere/wrapper_log.h index 7cde925292..1485222e8c 100644 --- a/cpp/src/wrapper/knowhere/wrapper_log.h +++ b/cpp/src/wrapper/knowhere/wrapper_log.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 69ea8f4ea1..03729071b9 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -1,8 +1,22 @@ #------------------------------------------------------------------------------- -# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -# Unauthorized copying of this file, via any medium is strictly prohibited. -# Proprietary and confidential. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. #------------------------------------------------------------------------------- + link_directories( "${CMAKE_BINARY_DIR}/lib" ) diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 8c203f329f..00b6f64dbf 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -1,8 +1,22 @@ #------------------------------------------------------------------------------- -# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -# Unauthorized copying of this file, via any medium is strictly prohibited. -# Proprietary and confidential. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. #------------------------------------------------------------------------------- + aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 05e7ba4def..372dd336d6 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "utils.h" #include "db/DB.h" #include "db/DBImpl.h" diff --git a/cpp/unittest/db/engine_test.cpp b/cpp/unittest/db/engine_test.cpp index 5cdad9f121..e06e9038c8 100644 --- a/cpp/unittest/db/engine_test.cpp +++ b/cpp/unittest/db/engine_test.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include #include diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 2760711df3..73f6d89f5e 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #include "gtest/gtest.h" #include "db/insert/VectorSource.h" diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 2f7a8394e5..62271b9a3b 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include #include "utils/easylogging++.h" diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index d9c2cbf03e..3fa6cb5268 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include #include "utils/easylogging++.h" diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp index 396377ab4d..35f086b033 100644 --- a/cpp/unittest/db/mysql_db_test.cpp +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "utils.h" #include "db/DB.h" #include "db/DBImpl.h" diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index 5d6da9c3fb..ab94c817fe 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include #include "utils/easylogging++.h" diff --git a/cpp/unittest/db/scheduler_test.cpp b/cpp/unittest/db/scheduler_test.cpp index fa941cbcb5..e61741a8fb 100644 --- a/cpp/unittest/db/scheduler_test.cpp +++ b/cpp/unittest/db/scheduler_test.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include #include "utils/easylogging++.h" diff --git a/cpp/unittest/db/search_test.cpp b/cpp/unittest/db/search_test.cpp index 2eb3300bc1..4bbc86be2d 100644 --- a/cpp/unittest/db/search_test.cpp +++ b/cpp/unittest/db/search_test.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "db/scheduler/task/SearchTask.h" #include "server/ServerConfig.h" #include "utils/TimeRecorder.h" diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 47f21d5d7a..4bc9699e05 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include diff --git a/cpp/unittest/db/utils.h b/cpp/unittest/db/utils.h index 5d8cca3bd1..c1dc730089 100644 --- a/cpp/unittest/db/utils.h +++ b/cpp/unittest/db/utils.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/unittest/knowhere/CMakeLists.txt b/cpp/unittest/knowhere/CMakeLists.txt index 59fff95509..27e2612ae7 100644 --- a/cpp/unittest/knowhere/CMakeLists.txt +++ b/cpp/unittest/knowhere/CMakeLists.txt @@ -1,3 +1,22 @@ +#------------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +#------------------------------------------------------------------------------- + include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") diff --git a/cpp/unittest/knowhere/knowhere_test.cpp b/cpp/unittest/knowhere/knowhere_test.cpp index 6e5f0fc686..6d8f57f322 100644 --- a/cpp/unittest/knowhere/knowhere_test.cpp +++ b/cpp/unittest/knowhere/knowhere_test.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include "utils/easylogging++.h" diff --git a/cpp/unittest/knowhere/utils.cpp b/cpp/unittest/knowhere/utils.cpp index e07bd2cd42..f404f000e9 100644 --- a/cpp/unittest/knowhere/utils.cpp +++ b/cpp/unittest/knowhere/utils.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include diff --git a/cpp/unittest/knowhere/utils.h b/cpp/unittest/knowhere/utils.h index 69653b6470..7a73d84713 100644 --- a/cpp/unittest/knowhere/utils.h +++ b/cpp/unittest/knowhere/utils.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/unittest/main.cpp b/cpp/unittest/main.cpp index 086b715aa9..b0c2be6d5a 100644 --- a/cpp/unittest/main.cpp +++ b/cpp/unittest/main.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include #include "utils/easylogging++.h" diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index b3ac068757..07f27e1e67 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -1,3 +1,22 @@ +#------------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +#------------------------------------------------------------------------------- + include_directories(../../src) diff --git a/cpp/unittest/metrics/metricbase_test.cpp b/cpp/unittest/metrics/metricbase_test.cpp index 10087b370a..31766403be 100644 --- a/cpp/unittest/metrics/metricbase_test.cpp +++ b/cpp/unittest/metrics/metricbase_test.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "metrics/Metrics.h" diff --git a/cpp/unittest/metrics/metrics_test.cpp b/cpp/unittest/metrics/metrics_test.cpp index dc9469a18c..2bd06ed091 100644 --- a/cpp/unittest/metrics/metrics_test.cpp +++ b/cpp/unittest/metrics/metrics_test.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include #include diff --git a/cpp/unittest/metrics/prometheus_test.cpp b/cpp/unittest/metrics/prometheus_test.cpp index 07aab27f48..b4fa84401a 100644 --- a/cpp/unittest/metrics/prometheus_test.cpp +++ b/cpp/unittest/metrics/prometheus_test.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "metrics/PrometheusMetrics.h" diff --git a/cpp/unittest/metrics/utils.cpp b/cpp/unittest/metrics/utils.cpp index 307572e9e8..4dc370c952 100644 --- a/cpp/unittest/metrics/utils.cpp +++ b/cpp/unittest/metrics/utils.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include diff --git a/cpp/unittest/metrics/utils.h b/cpp/unittest/metrics/utils.h index 73e7310094..ef25654ed0 100644 --- a/cpp/unittest/metrics/utils.h +++ b/cpp/unittest/metrics/utils.h @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once diff --git a/cpp/unittest/scheduler/CMakeLists.txt b/cpp/unittest/scheduler/CMakeLists.txt index f09f8639a1..9ada727f8e 100644 --- a/cpp/unittest/scheduler/CMakeLists.txt +++ b/cpp/unittest/scheduler/CMakeLists.txt @@ -1,8 +1,22 @@ #------------------------------------------------------------------------------- -# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -# Unauthorized copying of this file, via any medium is strictly prohibited. -# Proprietary and confidential. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. #------------------------------------------------------------------------------- + aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) diff --git a/cpp/unittest/scheduler/algorithm_test.cpp b/cpp/unittest/scheduler/algorithm_test.cpp index eeb7fc64f3..5272edf0d7 100644 --- a/cpp/unittest/scheduler/algorithm_test.cpp +++ b/cpp/unittest/scheduler/algorithm_test.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include diff --git a/cpp/unittest/scheduler/event_test.cpp b/cpp/unittest/scheduler/event_test.cpp index 0c5ec21037..895aabe19d 100644 --- a/cpp/unittest/scheduler/event_test.cpp +++ b/cpp/unittest/scheduler/event_test.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include diff --git a/cpp/unittest/scheduler/node_test.cpp b/cpp/unittest/scheduler/node_test.cpp index dfe0398d22..36dd10b73d 100644 --- a/cpp/unittest/scheduler/node_test.cpp +++ b/cpp/unittest/scheduler/node_test.cpp @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #include "scheduler/resource/Node.h" #include diff --git a/cpp/unittest/scheduler/normal_test.cpp b/cpp/unittest/scheduler/normal_test.cpp index 8c096c4df8..5ddfd49b55 100644 --- a/cpp/unittest/scheduler/normal_test.cpp +++ b/cpp/unittest/scheduler/normal_test.cpp @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #include "scheduler/ResourceFactory.h" #include "scheduler/ResourceMgr.h" #include "scheduler/Scheduler.h" diff --git a/cpp/unittest/scheduler/resource_factory_test.cpp b/cpp/unittest/scheduler/resource_factory_test.cpp index 82b05f6215..1bf39f669f 100644 --- a/cpp/unittest/scheduler/resource_factory_test.cpp +++ b/cpp/unittest/scheduler/resource_factory_test.cpp @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #include "scheduler/ResourceFactory.h" #include diff --git a/cpp/unittest/scheduler/resource_mgr_test.cpp b/cpp/unittest/scheduler/resource_mgr_test.cpp index 180072d87e..56db020058 100644 --- a/cpp/unittest/scheduler/resource_mgr_test.cpp +++ b/cpp/unittest/scheduler/resource_mgr_test.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "scheduler/resource/CpuResource.h" #include "scheduler/resource/GpuResource.h" diff --git a/cpp/unittest/scheduler/resource_test.cpp b/cpp/unittest/scheduler/resource_test.cpp index 0eb2de8e33..b5a83ac776 100644 --- a/cpp/unittest/scheduler/resource_test.cpp +++ b/cpp/unittest/scheduler/resource_test.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "scheduler/resource/Resource.h" #include "scheduler/resource/DiskResource.h" diff --git a/cpp/unittest/scheduler/schedinst_test.cpp b/cpp/unittest/scheduler/schedinst_test.cpp index b824c20022..27b6a17bbb 100644 --- a/cpp/unittest/scheduler/schedinst_test.cpp +++ b/cpp/unittest/scheduler/schedinst_test.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "scheduler/SchedInst.h" #include "server/ServerConfig.h" diff --git a/cpp/unittest/scheduler/scheduler_test.cpp b/cpp/unittest/scheduler/scheduler_test.cpp index 5a1824a185..6f680f4a60 100644 --- a/cpp/unittest/scheduler/scheduler_test.cpp +++ b/cpp/unittest/scheduler/scheduler_test.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "scheduler/Scheduler.h" #include #include diff --git a/cpp/unittest/scheduler/task_test.cpp b/cpp/unittest/scheduler/task_test.cpp index 04071dc0c9..8a8b455ae5 100644 --- a/cpp/unittest/scheduler/task_test.cpp +++ b/cpp/unittest/scheduler/task_test.cpp @@ -1,8 +1,20 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include "scheduler/task/SearchTask.h" #include diff --git a/cpp/unittest/scheduler/tasktable_test.cpp b/cpp/unittest/scheduler/tasktable_test.cpp index f13ccaecf9..9e1f9b2c3a 100644 --- a/cpp/unittest/scheduler/tasktable_test.cpp +++ b/cpp/unittest/scheduler/tasktable_test.cpp @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #include "scheduler/TaskTable.h" #include "scheduler/task/TestTask.h" #include diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index 48e37357b1..6bbe2183ce 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -1,8 +1,22 @@ #------------------------------------------------------------------------------- -# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -# Unauthorized copying of this file, via any medium is strictly prohibited. -# Proprietary and confidential. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. #------------------------------------------------------------------------------- + include_directories(${MILVUS_ENGINE_SRC}/) include_directories(/usr/include) diff --git a/cpp/unittest/server/cache_test.cpp b/cpp/unittest/server/cache_test.cpp index 8293805209..5bb1fb8612 100644 --- a/cpp/unittest/server/cache_test.cpp +++ b/cpp/unittest/server/cache_test.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include "cache/CpuCacheMgr.h" #include "cache/GpuCacheMgr.h" diff --git a/cpp/unittest/server/config_test.cpp b/cpp/unittest/server/config_test.cpp index ddfca182f5..d53d674b37 100644 --- a/cpp/unittest/server/config_test.cpp +++ b/cpp/unittest/server/config_test.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include diff --git a/cpp/unittest/server/rpc_test.cpp b/cpp/unittest/server/rpc_test.cpp index ea6fc3f94c..09a8a1e461 100644 --- a/cpp/unittest/server/rpc_test.cpp +++ b/cpp/unittest/server/rpc_test.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include #include diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index a600ace499..39a96115a5 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -1,8 +1,20 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include #include "utils/easylogging++.h" diff --git a/cpp/unittest/storage/CMakeLists.txt b/cpp/unittest/storage/CMakeLists.txt index 6a8c739431..79819b2deb 100644 --- a/cpp/unittest/storage/CMakeLists.txt +++ b/cpp/unittest/storage/CMakeLists.txt @@ -1,8 +1,22 @@ #------------------------------------------------------------------------------- -# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -# Unauthorized copying of this file, via any medium is strictly prohibited. -# Proprietary and confidential. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. #------------------------------------------------------------------------------- + aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 s3_client_src) set(util_files diff --git a/cpp/unittest/storage/S3ClientTest.cpp b/cpp/unittest/storage/S3ClientTest.cpp index 5147f701ae..540d9ab75e 100644 --- a/cpp/unittest/storage/S3ClientTest.cpp +++ b/cpp/unittest/storage/S3ClientTest.cpp @@ -1,8 +1,20 @@ -////////////////////////////////////////////////////////////////////////////////// -//// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -//// Unauthorized copying of this file, via any medium is strictly prohibited. -//// Proprietary and confidential. -////////////////////////////////////////////////////////////////////////////////// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + // //#include "storage/IStorage.h" //#include "storage/s3/S3ClientWrapper.h" From af9c167ed01ece054476dff60eca5191824cae9e Mon Sep 17 00:00:00 2001 From: wxyu Date: Tue, 17 Sep 2019 10:54:51 +0800 Subject: [PATCH 007/217] MS-556 Add Job Definition in Scheduler Former-commit-id: 87787757226f3b397d65125a687442222a5d3532 --- cpp/CHANGELOG.md | 2 + cpp/src/CMakeLists.txt | 2 + cpp/src/db/meta/MetaTypes.h | 9 +-- cpp/src/scheduler/job/DeleteJob.cpp | 40 +++++++++++++ cpp/src/scheduler/job/DeleteJob.h | 66 +++++++++++++++++++++ cpp/src/scheduler/job/Job.h | 57 ++++++++++++++++++ cpp/src/scheduler/job/SearchJob.cpp | 61 +++++++++++++++++++ cpp/src/scheduler/job/SearchJob.h | 90 +++++++++++++++++++++++++++++ 8 files changed, 323 insertions(+), 4 deletions(-) create mode 100644 cpp/src/scheduler/job/DeleteJob.cpp create mode 100644 cpp/src/scheduler/job/DeleteJob.h create mode 100644 cpp/src/scheduler/job/Job.h create mode 100644 cpp/src/scheduler/job/SearchJob.cpp create mode 100644 cpp/src/scheduler/job/SearchJob.h diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 78db90639b..0d2a90d19a 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -7,6 +7,8 @@ Please mark all change in change log and use the ticket from JIRA. ## Bug ## Improvement +- MS-556 - Add Job Definition in Scheduler + ## New Feature diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 387693a30c..b44af1dea0 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -25,12 +25,14 @@ aux_source_directory(wrapper/knowhere knowhere_files) aux_source_directory(scheduler/action scheduler_action_files) aux_source_directory(scheduler/event scheduler_event_files) +aux_source_directory(scheduler/job scheduler_job_files) aux_source_directory(scheduler/resource scheduler_resource_files) aux_source_directory(scheduler/task scheduler_task_files) aux_source_directory(scheduler scheduler_root_files) set(scheduler_srcs ${scheduler_action_files} ${scheduler_event_files} + ${scheduler_job_files} ${scheduler_resource_files} ${scheduler_task_files} ${scheduler_root_files} diff --git a/cpp/src/db/meta/MetaTypes.h b/cpp/src/db/meta/MetaTypes.h index e31be40ddc..252b557346 100644 --- a/cpp/src/db/meta/MetaTypes.h +++ b/cpp/src/db/meta/MetaTypes.h @@ -25,9 +25,9 @@ constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB; constexpr int64_t FLAG_MASK_NO_USERID = 0x1; constexpr int64_t FLAG_MASK_HAS_USERID = 0x1<<1; -typedef int DateT; +using DateT = int ; const DateT EmptyDate = -1; -typedef std::vector DatesT; +using DatesT = std::vector; struct TableSchema { typedef enum { @@ -76,8 +76,9 @@ struct TableFileSchema { int32_t metric_type_ = DEFAULT_METRIC_TYPE; //not persist to meta }; // TableFileSchema -typedef std::vector TableFilesSchema; -typedef std::map DatePartionedTableFilesSchema; +using TableFileSchemaPtr = std::shared_ptr; +using TableFilesSchema = std::vector; +using DatePartionedTableFilesSchema = std::map; } // namespace meta } // namespace engine diff --git a/cpp/src/scheduler/job/DeleteJob.cpp b/cpp/src/scheduler/job/DeleteJob.cpp new file mode 100644 index 0000000000..ababfba98d --- /dev/null +++ b/cpp/src/scheduler/job/DeleteJob.cpp @@ -0,0 +1,40 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ + +#include "DeleteJob.h" + + +namespace zilliz { +namespace milvus { +namespace scheduler { + +DeleteJob::DeleteJob(JobId id, + std::string table_id, + engine::meta::MetaPtr meta_ptr, + uint64_t num_resource) + : Job(id, JobType::DELETE), + table_id_(std::move(table_id)), + meta_ptr_(std::move(meta_ptr)), + num_resource_(num_resource) {} + +void DeleteJob::WaitAndDelete() { + std::unique_lock lock(mutex_); + cv_.wait(lock, [&] { return done_resource == num_resource_; }); + meta_ptr_->DeleteTableFiles(table_id_); +} + +void DeleteJob::ResourceDone() { + { + std::lock_guard lock(mutex_); + ++done_resource; + } + cv_.notify_one(); +} + +} +} +} + diff --git a/cpp/src/scheduler/job/DeleteJob.h b/cpp/src/scheduler/job/DeleteJob.h new file mode 100644 index 0000000000..214a77a574 --- /dev/null +++ b/cpp/src/scheduler/job/DeleteJob.h @@ -0,0 +1,66 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Job.h" +#include "db/meta/Meta.h" + + +namespace zilliz { +namespace milvus { +namespace scheduler { + +class DeleteJob : public Job { +public: + DeleteJob(JobId id, + std::string table_id, + engine::meta::MetaPtr meta_ptr, + uint64_t num_resource); + +public: + void + WaitAndDelete(); + + void + ResourceDone(); + +public: + std::string + table_id() const { + return table_id_; + } + + engine::meta::MetaPtr + meta() const { + return meta_ptr_; + } + +private: + std::string table_id_; + engine::meta::MetaPtr meta_ptr_; + + uint64_t num_resource_ = 0; + uint64_t done_resource = 0; + std::mutex mutex_; + std::condition_variable cv_; +}; + +using DeleteJobPtr = std::shared_ptr; + +} +} +} + diff --git a/cpp/src/scheduler/job/Job.h b/cpp/src/scheduler/job/Job.h new file mode 100644 index 0000000000..4d515c3a4e --- /dev/null +++ b/cpp/src/scheduler/job/Job.h @@ -0,0 +1,57 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace zilliz { +namespace milvus { +namespace scheduler { + +enum class JobType { + INVALID, + SEARCH, + DELETE, + BUILD, +}; + +using JobId = std::uint64_t; + +class Job { +public: + inline JobId + id() const { + return id_; + } + + inline JobType + type() const { + return type_; + } + +protected: + Job(JobId id, JobType type) : id_(id), type_(type) {} + +private: + JobId id_; + JobType type_; +}; + +using JobPtr = std::shared_ptr; + +} +} +} + diff --git a/cpp/src/scheduler/job/SearchJob.cpp b/cpp/src/scheduler/job/SearchJob.cpp new file mode 100644 index 0000000000..b1ee993360 --- /dev/null +++ b/cpp/src/scheduler/job/SearchJob.cpp @@ -0,0 +1,61 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ + +#include "utils/Log.h" + +#include "SearchJob.h" + + +namespace zilliz { +namespace milvus { +namespace scheduler { + +SearchJob::SearchJob(zilliz::milvus::scheduler::JobId id, + uint64_t topk, + uint64_t nq, + uint64_t nprobe, + const float *vectors) : Job(id, JobType::SEARCH) {} + +bool +SearchJob::AddIndexFile(const TableFileSchemaPtr &index_file) { + std::unique_lock lock(mutex_); + if (index_file == nullptr || index_files_.find(index_file->id_) != index_files_.end()) { + return false; + } + + SERVER_LOG_DEBUG << "SearchJob " << id() << " add index file: " << index_file->id_; + + index_files_[index_file->id_] = index_file; + return true; +} + + +void +SearchJob::WaitResult() { + std::unique_lock lock(mutex_); + cv_.wait(lock, [this] { return index_files_.empty(); }); + SERVER_LOG_DEBUG << "SearchJob " << id() << " all done"; +} + +void +SearchJob::SearchDone(size_t index_id) { + std::unique_lock lock(mutex_); + index_files_.erase(index_id); + cv_.notify_all(); + SERVER_LOG_DEBUG << "SearchJob " << id() << " finish index file: " << index_id; +} + +ResultSet & +SearchJob::GetResult() { + return result_; +} + + +} +} +} + + diff --git a/cpp/src/scheduler/job/SearchJob.h b/cpp/src/scheduler/job/SearchJob.h new file mode 100644 index 0000000000..aebddb0dde --- /dev/null +++ b/cpp/src/scheduler/job/SearchJob.h @@ -0,0 +1,90 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Job.h" +#include "db/meta/MetaTypes.h" + + +namespace zilliz { +namespace milvus { +namespace scheduler { + + +using engine::meta::TableFileSchemaPtr; + +using Id2IndexMap = std::unordered_map; +using Id2DistanceMap = std::vector>; +using ResultSet = std::vector; + +class SearchJob : public Job { +public: + SearchJob(JobId id, uint64_t topk, uint64_t nq, uint64_t nprobe, const float *vectors); + +public: + bool + AddIndexFile(const TableFileSchemaPtr &index_file); + + void + WaitResult(); + + void + SearchDone(size_t index_id); + + ResultSet & + GetResult(); + +public: + uint64_t + topk() const { + return topk_; + } + + uint64_t + nq() const { + return nq_; + } + + uint64_t + nprobe() const { + return nprobe_; + } + const float * + vectors() const { + return vectors_; + } + +private: + uint64_t topk_ = 0; + uint64_t nq_ = 0; + uint64_t nprobe_ = 0; + // TODO: smart pointer + const float *vectors_ = nullptr; + + Id2IndexMap index_files_; + // TODO: column-base better ? + ResultSet result_; + + std::mutex mutex_; + std::condition_variable cv_; +}; + +using SearchJobPtr = std::shared_ptr; + +} +} +} + From 63bd4cca5c333350fb1af38dbce7bcb85fafcdc2 Mon Sep 17 00:00:00 2001 From: starlord Date: Tue, 17 Sep 2019 10:57:41 +0800 Subject: [PATCH 008/217] merge Log.h Former-commit-id: 1204cfbfddee0b863abf10874ef97f0581c36a9a --- cpp/src/db/DBFactory.cpp | 2 +- cpp/src/db/DBImpl.cpp | 23 ++++++----- cpp/src/db/DBImpl.h | 4 +- cpp/src/db/Log.h | 26 ------------- cpp/src/db/Options.cpp | 9 ++--- cpp/src/db/Utils.cpp | 2 +- cpp/src/db/engine/EngineFactory.cpp | 2 +- cpp/src/db/engine/ExecutionEngineImpl.cpp | 17 ++++----- cpp/src/db/insert/MemManagerImpl.cpp | 2 +- cpp/src/db/insert/MemMenagerFactory.cpp | 5 +-- cpp/src/db/insert/MemTable.cpp | 2 +- cpp/src/db/insert/MemTableFile.cpp | 2 +- cpp/src/db/insert/VectorSource.cpp | 2 +- cpp/src/db/meta/MetaFactory.cpp | 5 +-- cpp/src/db/meta/MySQLConnectionPool.h | 2 +- cpp/src/db/meta/MySQLMetaImpl.cpp | 2 +- cpp/src/db/meta/SqliteMetaImpl.cpp | 2 +- cpp/src/db/scheduler/TaskDispatchStrategy.cpp | 2 +- cpp/src/db/scheduler/task/IndexLoadTask.cpp | 2 +- cpp/src/db/scheduler/task/SearchTask.cpp | 2 +- cpp/src/scheduler/ResourceMgr.cpp | 2 +- cpp/src/scheduler/task/SearchTask.cpp | 13 +++---- cpp/src/{db => utils}/Exception.h | 2 - cpp/src/utils/Log.h | 38 +++++++++++-------- cpp/src/utils/ThreadPool.h | 2 - cpp/src/utils/TimeRecorder.cpp | 2 - cpp/src/utils/TimeRecorder.h | 2 - cpp/src/wrapper/knowhere/vec_impl.cpp | 3 +- cpp/src/wrapper/knowhere/vec_index.cpp | 2 +- cpp/src/wrapper/knowhere/wrapper_log.h | 28 -------------- cpp/unittest/db/misc_test.cpp | 15 ++++---- cpp/unittest/db/search_test.cpp | 4 +- cpp/unittest/server/util_test.cpp | 2 +- 33 files changed, 83 insertions(+), 147 deletions(-) delete mode 100644 cpp/src/db/Log.h rename cpp/src/{db => utils}/Exception.h (97%) delete mode 100644 cpp/src/wrapper/knowhere/wrapper_log.h diff --git a/cpp/src/db/DBFactory.cpp b/cpp/src/db/DBFactory.cpp index b04067c38a..c62310a0b9 100644 --- a/cpp/src/db/DBFactory.cpp +++ b/cpp/src/db/DBFactory.cpp @@ -6,7 +6,7 @@ #include "DBFactory.h" #include "DBImpl.h" -#include "Exception.h" +#include "utils/Exception.h" #include "meta/MetaFactory.h" #include "meta/SqliteMetaImpl.h" #include "meta/MySQLMetaImpl.h" diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 80b1994778..c82c8b7a95 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -4,28 +4,27 @@ * Proprietary and confidential. ******************************************************************************/ #include "DBImpl.h" -#include "src/db/meta/SqliteMetaImpl.h" -#include "Log.h" -#include "Utils.h" +#include "cache/CpuCacheMgr.h" +#include "cache/GpuCacheMgr.h" #include "engine/EngineFactory.h" #include "insert/MemMenagerFactory.h" +#include "meta/SqliteMetaImpl.h" #include "meta/MetaFactory.h" +#include "meta/MetaConsts.h" #include "metrics/Metrics.h" #include "scheduler/TaskScheduler.h" - #include "scheduler/context/DeleteContext.h" +#include "scheduler/SchedInst.h" #include "utils/TimeRecorder.h" -#include "meta/MetaConsts.h" +#include "utils/Log.h" +#include "Utils.h" #include #include #include #include #include -#include #include -#include "scheduler/SchedInst.h" -#include namespace zilliz { namespace milvus { @@ -409,7 +408,7 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch const meta::DatesT& dates, QueryResults& results) { server::CollectQueryMetrics metrics(nq); - server::TimeRecorder rc(""); + TimeRecorder rc(""); //step 1: get files to search ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size() << " date range count: " << dates.size(); @@ -432,9 +431,9 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch double load_cost = context->LoadCost(); double search_cost = context->SearchCost(); double reduce_cost = context->ReduceCost(); - std::string load_info = server::TimeRecorder::GetTimeSpanStr(load_cost); - std::string search_info = server::TimeRecorder::GetTimeSpanStr(search_cost); - std::string reduce_info = server::TimeRecorder::GetTimeSpanStr(reduce_cost); + std::string load_info = TimeRecorder::GetTimeSpanStr(load_cost); + std::string search_info = TimeRecorder::GetTimeSpanStr(search_cost); + std::string reduce_info = TimeRecorder::GetTimeSpanStr(reduce_cost); if(search_cost > 0.0 || reduce_cost > 0.0) { double total_cost = load_cost + search_cost + reduce_cost; double load_percent = load_cost/total_cost; diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index e4bfbd5d34..5591d062fe 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -128,11 +128,11 @@ class DBImpl : public DB { MemManagerPtr mem_mgr_; std::mutex mem_serialize_mutex_; - server::ThreadPool compact_thread_pool_; + ThreadPool compact_thread_pool_; std::list> compact_thread_results_; std::set compact_table_ids_; - server::ThreadPool index_thread_pool_; + ThreadPool index_thread_pool_; std::list> index_thread_results_; std::mutex build_index_mutex_; diff --git a/cpp/src/db/Log.h b/cpp/src/db/Log.h deleted file mode 100644 index f36d861c3e..0000000000 --- a/cpp/src/db/Log.h +++ /dev/null @@ -1,26 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#pragma once - -#include "utils/easylogging++.h" - -namespace zilliz { -namespace milvus { -namespace engine { - -#define ENGINE_DOMAIN_NAME "[ENGINE] " -#define ENGINE_ERROR_TEXT "ENGINE Error:" - -#define ENGINE_LOG_TRACE LOG(TRACE) << ENGINE_DOMAIN_NAME -#define ENGINE_LOG_DEBUG LOG(DEBUG) << ENGINE_DOMAIN_NAME -#define ENGINE_LOG_INFO LOG(INFO) << ENGINE_DOMAIN_NAME -#define ENGINE_LOG_WARNING LOG(WARNING) << ENGINE_DOMAIN_NAME -#define ENGINE_LOG_ERROR LOG(ERROR) << ENGINE_DOMAIN_NAME -#define ENGINE_LOG_FATAL LOG(FATAL) << ENGINE_DOMAIN_NAME - -} // namespace sql -} // namespace zilliz -} // namespace server diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index ecebd30bdb..06505aa2fd 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -3,15 +3,14 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. * Proprietary and confidential. ******************************************************************************/ +#include "Options.h" +#include "utils/Exception.h" +#include "utils/easylogging++.h" + #include #include -#include "utils/easylogging++.h" #include -#include "Options.h" -#include "db/meta/SqliteMetaImpl.h" -#include "Exception.h" - namespace zilliz { namespace milvus { namespace engine { diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index 2c9173f6a7..e7cd7ae86d 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -5,7 +5,7 @@ ******************************************************************************/ #include "Utils.h" #include "utils/CommonUtil.h" -#include "Log.h" +#include "utils/Log.h" #include #include diff --git a/cpp/src/db/engine/EngineFactory.cpp b/cpp/src/db/engine/EngineFactory.cpp index a326d6a2c6..77a85d2aa4 100644 --- a/cpp/src/db/engine/EngineFactory.cpp +++ b/cpp/src/db/engine/EngineFactory.cpp @@ -5,7 +5,7 @@ ******************************************************************************/ #include "EngineFactory.h" #include "ExecutionEngineImpl.h" -#include "db/Log.h" +#include "utils/Log.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 8a738347d2..a8fa3aab8e 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -3,20 +3,19 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. * Proprietary and confidential. ******************************************************************************/ -#include -#include "src/cache/GpuCacheMgr.h" - -#include "src/metrics/Metrics.h" -#include "db/Log.h" -#include "utils/CommonUtil.h" - -#include "src/cache/CpuCacheMgr.h" #include "ExecutionEngineImpl.h" +#include "cache/GpuCacheMgr.h" +#include "cache/CpuCacheMgr.h" +#include "metrics/Metrics.h" +#include "utils/Log.h" +#include "utils/CommonUtil.h" +#include "utils/Exception.h" + #include "wrapper/knowhere/vec_index.h" #include "wrapper/knowhere/vec_impl.h" #include "knowhere/common/exception.h" -#include "db/Exception.h" +#include namespace zilliz { namespace milvus { diff --git a/cpp/src/db/insert/MemManagerImpl.cpp b/cpp/src/db/insert/MemManagerImpl.cpp index 240b2f2bb0..4988a8dc96 100644 --- a/cpp/src/db/insert/MemManagerImpl.cpp +++ b/cpp/src/db/insert/MemManagerImpl.cpp @@ -1,6 +1,6 @@ #include "MemManagerImpl.h" #include "VectorSource.h" -#include "db/Log.h" +#include "utils/Log.h" #include "db/Constants.h" #include diff --git a/cpp/src/db/insert/MemMenagerFactory.cpp b/cpp/src/db/insert/MemMenagerFactory.cpp index 965ba596bd..be3b7542e5 100644 --- a/cpp/src/db/insert/MemMenagerFactory.cpp +++ b/cpp/src/db/insert/MemMenagerFactory.cpp @@ -3,11 +3,10 @@ // Unauthorized copying of this file, via any medium is strictly prohibited. // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// - #include "MemMenagerFactory.h" #include "MemManagerImpl.h" -#include "db/Log.h" -#include "db/Exception.h" +#include "utils/Log.h" +#include "utils/Exception.h" #include #include diff --git a/cpp/src/db/insert/MemTable.cpp b/cpp/src/db/insert/MemTable.cpp index fe6cdf0967..8f97c1a64f 100644 --- a/cpp/src/db/insert/MemTable.cpp +++ b/cpp/src/db/insert/MemTable.cpp @@ -1,5 +1,5 @@ #include "MemTable.h" -#include "db/Log.h" +#include "utils/Log.h" namespace zilliz { diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index 946b658b4b..b7aea99d65 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -1,8 +1,8 @@ #include "MemTableFile.h" #include "db/Constants.h" -#include "db/Log.h" #include "db/engine/EngineFactory.h" #include "metrics/Metrics.h" +#include "utils/Log.h" #include diff --git a/cpp/src/db/insert/VectorSource.cpp b/cpp/src/db/insert/VectorSource.cpp index f7d6a48297..cebe956ec2 100644 --- a/cpp/src/db/insert/VectorSource.cpp +++ b/cpp/src/db/insert/VectorSource.cpp @@ -1,7 +1,7 @@ #include "VectorSource.h" #include "db/engine/ExecutionEngine.h" #include "db/engine/EngineFactory.h" -#include "db/Log.h" +#include "utils/Log.h" #include "metrics/Metrics.h" diff --git a/cpp/src/db/meta/MetaFactory.cpp b/cpp/src/db/meta/MetaFactory.cpp index 9e025658af..0c43cc19fd 100644 --- a/cpp/src/db/meta/MetaFactory.cpp +++ b/cpp/src/db/meta/MetaFactory.cpp @@ -3,12 +3,11 @@ // Unauthorized copying of this file, via any medium is strictly prohibited. // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// - #include "MetaFactory.h" #include "SqliteMetaImpl.h" #include "MySQLMetaImpl.h" -#include "db/Log.h" -#include "db/Exception.h" +#include "utils/Log.h" +#include "utils/Exception.h" #include #include diff --git a/cpp/src/db/meta/MySQLConnectionPool.h b/cpp/src/db/meta/MySQLConnectionPool.h index 9cde818b45..24f1238324 100644 --- a/cpp/src/db/meta/MySQLConnectionPool.h +++ b/cpp/src/db/meta/MySQLConnectionPool.h @@ -4,7 +4,7 @@ #include #include -#include "db/Log.h" +#include "utils/Log.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 6c89471741..86ad635a95 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -6,7 +6,7 @@ #include "MySQLMetaImpl.h" #include "db/IDGenerator.h" #include "db/Utils.h" -#include "db/Log.h" +#include "utils/Log.h" #include "MetaConsts.h" #include "metrics/Metrics.h" diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 38941abf38..59289612e4 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -6,7 +6,7 @@ #include "SqliteMetaImpl.h" #include "db/IDGenerator.h" #include "db/Utils.h" -#include "db/Log.h" +#include "utils/Log.h" #include "MetaConsts.h" #include "metrics/Metrics.h" diff --git a/cpp/src/db/scheduler/TaskDispatchStrategy.cpp b/cpp/src/db/scheduler/TaskDispatchStrategy.cpp index 38c9463117..9f730c0263 100644 --- a/cpp/src/db/scheduler/TaskDispatchStrategy.cpp +++ b/cpp/src/db/scheduler/TaskDispatchStrategy.cpp @@ -10,7 +10,7 @@ #include "task/DeleteTask.h" #include "cache/CpuCacheMgr.h" #include "utils/Error.h" -#include "db/Log.h" +#include "utils/Log.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/db/scheduler/task/IndexLoadTask.cpp b/cpp/src/db/scheduler/task/IndexLoadTask.cpp index 4efb041d61..c2e34727a8 100644 --- a/cpp/src/db/scheduler/task/IndexLoadTask.cpp +++ b/cpp/src/db/scheduler/task/IndexLoadTask.cpp @@ -6,8 +6,8 @@ #include "IndexLoadTask.h" #include "SearchTask.h" -#include "db/Log.h" #include "db/engine/EngineFactory.h" +#include "utils/Log.h" #include "utils/TimeRecorder.h" #include "metrics/Metrics.h" diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index 5117dc948c..93cd4525e2 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -5,7 +5,7 @@ ******************************************************************************/ #include "SearchTask.h" #include "metrics/Metrics.h" -#include "db/Log.h" +#include "utils/Log.h" #include "utils/TimeRecorder.h" #include diff --git a/cpp/src/scheduler/ResourceMgr.cpp b/cpp/src/scheduler/ResourceMgr.cpp index cbbf449763..a592cd6f54 100644 --- a/cpp/src/scheduler/ResourceMgr.cpp +++ b/cpp/src/scheduler/ResourceMgr.cpp @@ -5,7 +5,7 @@ * Proprietary and confidential. ******************************************************************************/ #include "ResourceMgr.h" -#include "db/Log.h" +#include "utils/Log.h" namespace zilliz { diff --git a/cpp/src/scheduler/task/SearchTask.cpp b/cpp/src/scheduler/task/SearchTask.cpp index 46d429ab56..6bbe072ba7 100644 --- a/cpp/src/scheduler/task/SearchTask.cpp +++ b/cpp/src/scheduler/task/SearchTask.cpp @@ -3,12 +3,11 @@ * Unauthorized copying of this file, via any medium is strictly prohibited. * Proprietary and confidential. ******************************************************************************/ - -#include "src/metrics/Metrics.h" -#include "src/utils/TimeRecorder.h" -#include "src/db/engine/EngineFactory.h" -#include "src/db/Log.h" #include "SearchTask.h" +#include "metrics/Metrics.h" +#include "db/engine/EngineFactory.h" +#include "utils/TimeRecorder.h" +#include "utils/Log.h" #include @@ -97,7 +96,7 @@ XSearchTask::XSearchTask(TableFileSchemaPtr file) void XSearchTask::Load(LoadType type, uint8_t device_id) { - server::TimeRecorder rc(""); + TimeRecorder rc(""); Status stat = Status::OK(); std::string error_msg; std::string type_str; @@ -166,7 +165,7 @@ XSearchTask::Execute() { ENGINE_LOG_DEBUG << "Searching in file id:" << index_id_ << " with " << search_contexts_.size() << " tasks"; - server::TimeRecorder rc("DoSearch file id:" + std::to_string(index_id_)); + TimeRecorder rc("DoSearch file id:" + std::to_string(index_id_)); server::CollectDurationMetrics metrics(index_type_); diff --git a/cpp/src/db/Exception.h b/cpp/src/utils/Exception.h similarity index 97% rename from cpp/src/db/Exception.h rename to cpp/src/utils/Exception.h index 2474b70283..b810fb2011 100644 --- a/cpp/src/db/Exception.h +++ b/cpp/src/utils/Exception.h @@ -10,7 +10,6 @@ namespace zilliz { namespace milvus { -namespace engine { class Exception : public std::exception { public: @@ -49,6 +48,5 @@ public: OutOfRangeException(const std::string& message) : Exception(message) {}; }; -} // namespace engine } // namespace milvus } // namespace zilliz diff --git a/cpp/src/utils/Log.h b/cpp/src/utils/Log.h index 204d5812b6..3189a7e205 100644 --- a/cpp/src/utils/Log.h +++ b/cpp/src/utils/Log.h @@ -5,13 +5,12 @@ ******************************************************************************/ #pragma once -#include "Error.h" #include "easylogging++.h" namespace zilliz { namespace milvus { -namespace server { +///////////////////////////////////////////////////////////////////////////////////////////////// #define SERVER_DOMAIN_NAME "[SERVER] " #define SERVER_ERROR_TEXT "SERVER Error:" @@ -22,20 +21,27 @@ namespace server { #define SERVER_LOG_ERROR LOG(ERROR) << SERVER_DOMAIN_NAME #define SERVER_LOG_FATAL LOG(FATAL) << SERVER_DOMAIN_NAME -#define SERVER_ERROR(error) \ - ({ \ - SERVER_LOG_ERROR << SERVER_ERROR_TEXT << error; \ - (error); \ - }) +///////////////////////////////////////////////////////////////////////////////////////////////// +#define ENGINE_DOMAIN_NAME "[ENGINE] " +#define ENGINE_ERROR_TEXT "ENGINE Error:" -#define SERVER_CHECK(func) \ - { \ - zilliz::milvus::server::ServerError error = func; \ - if (error != zilliz::milvus::server::SERVER_SUCCESS) { \ - return SERVER_ERROR(error); \ - } \ - } \ +#define ENGINE_LOG_TRACE LOG(TRACE) << ENGINE_DOMAIN_NAME +#define ENGINE_LOG_DEBUG LOG(DEBUG) << ENGINE_DOMAIN_NAME +#define ENGINE_LOG_INFO LOG(INFO) << ENGINE_DOMAIN_NAME +#define ENGINE_LOG_WARNING LOG(WARNING) << ENGINE_DOMAIN_NAME +#define ENGINE_LOG_ERROR LOG(ERROR) << ENGINE_DOMAIN_NAME +#define ENGINE_LOG_FATAL LOG(FATAL) << ENGINE_DOMAIN_NAME -} // namespace sql +///////////////////////////////////////////////////////////////////////////////////////////////// +#define WRAPPER_DOMAIN_NAME "[WRAPPER] " +#define WRAPPER_ERROR_TEXT "WRAPPER Error:" + +#define WRAPPER_LOG_TRACE LOG(TRACE) << WRAPPER_DOMAIN_NAME +#define WRAPPER_LOG_DEBUG LOG(DEBUG) << WRAPPER_DOMAIN_NAME +#define WRAPPER_LOG_INFO LOG(INFO) << WRAPPER_DOMAIN_NAME +#define WRAPPER_LOG_WARNING LOG(WARNING) << WRAPPER_DOMAIN_NAME +#define WRAPPER_LOG_ERROR LOG(ERROR) << WRAPPER_DOMAIN_NAME +#define WRAPPER_LOG_FATAL LOG(FATAL) << WRAPPER_DOMAIN_NAME + +} // namespace milvus } // namespace zilliz -} // namespace server diff --git a/cpp/src/utils/ThreadPool.h b/cpp/src/utils/ThreadPool.h index cf7cc56c65..8453e53f83 100644 --- a/cpp/src/utils/ThreadPool.h +++ b/cpp/src/utils/ThreadPool.h @@ -20,7 +20,6 @@ namespace zilliz { namespace milvus { -namespace server { class ThreadPool { public: @@ -114,5 +113,4 @@ inline ThreadPool::~ThreadPool() { } } -} diff --git a/cpp/src/utils/TimeRecorder.cpp b/cpp/src/utils/TimeRecorder.cpp index ee33680f54..8647203fdd 100644 --- a/cpp/src/utils/TimeRecorder.cpp +++ b/cpp/src/utils/TimeRecorder.cpp @@ -9,7 +9,6 @@ namespace zilliz { namespace milvus { -namespace server { TimeRecorder::TimeRecorder(const std::string &header, int64_t log_level) : @@ -91,4 +90,3 @@ TimeRecorder::ElapseFromBegin(const std::string &msg) { } } -} diff --git a/cpp/src/utils/TimeRecorder.h b/cpp/src/utils/TimeRecorder.h index 1656e0d77f..7f2baaf751 100644 --- a/cpp/src/utils/TimeRecorder.h +++ b/cpp/src/utils/TimeRecorder.h @@ -11,7 +11,6 @@ namespace zilliz { namespace milvus { -namespace server { class TimeRecorder { using stdclock = std::chrono::high_resolution_clock; @@ -40,4 +39,3 @@ private: } } -} diff --git a/cpp/src/wrapper/knowhere/vec_impl.cpp b/cpp/src/wrapper/knowhere/vec_impl.cpp index c012ea60fb..214ef19ead 100644 --- a/cpp/src/wrapper/knowhere/vec_impl.cpp +++ b/cpp/src/wrapper/knowhere/vec_impl.cpp @@ -4,7 +4,7 @@ // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// -#include +#include "utils/Log.h" #include "knowhere/index/vector_index/idmap.h" #include "knowhere/index/vector_index/gpu_ivf.h" #include "knowhere/common/exception.h" @@ -12,7 +12,6 @@ #include "vec_impl.h" #include "data_transfer.h" -#include "wrapper_log.h" namespace zilliz { diff --git a/cpp/src/wrapper/knowhere/vec_index.cpp b/cpp/src/wrapper/knowhere/vec_index.cpp index 2c368541cc..ee2a21e6f9 100644 --- a/cpp/src/wrapper/knowhere/vec_index.cpp +++ b/cpp/src/wrapper/knowhere/vec_index.cpp @@ -12,7 +12,7 @@ #include "vec_index.h" #include "vec_impl.h" -#include "wrapper_log.h" +#include "utils/Log.h" #include diff --git a/cpp/src/wrapper/knowhere/wrapper_log.h b/cpp/src/wrapper/knowhere/wrapper_log.h deleted file mode 100644 index 7cde925292..0000000000 --- a/cpp/src/wrapper/knowhere/wrapper_log.h +++ /dev/null @@ -1,28 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// - -#pragma once - -#include "utils/easylogging++.h" - -namespace zilliz { -namespace milvus { -namespace engine { - -#define WRAPPER_DOMAIN_NAME "[WRAPPER] " -#define WRAPPER_ERROR_TEXT "WRAPPER Error:" - -#define WRAPPER_LOG_TRACE LOG(TRACE) << WRAPPER_DOMAIN_NAME -#define WRAPPER_LOG_DEBUG LOG(DEBUG) << WRAPPER_DOMAIN_NAME -#define WRAPPER_LOG_INFO LOG(INFO) << WRAPPER_DOMAIN_NAME -#define WRAPPER_LOG_WARNING LOG(WARNING) << WRAPPER_DOMAIN_NAME -#define WRAPPER_LOG_ERROR LOG(ERROR) << WRAPPER_DOMAIN_NAME -#define WRAPPER_LOG_FATAL LOG(FATAL) << WRAPPER_DOMAIN_NAME - -} -} -} - diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index d9c2cbf03e..b7f116a511 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -3,18 +3,17 @@ // Unauthorized copying of this file, via any medium is strictly prohibited. // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// -#include -#include -#include "utils/easylogging++.h" -#include - -#include "db/Exception.h" #include "db/Status.h" #include "db/Options.h" #include "db/meta/SqliteMetaImpl.h" #include "db/engine/EngineFactory.h" #include "db/Utils.h" +#include "utils/Exception.h" +#include "utils/easylogging++.h" +#include +#include +#include #include using namespace zilliz::milvus; @@ -27,11 +26,11 @@ namespace { } TEST(DBMiscTest, EXCEPTION_TEST) { - engine::Exception ex1(""); + Exception ex1(""); std::string what = ex1.what(); ASSERT_FALSE(what.empty()); - engine::OutOfRangeException ex2; + OutOfRangeException ex2; what = ex2.what(); ASSERT_FALSE(what.empty()); } diff --git a/cpp/unittest/db/search_test.cpp b/cpp/unittest/db/search_test.cpp index 2eb3300bc1..621a81cfe4 100644 --- a/cpp/unittest/db/search_test.cpp +++ b/cpp/unittest/db/search_test.cpp @@ -226,7 +226,7 @@ TEST(DBSearchTest, PARALLEL_CLUSTER_TEST) { engine::SearchContext::ResultSet src_result; auto DoCluster = [&](int64_t nq, int64_t topk) { - server::TimeRecorder rc("DoCluster"); + TimeRecorder rc("DoCluster"); src_result.clear(); BuildResult(nq, topk, ascending, target_ids, target_distence); rc.RecordSection("build id/dietance map"); @@ -268,7 +268,7 @@ TEST(DBSearchTest, PARALLEL_TOPK_TEST) { src_result.clear(); insufficient_result.clear(); - server::TimeRecorder rc("DoCluster"); + TimeRecorder rc("DoCluster"); BuildResult(nq, topk, ascending, target_ids, target_distence); auto status = engine::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result); diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index a600ace499..e7048ebbbe 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -166,7 +166,7 @@ TEST(UtilTest, TIMERECORDER_TEST) { if(log_level == 5) { continue; //skip fatal } - server::TimeRecorder rc("time", log_level); + TimeRecorder rc("time", log_level); rc.RecordSection("end"); } } From 909e2a0934a91dc568b6f989311b16a1edd8d723 Mon Sep 17 00:00:00 2001 From: wxyu Date: Tue, 17 Sep 2019 10:59:51 +0800 Subject: [PATCH 009/217] add Apache License Former-commit-id: 8afccd9400189249b2f0c797e6e0f53f74547246 --- cpp/src/scheduler/job/DeleteJob.cpp | 21 ++++++++++++++++----- cpp/src/scheduler/job/DeleteJob.h | 21 ++++++++++++++++----- cpp/src/scheduler/job/Job.h | 21 ++++++++++++++++----- cpp/src/scheduler/job/SearchJob.cpp | 21 ++++++++++++++++----- cpp/src/scheduler/job/SearchJob.h | 21 ++++++++++++++++----- 5 files changed, 80 insertions(+), 25 deletions(-) diff --git a/cpp/src/scheduler/job/DeleteJob.cpp b/cpp/src/scheduler/job/DeleteJob.cpp index ababfba98d..9066afafaa 100644 --- a/cpp/src/scheduler/job/DeleteJob.cpp +++ b/cpp/src/scheduler/job/DeleteJob.cpp @@ -1,8 +1,19 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #include "DeleteJob.h" diff --git a/cpp/src/scheduler/job/DeleteJob.h b/cpp/src/scheduler/job/DeleteJob.h index 214a77a574..7410577563 100644 --- a/cpp/src/scheduler/job/DeleteJob.h +++ b/cpp/src/scheduler/job/DeleteJob.h @@ -1,8 +1,19 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #include #include diff --git a/cpp/src/scheduler/job/Job.h b/cpp/src/scheduler/job/Job.h index 4d515c3a4e..e79dd51e71 100644 --- a/cpp/src/scheduler/job/Job.h +++ b/cpp/src/scheduler/job/Job.h @@ -1,8 +1,19 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #include #include diff --git a/cpp/src/scheduler/job/SearchJob.cpp b/cpp/src/scheduler/job/SearchJob.cpp index b1ee993360..786c1fb7b5 100644 --- a/cpp/src/scheduler/job/SearchJob.cpp +++ b/cpp/src/scheduler/job/SearchJob.cpp @@ -1,8 +1,19 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #include "utils/Log.h" diff --git a/cpp/src/scheduler/job/SearchJob.h b/cpp/src/scheduler/job/SearchJob.h index aebddb0dde..4530c6485b 100644 --- a/cpp/src/scheduler/job/SearchJob.h +++ b/cpp/src/scheduler/job/SearchJob.h @@ -1,8 +1,19 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #include #include From 9d53224b6a710cf995f3939d12304ff7efb0b586 Mon Sep 17 00:00:00 2001 From: starlord Date: Tue, 17 Sep 2019 11:06:26 +0800 Subject: [PATCH 010/217] MS-557 merge Log.h Former-commit-id: 0fb5985bc71132307d66eea85c098d3518935c8c --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index a14997bb75..a3696e25f8 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -9,6 +9,7 @@ Please mark all change in change log and use the ticket from JIRA. ## Improvement - MS-552 - Add and change the easylogging library - MS-553 - Refine cache code +- MS-557 - Merge Log.h ## New Feature From 49f80d3cb743580819c9e6021ab60e8711eeb719 Mon Sep 17 00:00:00 2001 From: zhiru Date: Tue, 17 Sep 2019 14:14:15 +0800 Subject: [PATCH 011/217] update main CMakeList Former-commit-id: 68124f2a284c90ef72edc07d530e88fa67a7ed6a --- cpp/CMakeLists.txt | 57 +++++++++------------------------------------- 1 file changed, 11 insertions(+), 46 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 1754bfeff3..21c79ac07d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -28,14 +28,12 @@ set(MILVUS_VERSION "${GIT_BRANCH_NAME}") string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}") if(CMAKE_BUILD_TYPE STREQUAL "Release") - set(BUILD_TYPE "release") + set(BUILD_TYPE "Release") else() - set(BUILD_TYPE "debug") + set(BUILD_TYPE "Debug") endif() message(STATUS "Build type = ${BUILD_TYPE}") -add_definitions(-DNEW_SCHEDULER) - project(milvus VERSION "${MILVUS_VERSION}") project(milvus_engine LANGUAGES CUDA CXX) @@ -69,33 +67,30 @@ set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR}) find_package(CUDA) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda") -set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") -message(STATUS "CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}") -message(STATUS "CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}") set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED on) if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") - message(STATUS "building milvus_engine on x86 architecture") + message(STATUS "Building milvus_engine on x86 architecture") set(MILVUS_BUILD_ARCH x86_64) elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)") - message(STATUS "building milvus_engine on ppc architecture") + message(STATUS "Building milvus_engine on ppc architecture") set(MILVUS_BUILD_ARCH ppc64le) else() - message(WARNING "unknown processor type") + message(WARNING "Unknown processor type") message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}") set(MILVUS_BUILD_ARCH unknown) endif() if(CMAKE_BUILD_TYPE STREQUAL "Release") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp") + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3") else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp") + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") endif() -set(ALLOW_DUPLICATE_CUSTOM_TARGETS TRUE) - set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") include(ExternalProject) @@ -103,47 +98,26 @@ include(DefineOptions) include(BuildUtils) include(ThirdPartyPackages) -include_directories(${MILVUS_SOURCE_DIR}) -link_directories(${MILVUS_BINARY_DIR}) +config_summary() -## Following should be check - -set(MILVUS_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include) set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) - -add_compile_definitions(PROFILER=${PROFILER}) - -message(STATUS "MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}") -if (MILVUS_ENABLE_PROFILING STREQUAL "ON") - ADD_DEFINITIONS(-DMILVUS_ENABLE_PROFILING) -endif() - -include_directories(${MILVUS_ENGINE_INCLUDE}) -include_directories(${MILVUS_ENGINE_SRC}) - -link_directories(${CMAKE_CURRRENT_BINARY_DIR}) - add_subdirectory(src) -if (BUILD_COVERAGE STREQUAL "ON") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") -endif() - - if (BUILD_UNIT_TEST STREQUAL "ON") + if (BUILD_COVERAGE STREQUAL "ON") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") + endif() add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/unittest) endif() add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean) - if("${MILVUS_DB_PATH}" STREQUAL "") set(MILVUS_DB_PATH "/tmp/milvus") endif() configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf) -#install install(DIRECTORY scripts/ DESTINATION scripts FILE_PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ @@ -155,12 +129,3 @@ install(FILES conf/log_config.conf DESTINATION conf) -install(FILES - ./Milvus-EULA-cn.md - ./Milvus-EULA-en.md - DESTINATION - license - ) - - -config_summary() From 97539416513d8f5ed27a4178b5b869d9b409b2c5 Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Tue, 17 Sep 2019 14:15:42 +0800 Subject: [PATCH 012/217] Add new file Former-commit-id: 718692a49171c7bb36219dbbd0db024427b40904 --- cpp/CONTRIBUTING.md | 52 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 cpp/CONTRIBUTING.md diff --git a/cpp/CONTRIBUTING.md b/cpp/CONTRIBUTING.md new file mode 100644 index 0000000000..7e6398715e --- /dev/null +++ b/cpp/CONTRIBUTING.md @@ -0,0 +1,52 @@ +# Contributing to Milvus + +First of all, thanks for taking the time to contribute to Milvus! It's people like you that help Milvus come to fruition. + +The following are a set of guidelines for contributing to Milvus. Following these guidelines helps contributing to this project easy and transparent. These are mostly guideline, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request. + +As for everything else in the project, the contributions to Milvus are governed by our [Code of Conduct](http://hood.ie/code-of-conduct/). + +TOC + +## Contribution Checklist + +Before you make any contributions, make sure you follow this list. + +- Read [contributing to Milvus](CONTRIBUTING.md). +- Check if the changes are consistent with the [Coding Style]. +- Run [Unit Tests]. + +## What contributions can I make? + +Contributions to Milvus fall into the following categories. + +1. To report a bug or a problem with documentation, please file an [issue](https://github.com/rapidsai/cudf/issues/new/choose) providing the details of the problem. If you believe the issue needs priority attention, please comment on the issue to notify the team. +2. To propose a new feature, please file a new feature request [issue](https://github.com/rapidsai/cudf/issues/new/choose). Describe the intended feature and discuss the design and implementation with the team and community. Once the team agrees that the plan looks good, go ahead and implement it, following the [Contributing code]. +3. To implement a feature or bug-fix for an existing outstanding issue, follow the [Contributing code]. If you need more context on a particular issue, comment on the issue to let people know. + +## How can I contribute? + +### Contributor License Agreement (CLA) + +We greatly welcome your contributions. However, before we can accept your pull requests, we will need you to [sign a Contributor License Agreement (CLA)]. + +### Contributing code + +If you have improvements to Milvus, send us your pull requests! For those just getting started, GitHub has a [how-to](https://help.github.com/en/articles/about-pull-requests). + +The Milvus team members will review your pull requests, and once it is accepted, it will be given a `ready to merge` label. This means we are working on submitting your pull request to the internal repository. After the change has been submitted internally, your pull request will be merged automatically on GitHub. + + + +### General guidelines + +Before sending your pull requests for review, make sure your changes are consistent with the guidelines and follow the Milvus coding style. + +- Include unit tests when you contribute new features, as they help to a) prove that your code works correctly, and b) guard against future breaking changes to lower the maintenance cost. +- Bug fixes also generally require unit tests, because the presence of bugs usually indicates insufficient test coverage. +- Keep API compatibility in mind when you change code in Milvus. Reviewers of your pull request will comment on any API compatibility issues. +- When you contribute a new feature to Milvus, the maintenance burden is (by default) transferred to the Milvus team. This means that the benefit of the contribution must be compared against the cost of maintaining the feature. + + + +## Coding Style \ No newline at end of file From 359a71de5acf90de4089384f88e84b3738ef0124 Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Tue, 17 Sep 2019 14:23:31 +0800 Subject: [PATCH 013/217] Update CONTRIBUTING.md Former-commit-id: 17c0b8e8ac44b035ed441c91a661b60e870d56db --- cpp/CONTRIBUTING.md | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/cpp/CONTRIBUTING.md b/cpp/CONTRIBUTING.md index 7e6398715e..58c41cd313 100644 --- a/cpp/CONTRIBUTING.md +++ b/cpp/CONTRIBUTING.md @@ -13,31 +13,25 @@ TOC Before you make any contributions, make sure you follow this list. - Read [contributing to Milvus](CONTRIBUTING.md). -- Check if the changes are consistent with the [Coding Style]. -- Run [Unit Tests]. +- Check if the changes are consistent with the [Coding Style](CONTRIBUTING.md#coding-style). +- Run [Unit Tests](CONTRIBUTING.md#run-unit-test). ## What contributions can I make? Contributions to Milvus fall into the following categories. -1. To report a bug or a problem with documentation, please file an [issue](https://github.com/rapidsai/cudf/issues/new/choose) providing the details of the problem. If you believe the issue needs priority attention, please comment on the issue to notify the team. -2. To propose a new feature, please file a new feature request [issue](https://github.com/rapidsai/cudf/issues/new/choose). Describe the intended feature and discuss the design and implementation with the team and community. Once the team agrees that the plan looks good, go ahead and implement it, following the [Contributing code]. +1. To report a bug or a problem with documentation, please file an [issue](https://github.com/milvus-io/milvus/issues) providing the details of the problem. If you believe the issue needs priority attention, please comment on the issue to notify the team. +2. To propose a new feature, please file a new feature request [issue](https://github.com/milvus-io/milvus/issues). Describe the intended feature and discuss the design and implementation with the team and community. Once the team agrees that the plan looks good, go ahead and implement it, following the [Contributing code]. 3. To implement a feature or bug-fix for an existing outstanding issue, follow the [Contributing code]. If you need more context on a particular issue, comment on the issue to let people know. ## How can I contribute? -### Contributor License Agreement (CLA) - -We greatly welcome your contributions. However, before we can accept your pull requests, we will need you to [sign a Contributor License Agreement (CLA)]. - ### Contributing code If you have improvements to Milvus, send us your pull requests! For those just getting started, GitHub has a [how-to](https://help.github.com/en/articles/about-pull-requests). The Milvus team members will review your pull requests, and once it is accepted, it will be given a `ready to merge` label. This means we are working on submitting your pull request to the internal repository. After the change has been submitted internally, your pull request will be merged automatically on GitHub. - - ### General guidelines Before sending your pull requests for review, make sure your changes are consistent with the guidelines and follow the Milvus coding style. @@ -48,5 +42,14 @@ Before sending your pull requests for review, make sure your changes are consist - When you contribute a new feature to Milvus, the maintenance burden is (by default) transferred to the Milvus team. This means that the benefit of the contribution must be compared against the cost of maintaining the feature. +## Coding Style + +## Run unit test + +```shell +$ ./build.sh -u +or +$ ./build.sh --unittest +``` + -## Coding Style \ No newline at end of file From fa9b470ddf726c998394c29608479bfbc2065004 Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Tue, 17 Sep 2019 14:25:31 +0800 Subject: [PATCH 014/217] Update CONTRIBUTING.md Former-commit-id: 91a46fa2630cbcd50cdd35168449d474dcb36337 --- cpp/CONTRIBUTING.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/CONTRIBUTING.md b/cpp/CONTRIBUTING.md index 58c41cd313..a44a933c85 100644 --- a/cpp/CONTRIBUTING.md +++ b/cpp/CONTRIBUTING.md @@ -12,16 +12,16 @@ TOC Before you make any contributions, make sure you follow this list. -- Read [contributing to Milvus](CONTRIBUTING.md). -- Check if the changes are consistent with the [Coding Style](CONTRIBUTING.md#coding-style). -- Run [Unit Tests](CONTRIBUTING.md#run-unit-test). +- Read [Contributing to Milvus](CONTRIBUTING.md). +- Check if the changes are consistent with the [coding style](CONTRIBUTING.md#coding-style). +- Run [unit tests](CONTRIBUTING.md#run-unit-test). ## What contributions can I make? Contributions to Milvus fall into the following categories. -1. To report a bug or a problem with documentation, please file an [issue](https://github.com/milvus-io/milvus/issues) providing the details of the problem. If you believe the issue needs priority attention, please comment on the issue to notify the team. -2. To propose a new feature, please file a new feature request [issue](https://github.com/milvus-io/milvus/issues). Describe the intended feature and discuss the design and implementation with the team and community. Once the team agrees that the plan looks good, go ahead and implement it, following the [Contributing code]. +1. To report a bug or a problem with documentation, please file an [issue](https://github.com/milvus-io/milvus/issues/new) providing the details of the problem. If you believe the issue needs priority attention, please comment on the issue to notify the team. +2. To propose a new feature, please file a new feature request [issue](https://github.com/milvus-io/milvus/issues/new). Describe the intended feature and discuss the design and implementation with the team and community. Once the team agrees that the plan looks good, go ahead and implement it, following the [Contributing code]. 3. To implement a feature or bug-fix for an existing outstanding issue, follow the [Contributing code]. If you need more context on a particular issue, comment on the issue to let people know. ## How can I contribute? From 7941aa5caf308ae24e4667e78293a3d0cb3dee28 Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Tue, 17 Sep 2019 14:29:04 +0800 Subject: [PATCH 015/217] Update README.md Former-commit-id: 7b155725e3407f47c6432ce0d678a1c783b29ed8 --- cpp/README.md | 240 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 188 insertions(+), 52 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index 8e09d821e2..d304f58304 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -1,72 +1,208 @@ -### Compilation -#### Step 1: install necessery tools +# Welcome to Milvus - centos7 : - yum install gfortran qt4 flex bison mysql-devel mysql - - ubuntu16.04 : - sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev mysql-client - - cd scripts && sudo ./requirements.sh +Firstly, welcome, and thanks for your interest in [Milvus](https://milvus.io)! No matter who you are, what you do, we greatly appreciate your contribution to help us reinvent data science with Milvus. -If `libmysqlclient_r.so` does not exist after installing MySQL Development Files, you need to create a symbolic link: +## What is Milvus -``` -sudo ln -s /path/to/libmysqlclient.so /path/to/libmysqlclient_r.so +Milvus is an open source vector search engine that makes incredibly fast querying speed enhancement over current solutions of massive vector processing. Built on optimized indexing algorithm, it is compatible with major AI/ML models. + +Milvus was developed by researchers and engineers in ZILLIZ, a tech startup that intends to reinvent data science, with the purpose of providing enterprises with efficient and scalable similarity search and analysis of feature vectors and unstructured data. + +Milvus provides stable [Python](https://www.tensorflow.org/api_docs/python) and [C++](https://www.tensorflow.org/api_docs/cc) APIs, as well as RESTful API. + +Keep up-to-date with newest releases and latest updates by subscribing to xxx. + +- GPU-accelerated search engine + + Milvus is designed for the largest scale of vector index. CPU/GPU heterogeneous computing architecture allows you to process data at a speed 1000 times faster. + +- Intelligent index + + With a “Decide Your Own Algorithm” approach, you can embed machine learning and advanced algorithms into Milvus without the headache of complex data engineering or migrating data between disparate systems. Milvus is built on optimized indexing algorithm based on quantization indexing, tree-based and graph indexing methods. + +- Strong scalability + + The data is stored and computed on a distributed architecture. This lets you scale data sizes up and down without redesigning the system. + +## Get started + +### Install and start Milvus server + +#### Use Docker + +Use Docker to install Milvus is a breeze. See the [Milvus install guide](doc/master/userguide/install_milvus.md) for details. + +#### Use source code + +##### Compilation + +###### Step 1 Install necessary tools + +```shell +# Install tools +Centos7 : +$ yum install gfortran qt4 flex bison mysql-devel mysql + +Ubuntu16.04 : +$ sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev mysql-client + ``` -#### Step 2: build(output to cmake_build folder) +Verify the existence of `libmysqlclient_r.so`: -cmake_build/src/milvus_server is the server +```shell +# Verify existence +$ locate libmysqlclient_r.so +``` - cd [sourcecode path]/cpp/thirdparty - git clone git@192.168.1.105:megasearch/knowhere.git - cd knowhere - ./build.sh -t Debug - or ./build.sh -t Release +If not, you need to create a symbolic link: - cd [sourcecode path]/cpp - ./build.sh -t Debug - or ./build.sh -t Release +```shell +# Create symbolic link +$ sudo ln -s /path/to/libmysqlclient.so /path/to/libmysqlclient_r.so +``` -If you encounter the following error when building: +###### Step 2 Build + +```shell +TBD +cd [Milvus sourcecode path]/cpp/thirdparty +git clone git@192.168.1.105:megasearch/knowhere.git +cd knowhere +./build.sh -t Debug +or ./build.sh -t Release + +cd [sourcecode path]/cpp +./build.sh -t Debug +or ./build.sh -t Release +``` + +When the build is completed, all the stuff that you need in order to run Milvus will be installed under `[Milvus root path]/cpp/milvus`. + +If you encounter the following error message, `protocol https not supported or disabled in libcurl` -1. Install libcurl4-openssl-dev +please reinstall CMake with curl: -2. Install cmake 3.14: - - ``` - ./bootstrap --system-curl - make - sudo make install +1. ```shell + CentOS 7: + $ yum install curl-devel + Ubuntu 16.04: + $ sudo apt-get install libcurl4-openssl-dev ``` -#### To build unittest: +2. Install [CMake 3.14](https://github.com/Kitware/CMake/releases/download/v3.14.6/cmake-3.14.6.tar.gz): - ./build.sh -u - or - ./build.sh --unittest + ```shell + $ ./bootstrap --system-curl + $ make + $ sudo make install + ``` -#### To run code coverage +##### Run unit test - apt-get install lcov - ./build.sh -u -c - -### Launch server -Set config in cpp/conf/server_config.yaml - -Add milvus/lib to LD_LIBRARY_PATH - -``` -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/path/to/milvus/lib +```shell +$ ./build.sh -u +or +$ ./build.sh --unittest ``` -Then launch server with config: - cd [build output path] - start_server.sh - stop_server.sh +##### Run code coverage -### Launch test_client(only for debug) -If you want to test remote api, you can run sdk example. - [build output path]/sdk/examples/grpcsimple/sdk_simple +```shell +CentOS 7: +$ yum install lcov +Ubuntu 16.04: +$ sudo apt-get install lcov + +$ ./build.sh -u -c +``` + +##### Launch Milvus server + +```shell +$ cd [Milvus root path]/cpp/milvus +``` + +Add `lib/` directory to `LD_LIBRARY_PATH` + +``` +$ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/path/to/milvus/lib +``` + +Then start Milvus server: + +``` +$ cd scripts +$ ./start_server.sh +``` + +To stop Milvus server, run: + +```shell +$ ./stop_server.sh +``` + +To edit Milvus settings in `conf/server_config.yaml` and `conf/log_config.conf`, please read [Milvus Configuration](milvus-io/milvus/doc/master/reference/milvus_config.md). + +### Try your first Milvus program + +#### Run Python example code + +Make sure [Python 3.4](https://www.python.org/downloads/) or higher is already installed and in use. + +Install Milvus Python SDK. + +```shell +# Install Milvus Python SDK +$ pip install pymilvus==0.2.0 +``` + +Create a new file `example.py`, and add [Python example code](https://github.com/milvus-io/pymilvus/blob/branch-0.3.1/examples/AdvancedExample.py) to it. + +Run the example code. + +```python +# Run Milvus Python example +$ python3 example.py +``` + +#### Run C++ example code + +```shell + # Run Milvus C++ example + $ cd [Milvus root path]/cpp/milvus/bin + $ ./sdk_simple +``` + +## Contribution guidelines + +Contributions are welcomed and greatly appreciated. If you want to contribute to Milvus, please read the [contribution guidelines](CONTRIBUTING.md). This project adheres to the [code of conduct](CODE OF CONDUCT.md) of Milvus. By participating, you are expected to uphold this code. + +We use [GitHub issues] to track issues and bugs. For general questions and discussions, please go to [Milvus Forum]. + +## Join the Milvus community + +For public discussion of Milvus, please join our [discussion group]. + +## Milvus Roadmap + +Please read our [roadmap](milvus-io/milvus/doc/master/roadmap.md) to learn about upcoming features. + +## Resources + +[Milvus official website](https://milvus.io) + +[Milvus docs](https://www.milvus.io/docs/en/QuickStart/) + +[Milvus blog](https://www.milvus.io/blog/) + +[Milvus CSDN](https://mp.csdn.net/mdeditor/100041006#) + +[Milvus roadmap](milvus-io/milvus/doc/master/roadmap.md) + +[Milvus white paper] + +## License + +[Apache 2.0 license](milvus-io/milvus/LICENSE.md) \ No newline at end of file From 17792a37ae5b4db13a00e6e408ed6a4157e98a79 Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Tue, 17 Sep 2019 14:31:17 +0800 Subject: [PATCH 016/217] Update README.md Former-commit-id: 625e0ce886fdb20c072e136364b6652517c1bb65 --- cpp/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index d304f58304..1cdff6bee8 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -8,9 +8,9 @@ Milvus is an open source vector search engine that makes incredibly fast queryin Milvus was developed by researchers and engineers in ZILLIZ, a tech startup that intends to reinvent data science, with the purpose of providing enterprises with efficient and scalable similarity search and analysis of feature vectors and unstructured data. -Milvus provides stable [Python](https://www.tensorflow.org/api_docs/python) and [C++](https://www.tensorflow.org/api_docs/cc) APIs, as well as RESTful API. +Milvus provides stable Python and C++ APIs, as well as RESTful API. -Keep up-to-date with newest releases and latest updates by subscribing to xxx. +Keep up-to-date with newest releases and latest updates by reading [Releases](milvus-io/doc/master/releases). - GPU-accelerated search engine From 3cea080566b68bbb11d76a2abd36e22cfd3adca0 Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Tue, 17 Sep 2019 14:39:43 +0800 Subject: [PATCH 017/217] Add new file Former-commit-id: a07c10c27a7ca33125b7fb3c4634f3dd24317905 --- cpp/CODE OF CONDUCT.md | 48 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 cpp/CODE OF CONDUCT.md diff --git a/cpp/CODE OF CONDUCT.md b/cpp/CODE OF CONDUCT.md new file mode 100644 index 0000000000..ef3cc592c6 --- /dev/null +++ b/cpp/CODE OF CONDUCT.md @@ -0,0 +1,48 @@ +# Milvus Code of Conduct + +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment include: + +- Using welcoming and inclusive language. +- Being respectful of differing viewpoints and experiences. +- Gracefully accepting constructive criticism. +- Focusing on what is best for the community. +- Showing empathy towards other community members. + +Examples of unacceptable behavior by participants include: + +- The use of sexualized language or imagery and unwelcome sexual attention or advances. +- Trolling, insulting/derogatory comments, and personal or political attacks. +- Public or private harassment. +- Publishing others' private information, such as a physical or electronic address, without explicit permission. +- Conduct which could reasonably be considered inappropriate for the forum in which it occurs. + +All Milvus forums and spaces are meant for professional interactions, and any behavior which could reasonably be considered inappropriate in a professional setting is unacceptable. + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies to all content on milvus.io, Milvus’s GitHub organization, or any other official Milvus web presence allowing for community interactions, as well as at all official Milvus events, whether offline or online. + +The Code of Conduct also applies within all project spaces and in public spaces whenever an individual is representing Milvus or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed or de facto representative at an online or offline event. Representation of Milvus may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at support@zilliz.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq + From e16ac72a25f0a3a56144f6768c9e9726a949743f Mon Sep 17 00:00:00 2001 From: starlord Date: Tue, 17 Sep 2019 17:37:31 +0800 Subject: [PATCH 018/217] MS-558 refine status code Former-commit-id: cb176cf273ec66e1f7f5e89e9ba4c39cabc511ea --- cpp/CHANGELOG.md | 1 + cpp/src/db/DB.h | 5 +- cpp/src/db/engine/EngineFactory.h | 3 +- cpp/src/db/engine/ExecutionEngine.h | 2 +- cpp/src/db/insert/MemManager.h | 2 +- cpp/src/db/insert/MemManagerImpl.h | 4 +- cpp/src/db/insert/MemTable.h | 2 +- cpp/src/db/insert/MemTableFile.h | 4 +- cpp/src/db/insert/VectorSource.h | 2 +- cpp/src/db/meta/Meta.h | 3 +- cpp/src/metrics/Metrics.h | 5 +- cpp/src/sdk/CMakeLists.txt | 9 +- .../examples/grpcsimple/src/ClientTest.cpp | 6 +- cpp/src/sdk/grpc/ClientProxy.cpp | 2 +- cpp/src/sdk/include/Status.h | 349 ++---------------- cpp/src/sdk/interface/Status.cpp | 167 ++++----- cpp/src/sdk/util/Exception.h | 44 --- cpp/src/server/grpc_impl/GrpcRequestTask.cpp | 28 +- cpp/src/{db => utils}/Status.cpp | 81 ++-- cpp/src/{db => utils}/Status.h | 51 +-- cpp/unittest/db/CMakeLists.txt | 4 +- cpp/unittest/db/db_tests.cpp | 16 +- cpp/unittest/db/mem_test.cpp | 16 +- cpp/unittest/db/misc_test.cpp | 39 +- cpp/unittest/db/mysql_db_test.cpp | 8 +- cpp/unittest/metrics/CMakeLists.txt | 4 +- cpp/unittest/metrics/metrics_test.cpp | 6 +- cpp/unittest/metrics/utils.h | 2 +- cpp/unittest/scheduler/CMakeLists.txt | 4 +- cpp/unittest/server/CMakeLists.txt | 4 +- cpp/unittest/server/util_test.cpp | 42 +++ 31 files changed, 301 insertions(+), 614 deletions(-) delete mode 100644 cpp/src/sdk/util/Exception.h rename cpp/src/{db => utils}/Status.cpp (64%) rename cpp/src/{db => utils}/Status.h (55%) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index de7d8306fd..1f0aedef85 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -11,6 +11,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-553 - Refine cache code - MS-557 - Merge Log.h - MS-556 - Add Job Definition in Scheduler +- MS-558 - Refine status code ## New Feature diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index 1de424ed78..5fe6df491f 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -15,12 +15,13 @@ // specific language governing permissions and limitations // under the License. + #pragma once #include "Options.h" -#include "meta/Meta.h" -#include "Status.h" #include "Types.h" +#include "meta/Meta.h" +#include "utils/Status.h" #include #include diff --git a/cpp/src/db/engine/EngineFactory.h b/cpp/src/db/engine/EngineFactory.h index 906260b966..c3fb41adde 100644 --- a/cpp/src/db/engine/EngineFactory.h +++ b/cpp/src/db/engine/EngineFactory.h @@ -15,10 +15,11 @@ // specific language governing permissions and limitations // under the License. + #pragma once -#include "db/Status.h" #include "ExecutionEngine.h" +#include "utils/Status.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/db/engine/ExecutionEngine.h b/cpp/src/db/engine/ExecutionEngine.h index 417d002a21..5868f15d33 100644 --- a/cpp/src/db/engine/ExecutionEngine.h +++ b/cpp/src/db/engine/ExecutionEngine.h @@ -17,7 +17,7 @@ #pragma once -#include "db/Status.h" +#include "utils/Status.h" #include #include diff --git a/cpp/src/db/insert/MemManager.h b/cpp/src/db/insert/MemManager.h index de6ee85ed1..37f0548646 100644 --- a/cpp/src/db/insert/MemManager.h +++ b/cpp/src/db/insert/MemManager.h @@ -18,7 +18,7 @@ #pragma once -#include "db/Status.h" +#include "utils/Status.h" #include "db/Types.h" #include diff --git a/cpp/src/db/insert/MemManagerImpl.h b/cpp/src/db/insert/MemManagerImpl.h index 9b503216a7..51e8f1044e 100644 --- a/cpp/src/db/insert/MemManagerImpl.h +++ b/cpp/src/db/insert/MemManagerImpl.h @@ -18,10 +18,10 @@ #pragma once -#include "db/meta/Meta.h" #include "MemTable.h" -#include "db/Status.h" #include "MemManager.h" +#include "db/meta/Meta.h" +#include "utils/Status.h" #include #include diff --git a/cpp/src/db/insert/MemTable.h b/cpp/src/db/insert/MemTable.h index a443186900..def4a6dcca 100644 --- a/cpp/src/db/insert/MemTable.h +++ b/cpp/src/db/insert/MemTable.h @@ -18,9 +18,9 @@ #pragma once -#include "db/Status.h" #include "MemTableFile.h" #include "VectorSource.h" +#include "utils/Status.h" #include diff --git a/cpp/src/db/insert/MemTableFile.h b/cpp/src/db/insert/MemTableFile.h index d4b11cda7a..5ac3e15ad2 100644 --- a/cpp/src/db/insert/MemTableFile.h +++ b/cpp/src/db/insert/MemTableFile.h @@ -18,10 +18,10 @@ #pragma once -#include "db/Status.h" -#include "db/meta/Meta.h" #include "VectorSource.h" +#include "db/meta/Meta.h" #include "db/engine/ExecutionEngine.h" +#include "utils/Status.h" namespace zilliz { diff --git a/cpp/src/db/insert/VectorSource.h b/cpp/src/db/insert/VectorSource.h index 5c44f580d2..a0292181ce 100644 --- a/cpp/src/db/insert/VectorSource.h +++ b/cpp/src/db/insert/VectorSource.h @@ -18,10 +18,10 @@ #pragma once -#include "db/Status.h" #include "db/meta/Meta.h" #include "db/IDGenerator.h" #include "db/engine/ExecutionEngine.h" +#include "utils/Status.h" namespace zilliz { diff --git a/cpp/src/db/meta/Meta.h b/cpp/src/db/meta/Meta.h index 755e0170f2..ee4972ae6b 100644 --- a/cpp/src/db/meta/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -15,12 +15,13 @@ // specific language governing permissions and limitations // under the License. + #pragma once #include "MetaTypes.h" #include "db/Options.h" -#include "db/Status.h" #include "db/Types.h" +#include "utils/Status.h" #include #include diff --git a/cpp/src/metrics/Metrics.h b/cpp/src/metrics/Metrics.h index e298af7acf..30d6e68f52 100644 --- a/cpp/src/metrics/Metrics.h +++ b/cpp/src/metrics/Metrics.h @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. + #pragma once #include "MetricBase.h" @@ -63,7 +64,7 @@ protected: //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CollectInsertMetrics : CollectMetricsBase { public: - CollectInsertMetrics(size_t n, engine::Status& status) : n_(n), status_(status) { + CollectInsertMetrics(size_t n, Status& status) : n_(n), status_(status) { } ~CollectInsertMetrics() { @@ -87,7 +88,7 @@ public: private: size_t n_; - engine::Status& status_; + Status& status_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/cpp/src/sdk/CMakeLists.txt b/cpp/src/sdk/CMakeLists.txt index 6c3a68b8b8..5d6a1ea465 100644 --- a/cpp/src/sdk/CMakeLists.txt +++ b/cpp/src/sdk/CMakeLists.txt @@ -20,8 +20,6 @@ aux_source_directory(interface interface_files) -aux_source_directory(util util_files) - include_directories(/usr/include) include_directories(include) include_directories(/usr/local/include) @@ -42,12 +40,15 @@ set(grpc_service_files add_library(milvus_sdk STATIC ${interface_files} ${grpc_client_files} - ${util_files} ${grpc_service_files} ) target_link_libraries(milvus_sdk - ${third_party_libs} + ${client_grpc_lib} + bzip2 + lz4 + snappy + zlib ) install(TARGETS milvus_sdk DESTINATION lib) diff --git a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp index abfc92cc0b..f0f7cf5292 100644 --- a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp @@ -36,10 +36,10 @@ const std::string TABLE_NAME = GetTableName(); constexpr int64_t TABLE_DIMENSION = 512; constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; constexpr int64_t BATCH_ROW_COUNT = 100000; -constexpr int64_t NQ = 100; -constexpr int64_t TOP_K = 1; +constexpr int64_t NQ = 5; +constexpr int64_t TOP_K = 10; constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different -constexpr int64_t ADD_VECTOR_LOOP = 1; +constexpr int64_t ADD_VECTOR_LOOP = 10; constexpr int64_t SECONDS_EACH_HOUR = 3600; #define BLOCK_SPLITER std::cout << "===========================================" << std::endl; diff --git a/cpp/src/sdk/grpc/ClientProxy.cpp b/cpp/src/sdk/grpc/ClientProxy.cpp index 56ba1e2de6..2a87f7f929 100644 --- a/cpp/src/sdk/grpc/ClientProxy.cpp +++ b/cpp/src/sdk/grpc/ClientProxy.cpp @@ -51,7 +51,7 @@ ClientProxy::Connect(const ConnectParam ¶m) { Status ClientProxy::Connect(const std::string &uri) { if (!UriCheck(uri)) { - return Status::Invalid("Invalid uri"); + return Status(StatusCode::InvalidAgument, "Invalid uri"); } size_t index = uri.find_first_of(':', 0); diff --git a/cpp/src/sdk/include/Status.h b/cpp/src/sdk/include/Status.h index 6dbc762c0a..145d09d47a 100644 --- a/cpp/src/sdk/include/Status.h +++ b/cpp/src/sdk/include/Status.h @@ -19,7 +19,6 @@ #pragma once #include -#include /** \brief Milvus SDK namespace */ @@ -30,345 +29,61 @@ namespace milvus { */ enum class StatusCode { OK = 0, -// system error section + // system error section UnknownError = 1, NotSupported, NotConnected, -// function error section + // function error section InvalidAgument = 1000, RPCFailed, ServerFailed, }; +using ErrorCode = StatusCode; + /** * @brief Status for SDK interface return */ class Status { public: - /** - * @brief Status - * - * Default constructor. - * - */ - Status() = default; + Status(ErrorCode code, const std::string &msg); + Status(); + ~Status(); - /** - * @brief Status - * - * Destructor. - * - */ - ~Status() noexcept; + Status(const Status &s); - /** - * @brief Status - * - * Constructor - * - * @param code, status code. - * @param message, status message. - * - */ - Status(StatusCode code, const std::string &message); + Status & + operator=(const Status &s); - /** - * @brief Status - * - * Copy constructor - * - * @param status, status to be copied. - * - */ - inline - Status(const Status &status); + Status(Status &&s); - /** - * @brief Status - * - * Assignment operator - * - * @param status, status to be copied. - * @return, the status is assigned. - * - */ - Status - &operator=(const Status &s); + Status & + operator=(Status &&s); - /** - * @brief Status - * - * Move constructor - * - * @param status, status to be moved. - * - */ - Status(Status &&s) noexcept ; - - /** - * @brief Status - * - * Move assignment operator - * - * @param status, status to be moved. - * @return, the status is moved. - * - */ - Status - &operator=(Status &&s) noexcept; - - /** - * @brief Status - * - * AND operator - * - * @param status, status to be AND. - * @return, the status after AND operation. - * - */ - inline - Status operator&(const Status &s) const noexcept; - - /** - * @brief Status - * - * AND operator - * - * @param status, status to be AND. - * @return, the status after AND operation. - * - */ - inline - Status operator&(Status &&s) const noexcept; - - /** - * @brief Status - * - * AND operator - * - * @param status, status to be AND. - * @return, the status after AND operation. - * - */ - inline - Status &operator&=(const Status &s) noexcept; - - /** - * @brief Status - * - * AND operator - * - * @param status, status to be AND. - * @return, the status after AND operation. - * - */ - inline - Status &operator&=(Status &&s) noexcept; - - /** - * @brief OK - * - * static OK status constructor - * - * @return, the status with OK. - * - */ - static - Status OK() { return Status(); } - - /** - * @brief OK - * - * static OK status constructor with a specific message - * - * @param, serveral specific messages - * @return, the status with OK. - * - */ - template static Status - OK(Args &&... args) { - return Status(StatusCode::OK, MessageBuilder(std::forward(args)...)); + OK() { return Status(); } + + bool + ok() const { return state_ == nullptr || code() == StatusCode::OK; } + + std::string + ToString() const; + + ErrorCode + code() const { + return (state_ == nullptr) ? StatusCode::OK : *(ErrorCode*)(state_); } -/** - * @brief Invalid - * - * static Invalid status constructor with a specific message - * - * @param, serveral specific messages - * @return, the status with Invalid. - * - */ -template -static Status -Invalid(Args &&... args) { - return Status(StatusCode::InvalidAgument, - MessageBuilder(std::forward(args)...)); -} +private: + inline void + CopyFrom(const Status &s); -/** - * @brief Unknown Error - * - * static unknown error status constructor with a specific message - * - * @param, serveral specific messages - * @return, the status with unknown error. - * - */ -template -static Status -UnknownError(Args &&... args) { - return Status(StatusCode::UnknownError, MessageBuilder(std::forward(args)...)); -} - -/** - * @brief not supported Error - * - * static not supported status constructor with a specific message - * - * @param, serveral specific messages - * @return, the status with not supported error. - * - */ -template -static Status -NotSupported(Args &&... args) { - return Status(StatusCode::NotSupported, MessageBuilder(std::forward(args)...)); -} - -/** - * @brief ok - * - * Return true iff the status indicates success. - * - * @return, if the status indicates success. - * - */ -bool -ok() const { return (state_ == nullptr); } - -/** - * @brief IsInvalid - * - * Return true iff the status indicates invalid. - * - * @return, if the status indicates invalid. - * - */ -bool -IsInvalid() const { return code() == StatusCode::InvalidAgument; } - -/** - * @brief IsUnknownError - * - * Return true iff the status indicates unknown error. - * - * @return, if the status indicates unknown error. - * - */ -bool -IsUnknownError() const { return code() == StatusCode::UnknownError; } - -/** - * @brief IsNotSupported - * - * Return true iff the status indicates not supported. - * - * @return, if the status indicates not supported. - * - */ -bool -IsNotSupported() const { return code() == StatusCode::NotSupported; } - -/** - * @brief ToString - * - * Return error message string. - * - * @return, error message string. - * - */ -std::string -ToString() const; - -/** - * @brief CodeAsString - * - * Return a string representation of the status code. - * - * @return, a string representation of the status code. - * - */ -std::string -CodeAsString() const; - -/** - * @brief code - * - * Return the StatusCode value attached to this status. - * - * @return, the status code value attached to this status. - * - */ -StatusCode -code() const { return ok() ? StatusCode::OK : state_->code; } - -/** - * @brief message - * - * Return the specific error message attached to this status. - * - * @return, the specific error message attached to this status. - * - */ -std::string -message() const { return ok() ? "" : state_->message; } + inline void + MoveFrom(Status &s); private: -struct State { - StatusCode code; - std::string message; -}; + const char *state_ = nullptr; +}; // Status -// OK status has a `nullptr` state_. Otherwise, `state_` points to -// a `State` structure containing the error code and message. -State *state_ = nullptr; - -void -DeleteState() { - delete state_; - state_ = nullptr; -} - -void -CopyFrom(const Status &s); - -inline void -MoveFrom(Status &s); - -template -static void -MessageBuilderRecursive(std::stringstream &stream, Head &&head) { - stream << head; -} - -template -static void -MessageBuilderRecursive(std::stringstream &stream, Head &&head, Tail &&... tail) { - MessageBuilderRecursive(stream, std::forward(head)); - MessageBuilderRecursive(stream, std::forward(tail)...); -} - -template -static std::string -MessageBuilder(Args &&... args) { - std::stringstream stream; - - MessageBuilderRecursive(stream, std::forward(args)...); - - return stream.str(); -} -}; -} \ No newline at end of file +} //Milvus \ No newline at end of file diff --git a/cpp/src/sdk/interface/Status.cpp b/cpp/src/sdk/interface/Status.cpp index 344d86c12f..b04c6581fe 100644 --- a/cpp/src/sdk/interface/Status.cpp +++ b/cpp/src/sdk/interface/Status.cpp @@ -17,18 +17,66 @@ #include "Status.h" +#include + namespace milvus { -Status::~Status() noexcept { - if (state_ != nullptr) { - delete state_; - state_ = nullptr; - } +constexpr int CODE_WIDTH = sizeof(ErrorCode); + +Status::Status(ErrorCode code, const std::string& msg) { + //4 bytes store code + //4 bytes store message length + //the left bytes store message string + const uint32_t length = (uint32_t)msg.size(); + char* result = new char[length + sizeof(length) + CODE_WIDTH]; + memcpy(result, &code, CODE_WIDTH); + memcpy(result + CODE_WIDTH, &length, sizeof(length)); + memcpy(result + sizeof(length) + CODE_WIDTH, msg.data(), length); + + state_ = result; } -static inline std::ostream &operator<<(std::ostream &os, const Status &x) { - os << x.ToString(); - return os; +Status::Status() + : state_(nullptr) { + +} + +Status::~Status() { + delete state_; +} + +Status::Status(const Status &s) + : state_(nullptr) { + CopyFrom(s); +} + +Status &Status::operator=(const Status &s) { + CopyFrom(s); + return *this; +} + +Status::Status(Status &&s) + : state_(nullptr) { + MoveFrom(s); +} + +Status &Status::operator=(Status &&s) { + MoveFrom(s); + return *this; +} + +void Status::CopyFrom(const Status &s) { + delete state_; + state_ = nullptr; + if(s.state_ == nullptr) { + return; + } + + uint32_t length = 0; + std::memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length)); + int buff_len = length + sizeof(length) + CODE_WIDTH; + state_ = new char[buff_len]; + memcpy((void*)state_, (void*)s.state_, buff_len); } void Status::MoveFrom(Status &s) { @@ -37,106 +85,45 @@ void Status::MoveFrom(Status &s) { s.state_ = nullptr; } -Status::Status(const Status &s) - : state_((s.state_ == nullptr) ? nullptr : new State(*s.state_)) {} - -Status::Status(Status &&s) noexcept { - MoveFrom(s); -} - -Status &Status::operator=(const Status &s) { - if (state_ != s.state_) { - CopyFrom(s); - } - return *this; -} - -Status &Status::operator=(Status &&s) noexcept { - MoveFrom(s); - return *this; -} - -Status Status::operator&(const Status &status) const noexcept { - if (ok()) { - return status; - } else { - return *this; - } -} - -Status Status::operator&(Status &&s) const noexcept { - if (ok()) { - return std::move(s); - } else { - return *this; - } -} - -Status &Status::operator&=(const Status &s) noexcept { - if (ok() && !s.ok()) { - CopyFrom(s); - } - return *this; -} - -Status &Status::operator&=(Status &&s) noexcept { - if (ok() && !s.ok()) { - MoveFrom(s); - } - return *this; -} - -Status::Status(StatusCode code, const std::string &message) { - state_ = new State; - state_->code = code; - state_->message = message; -} - -void Status::CopyFrom(const Status &status) { - delete state_; - if (status.state_ == nullptr) { - state_ = nullptr; - } else { - state_ = new State(*status.state_); - } -} - -std::string Status::CodeAsString() const { +std::string Status::ToString() const { if (state_ == nullptr) { return "OK"; } - const char *type = nullptr; + std::string result; switch (code()) { case StatusCode::OK: - type = "OK"; - break; - case StatusCode::InvalidAgument: - type = "Invalid agument"; + result = "OK "; break; case StatusCode::UnknownError: - type = "Unknown error"; + result = "Unknown error: "; break; case StatusCode::NotSupported: - type = "Not Supported"; + result = "Not supported: "; break; case StatusCode::NotConnected: - type = "Not Connected"; + result = "Not connected: "; + break; + case StatusCode::InvalidAgument: + result = "Invalid agument: "; + break; + case StatusCode::RPCFailed: + result = "Remote call failed: "; + break; + case StatusCode::ServerFailed: + result = "Service error: "; break; default: - type = "Unknown"; + result = "Error code(" + std::to_string((int)code()) + "): "; break; } - return std::string(type); -} -std::string Status::ToString() const { - std::string result(CodeAsString()); - if (state_ == nullptr) { - return result; + uint32_t length = 0; + memcpy(&length, state_ + CODE_WIDTH, sizeof(length)); + if(length > 0) { + result.append(state_ + sizeof(length) + CODE_WIDTH, length); } - result += ": "; - result += state_->message; + return result; } diff --git a/cpp/src/sdk/util/Exception.h b/cpp/src/sdk/util/Exception.h deleted file mode 100644 index f55b8f0632..0000000000 --- a/cpp/src/sdk/util/Exception.h +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "Status.h" - -#include - -namespace milvus { -class Exception : public std::exception { -public: - Exception(StatusCode error_code, - const std::string &message = std::string()) - : error_code_(error_code), message_(message) {} - -public: - StatusCode error_code() const { - return error_code_; - } - - virtual const char *what() const noexcept { - return message_.c_str(); - } - -private: - StatusCode error_code_; - std::string message_; -}; -} \ No newline at end of file diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index 8c7697e1dc..d7b39bea05 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -170,7 +170,7 @@ CreateTableTask::OnExecute() { table_info.metric_type_ = schema_->metric_type(); //step 3: create table - engine::Status stat = DBWrapper::DB()->CreateTable(table_info); + auto stat = DBWrapper::DB()->CreateTable(table_info); if (!stat.ok()) { //table could exist if(stat.code() == DB_ALREADY_EXIST) { @@ -214,7 +214,7 @@ DescribeTableTask::OnExecute() { //step 2: get table info engine::meta::TableSchema table_info; table_info.table_id_ = table_name_; - engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); + auto stat = DBWrapper::DB()->DescribeTable(table_info); if (!stat.ok()) { return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } @@ -261,7 +261,7 @@ CreateIndexTask::OnExecute() { } bool has_table = false; - engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table); + auto stat = DBWrapper::DB()->HasTable(table_name_, has_table); if (!stat.ok()) { return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } @@ -323,7 +323,7 @@ HasTableTask::OnExecute() { } //step 2: check table existence - engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table_); + auto stat = DBWrapper::DB()->HasTable(table_name_, has_table_); if (!stat.ok()) { return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } @@ -362,7 +362,7 @@ DropTableTask::OnExecute() { //step 2: check table existence engine::meta::TableSchema table_info; table_info.table_id_ = table_name_; - engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); + auto stat = DBWrapper::DB()->DescribeTable(table_info); if (!stat.ok()) { if (stat.code() == DB_NOT_FOUND) { return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); @@ -403,7 +403,7 @@ ShowTablesTask::Create(::grpc::ServerWriter<::milvus::grpc::TableName> *writer) ErrorCode ShowTablesTask::OnExecute() { std::vector schema_array; - engine::Status stat = DBWrapper::DB()->AllTables(schema_array); + auto stat = DBWrapper::DB()->AllTables(schema_array); if (!stat.ok()) { return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } @@ -460,7 +460,7 @@ InsertTask::OnExecute() { //step 2: check table existence engine::meta::TableSchema table_info; table_info.table_id_ = insert_param_->table_name(); - engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); + auto stat = DBWrapper::DB()->DescribeTable(table_info); if (!stat.ok()) { if (stat.code() == DB_NOT_FOUND) { return SetError(SERVER_TABLE_NOT_EXIST, @@ -599,7 +599,7 @@ SearchTask::OnExecute() { //step 2: check table existence engine::meta::TableSchema table_info; table_info.table_id_ = table_name_; - engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); + auto stat = DBWrapper::DB()->DescribeTable(table_info); if (!stat.ok()) { if (stat.code() == DB_NOT_FOUND) { return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); @@ -746,7 +746,7 @@ CountTableTask::OnExecute() { //step 2: get row count uint64_t row_count = 0; - engine::Status stat = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); + auto stat = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); if (!stat.ok()) { return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } @@ -819,7 +819,7 @@ DeleteByRangeTask::OnExecute() { //step 2: check table existence engine::meta::TableSchema table_info; table_info.table_id_ = table_name; - engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); + auto stat = DBWrapper::DB()->DescribeTable(table_info); if (!stat.ok()) { if (stat.code(), DB_NOT_FOUND) { return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name + " not exists"); @@ -846,8 +846,8 @@ DeleteByRangeTask::OnExecute() { std::string fname = "/tmp/search_nq_" + this->delete_by_range_param_->table_name() + ".profiling"; ProfilerStart(fname.c_str()); #endif - engine::Status status = DBWrapper::DB()->DeleteTable(table_name, dates); - if (!status.ok()) { + stat = DBWrapper::DB()->DeleteTable(table_name, dates); + if (!stat.ok()) { return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } @@ -882,7 +882,7 @@ PreloadTableTask::OnExecute() { } //step 2: check table existence - engine::Status stat = DBWrapper::DB()->PreloadTable(table_name_); + auto stat = DBWrapper::DB()->PreloadTable(table_name_); if (!stat.ok()) { return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } @@ -923,7 +923,7 @@ DescribeIndexTask::OnExecute() { //step 2: check table existence engine::TableIndex index; - engine::Status stat = DBWrapper::DB()->DescribeIndex(table_name_, index); + auto stat = DBWrapper::DB()->DescribeIndex(table_name_, index); if (!stat.ok()) { return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } diff --git a/cpp/src/db/Status.cpp b/cpp/src/utils/Status.cpp similarity index 64% rename from cpp/src/db/Status.cpp rename to cpp/src/utils/Status.cpp index 0bc51ca59e..0232c823f0 100644 --- a/cpp/src/db/Status.cpp +++ b/cpp/src/utils/Status.cpp @@ -14,15 +14,12 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - -#include -#include -#include #include "Status.h" +#include + namespace zilliz { namespace milvus { -namespace engine { constexpr int CODE_WIDTH = sizeof(ErrorCode); @@ -45,55 +42,87 @@ Status::Status() } Status::~Status() { - delete[] state_; + delete state_; } -const char* Status::CopyState(const char* state) { +Status::Status(const Status &s) + : state_(nullptr) { + CopyFrom(s); +} + +Status &Status::operator=(const Status &s) { + CopyFrom(s); + return *this; +} + +Status::Status(Status &&s) + : state_(nullptr) { + MoveFrom(s); +} + +Status &Status::operator=(Status &&s) { + MoveFrom(s); + return *this; +} + +void Status::CopyFrom(const Status &s) { + delete state_; + state_ = nullptr; + if(s.state_ == nullptr) { + return; + } + uint32_t length = 0; - std::memcpy(&length, state + CODE_WIDTH, sizeof(length)); + std::memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length)); int buff_len = length + sizeof(length) + CODE_WIDTH; - char* result = new char[buff_len]; - memcpy(result, state, buff_len); - return result; + state_ = new char[buff_len]; + memcpy((void*)state_, (void*)s.state_, buff_len); +} + +void Status::MoveFrom(Status &s) { + delete state_; + state_ = s.state_; + s.state_ = nullptr; } std::string Status::ToString() const { - if (state_ == nullptr) return "OK"; - char tmp[32]; - const char* type; + if (state_ == nullptr) { + return "OK"; + } + + std::string result; switch (code()) { case DB_SUCCESS: - type = "OK"; + result = "OK "; break; case DB_ERROR: - type = "Error: "; + result = "Error: "; break; case DB_META_TRANSACTION_FAILED: - type = "DBTransactionError: "; + result = "Database error: "; break; case DB_NOT_FOUND: - type = "NotFound: "; + result = "Not found: "; break; case DB_ALREADY_EXIST: - type = "AlreadyExist: "; + result = "Already exist: "; break; case DB_INVALID_PATH: - type = "InvalidPath: "; + result = "Invalid path: "; break; default: - snprintf(tmp, sizeof(tmp), "Error code(0x%x): ", - static_cast(code())); - type = tmp; + result = "Error code(" + std::to_string(code()) + "): "; break; } - std::string result(type); uint32_t length = 0; memcpy(&length, state_ + CODE_WIDTH, sizeof(length)); - result.append(state_ + sizeof(length) + CODE_WIDTH, length); + if(length > 0) { + result.append(state_ + sizeof(length) + CODE_WIDTH, length); + } + return result; } -} // namespace engine } // namespace milvus } // namespace zilliz diff --git a/cpp/src/db/Status.h b/cpp/src/utils/Status.h similarity index 55% rename from cpp/src/db/Status.h rename to cpp/src/utils/Status.h index e3dae4c0ad..153ff61dc5 100644 --- a/cpp/src/db/Status.h +++ b/cpp/src/utils/Status.h @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. + #pragma once #include "utils/Error.h" @@ -23,7 +24,6 @@ namespace zilliz { namespace milvus { -namespace engine { class Status { public: @@ -31,51 +31,40 @@ class Status { Status(); ~Status(); - Status(const Status &rhs); + Status(const Status &s); Status & - operator=(const Status &rhs); + operator=(const Status &s); - Status(Status &&rhs) noexcept : state_(rhs.state_) { rhs.state_ = nullptr; } + Status(Status &&s); Status & - operator=(Status &&rhs_) noexcept; + operator=(Status &&s); static Status OK() { return Status(); } - bool ok() const { return state_ == nullptr || code() == DB_SUCCESS; } + bool + ok() const { return state_ == nullptr || code() == 0; } - std::string ToString() const; + std::string + ToString() const; - ErrorCode code() const { - return (state_ == nullptr) ? DB_SUCCESS : *(ErrorCode*)(state_); + ErrorCode + code() const { + return (state_ == nullptr) ? 0 : *(ErrorCode*)(state_); } - private: +private: + inline void + CopyFrom(const Status &s); + + inline void + MoveFrom(Status &s); + +private: const char *state_ = nullptr; - - static const char *CopyState(const char *s); - }; // Status -inline Status::Status(const Status &rhs) { - state_ = (rhs.state_ == nullptr) ? nullptr : CopyState(rhs.state_); -} - -inline Status &Status::operator=(const Status &rhs) { - if (state_ != rhs.state_) { - delete[] state_; - state_ = (rhs.state_ == nullptr) ? nullptr : CopyState(rhs.state_); - } - return *this; -} - -inline Status &Status::operator=(Status &&rhs) noexcept { - std::swap(state_, rhs.state_); - return *this; -} - -} // namespace engine } // namespace milvus } // namespace zilliz diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 00b6f64dbf..29440dcf1a 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -33,9 +33,9 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs) aux_source_directory(./ test_srcs) set(util_files + ${MILVUS_ENGINE_SRC}/utils/Status.cpp ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp - ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc - ${MILVUS_ENGINE_SRC}/utils/easylogging++.h) + ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 372dd336d6..a1fd14a8f4 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -154,7 +154,7 @@ TEST_F(DBTest, CONFIG_TEST) { TEST_F(DBTest, DB_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); + auto stat = db_->CreateTable(table_info); engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; @@ -230,7 +230,7 @@ TEST_F(DBTest, DB_TEST) { TEST_F(DBTest, SEARCH_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); + auto stat = db_->CreateTable(table_info); engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; @@ -296,7 +296,7 @@ TEST_F(DBTest, SEARCH_TEST) { TEST_F(DBTest, PRELOADTABLE_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); + auto stat = db_->CreateTable(table_info); engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; @@ -331,7 +331,7 @@ TEST_F(DBTest, SHUTDOWN_TEST) { db_->Stop(); engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); + auto stat = db_->CreateTable(table_info); ASSERT_FALSE(stat.ok()); stat = db_->DescribeTable(table_info); @@ -373,7 +373,7 @@ TEST_F(DBTest, SHUTDOWN_TEST) { TEST_F(DBTest, INDEX_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); + auto stat = db_->CreateTable(table_info); int64_t nb = VECTOR_COUNT; std::vector xb; @@ -403,7 +403,7 @@ TEST_F(DBTest, INDEX_TEST) { TEST_F(DBTest2, ARHIVE_DISK_CHECK) { engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); + auto stat = db_->CreateTable(table_info); std::vector table_schema_array; stat = db_->AllTables(table_schema_array); @@ -446,7 +446,7 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) { TEST_F(DBTest2, DELETE_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); + auto stat = db_->CreateTable(table_info); engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; @@ -480,7 +480,7 @@ TEST_F(DBTest2, DELETE_TEST) { TEST_F(DBTest2, DELETE_BY_RANGE_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); + auto stat = db_->CreateTable(table_info); engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 73f6d89f5e..4e958795ad 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -216,7 +216,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) { TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); + auto stat = db_->CreateTable(table_info); engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; @@ -230,8 +230,8 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) { int64_t nb = 100000; std::vector xb; BuildVectors(nb, xb); - engine::Status status = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); - ASSERT_TRUE(status.ok()); + stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + ASSERT_TRUE(stat.ok()); std::this_thread::sleep_for(std::chrono::seconds(3)); @@ -262,7 +262,7 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) { TEST_F(MemManagerTest2, INSERT_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); + auto stat = db_->CreateTable(table_info); engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; @@ -278,8 +278,8 @@ TEST_F(MemManagerTest2, INSERT_TEST) { std::vector xb; BuildVectors(nb, xb); engine::IDNumbers vector_ids; - engine::Status status = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); - ASSERT_TRUE(status.ok()); + stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + ASSERT_TRUE(stat.ok()); } auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); @@ -288,7 +288,7 @@ TEST_F(MemManagerTest2, INSERT_TEST) { TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); + auto stat = db_->CreateTable(table_info); engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; @@ -359,7 +359,7 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) { TEST_F(MemManagerTest2, VECTOR_IDS_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); + auto stat = db_->CreateTable(table_info); engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index 28ec0f2f9c..ff530b0a73 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -#include "db/Status.h" #include "db/Options.h" #include "db/meta/SqliteMetaImpl.h" #include "db/engine/EngineFactory.h" #include "db/Utils.h" +#include "utils/Status.h" #include "utils/Exception.h" #include "utils/easylogging++.h" @@ -30,13 +30,6 @@ using namespace zilliz::milvus; -namespace { - void CopyStatus(engine::Status& st1, engine::Status& st2) { - st1 = st2; - } - -} - TEST(DBMiscTest, EXCEPTION_TEST) { Exception ex1(""); std::string what = ex1.what(); @@ -47,36 +40,6 @@ TEST(DBMiscTest, EXCEPTION_TEST) { ASSERT_FALSE(what.empty()); } -TEST(DBMiscTest, STATUS_TEST) { - engine::Status status = engine::Status::OK(); - std::string str = status.ToString(); - ASSERT_FALSE(str.empty()); - - status = engine::Status(DB_ERROR, "mistake"); - ASSERT_EQ(status.code(), DB_ERROR); - str = status.ToString(); - ASSERT_FALSE(str.empty()); - - status = engine::Status(DB_NOT_FOUND, "mistake"); - ASSERT_EQ(status.code(), DB_NOT_FOUND); - str = status.ToString(); - ASSERT_FALSE(str.empty()); - - status = engine::Status(DB_ALREADY_EXIST, "mistake"); - ASSERT_EQ(status.code(), DB_ALREADY_EXIST); - str = status.ToString(); - ASSERT_FALSE(str.empty()); - - status = engine::Status(DB_META_TRANSACTION_FAILED, "mistake"); - ASSERT_EQ(status.code(), DB_META_TRANSACTION_FAILED); - str = status.ToString(); - ASSERT_FALSE(str.empty()); - - engine::Status status_copy = engine::Status::OK(); - CopyStatus(status_copy, status); - ASSERT_EQ(status.code(), DB_META_TRANSACTION_FAILED); -} - TEST(DBMiscTest, OPTIONS_TEST) { try { engine::ArchiveConf archive("$$##"); diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp index 35f086b033..9d99b4095b 100644 --- a/cpp/unittest/db/mysql_db_test.cpp +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -59,7 +59,7 @@ namespace { TEST_F(MySqlDBTest, DB_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); + auto stat = db_->CreateTable(table_info); engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; @@ -144,7 +144,7 @@ TEST_F(MySqlDBTest, DB_TEST) { TEST_F(MySqlDBTest, SEARCH_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); + auto stat = db_->CreateTable(table_info); engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; @@ -196,7 +196,7 @@ TEST_F(MySqlDBTest, SEARCH_TEST) { TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) { engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); + auto stat = db_->CreateTable(table_info); std::vector table_schema_array; stat = db_->AllTables(table_schema_array); @@ -241,7 +241,7 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) { TEST_F(MySqlDBTest, DELETE_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); + auto stat = db_->CreateTable(table_info); // std::cout << stat.ToString() << std::endl; engine::meta::TableSchema table_info_get; diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 07f27e1e67..dddba2906a 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -37,9 +37,9 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/src/metrics metrics_src) aux_source_directory(./ test_srcs) set(util_files + ${MILVUS_ENGINE_SRC}/utils/Status.cpp ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp - ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc - ${MILVUS_ENGINE_SRC}/utils/easylogging++.h) + ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) diff --git a/cpp/unittest/metrics/metrics_test.cpp b/cpp/unittest/metrics/metrics_test.cpp index 2bd06ed091..636e2ffbb1 100644 --- a/cpp/unittest/metrics/metrics_test.cpp +++ b/cpp/unittest/metrics/metrics_test.cpp @@ -57,7 +57,7 @@ TEST_F(MetricTest, METRIC_TEST) { engine::meta::TableSchema group_info; group_info.dimension_ = group_dim; group_info.table_id_ = group_name; - engine::Status stat = db_->CreateTable(group_info); + auto stat = db_->CreateTable(group_info); engine::meta::TableSchema group_info_get; group_info_get.table_id_ = group_name; @@ -134,9 +134,9 @@ TEST_F(MetricTest, METRIC_TEST) { }; TEST_F(MetricTest, COLLECTOR_METRICS_TEST){ - engine::Status status = engine::Status::OK(); + auto status = Status::OK(); server::CollectInsertMetrics insert_metrics0(0, status); - status = engine::Status(DB_ERROR, "error"); + status = Status(DB_ERROR, "error"); server::CollectInsertMetrics insert_metrics1(0, status); server::CollectQueryMetrics query_metrics(10); diff --git a/cpp/unittest/metrics/utils.h b/cpp/unittest/metrics/utils.h index ef25654ed0..7f093dd091 100644 --- a/cpp/unittest/metrics/utils.h +++ b/cpp/unittest/metrics/utils.h @@ -42,7 +42,7 @@ #define STOP_TIMER(name) #endif -void ASSERT_STATS(zilliz::milvus::engine::Status& stat); +void ASSERT_STATS(zilliz::milvus::Status& stat); //class TestEnv : public ::testing::Environment { //public: diff --git a/cpp/unittest/scheduler/CMakeLists.txt b/cpp/unittest/scheduler/CMakeLists.txt index 9ada727f8e..7359420e16 100644 --- a/cpp/unittest/scheduler/CMakeLists.txt +++ b/cpp/unittest/scheduler/CMakeLists.txt @@ -36,9 +36,9 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs) aux_source_directory(./ test_srcs) set(util_files + ${MILVUS_ENGINE_SRC}/utils/Status.cpp ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp - ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc - ${MILVUS_ENGINE_SRC}/utils/easylogging++.h) + ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc) set(db_scheduler_srcs ${scheduler_files} diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index 6bbe2183ce..8e06859929 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -54,9 +54,9 @@ set(db_scheduler_srcs ) set(util_files + ${MILVUS_ENGINE_SRC}/utils/Status.cpp ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp - ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc - ${MILVUS_ENGINE_SRC}/utils/easylogging++.h) + ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc) set(db_src ${config_files} diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index 3849f79d19..cfbcbab683 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -37,6 +37,10 @@ namespace { static const char* LOG_FILE_PATH = "./milvus/conf/log_config.conf"; +void CopyStatus(Status& st1, Status& st2) { + st1 = st2; +} + } TEST(UtilTest, EXCEPTION_TEST) { @@ -183,6 +187,44 @@ TEST(UtilTest, TIMERECORDER_TEST) { } } +TEST(UtilTest, STATUS_TEST) { + auto status = Status::OK(); + std::string str = status.ToString(); + ASSERT_FALSE(str.empty()); + + status = Status(DB_ERROR, "mistake"); + ASSERT_EQ(status.code(), DB_ERROR); + str = status.ToString(); + ASSERT_FALSE(str.empty()); + + status = Status(DB_NOT_FOUND, "mistake"); + ASSERT_EQ(status.code(), DB_NOT_FOUND); + str = status.ToString(); + ASSERT_FALSE(str.empty()); + + status = Status(DB_ALREADY_EXIST, "mistake"); + ASSERT_EQ(status.code(), DB_ALREADY_EXIST); + str = status.ToString(); + ASSERT_FALSE(str.empty()); + + status = Status(DB_META_TRANSACTION_FAILED, "mistake"); + ASSERT_EQ(status.code(), DB_META_TRANSACTION_FAILED); + str = status.ToString(); + ASSERT_FALSE(str.empty()); + + auto status_copy = Status::OK(); + CopyStatus(status_copy, status); + ASSERT_EQ(status.code(), DB_META_TRANSACTION_FAILED); + + auto status_ref(status); + ASSERT_EQ(status_ref.code(), status.code()); + ASSERT_EQ(status_ref.ToString(), status.ToString()); + + auto status_move = std::move(status); + ASSERT_EQ(status_move.code(), status_ref.code()); + ASSERT_EQ(status_move.ToString(), status_ref.ToString()); +} + TEST(ValidationUtilTest, VALIDATE_TABLENAME_TEST) { std::string table_name = "Normal123_"; ErrorCode res = server::ValidationUtil::ValidateTableName(table_name); From a5ad6a19ed3b5ba909d7c029da657e8f37df99a9 Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Tue, 17 Sep 2019 18:49:58 +0800 Subject: [PATCH 019/217] Update CHANGELOG.md Former-commit-id: df346c406bdecbf100761b5aa3de710ba8325a46 --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index de7d8306fd..7106cc2d08 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -16,6 +16,7 @@ Please mark all change in change log and use the ticket from JIRA. ## Task - MS-554 - Change license to Apache 2.0 +- MS-561 - Add contributing guidelines, code of conduct and Readme docs # Milvus 0.4.0 (2019-09-12) From edd566b046b8ded9ee099a717e8ce5639cea8c7c Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Tue, 17 Sep 2019 18:53:43 +0800 Subject: [PATCH 020/217] Update CHANGELOG.md Former-commit-id: d822441834b09e510ed386ee64f436ac6c15e6af --- cpp/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 7106cc2d08..3ffbee2ff9 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -16,7 +16,7 @@ Please mark all change in change log and use the ticket from JIRA. ## Task - MS-554 - Change license to Apache 2.0 -- MS-561 - Add contributing guidelines, code of conduct and Readme docs +- MS-561 - Add contributing guidelines, code of conduct and README docs # Milvus 0.4.0 (2019-09-12) From 0f955f5183025ead8cc37dc54a0791c447689f66 Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Tue, 17 Sep 2019 18:56:33 +0800 Subject: [PATCH 021/217] Update README.md Former-commit-id: 9a58a8165f60984fb06ba98f863c571989052dde --- cpp/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/README.md b/cpp/README.md index 1cdff6bee8..3029317810 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -84,7 +84,8 @@ If you encounter the following error message, please reinstall CMake with curl: -1. ```shell +1. Install curl development files: + ```shell CentOS 7: $ yum install curl-devel Ubuntu 16.04: From 918f299c6916c1ebbef202a2c6f0339397c72a2c Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Tue, 17 Sep 2019 18:57:14 +0800 Subject: [PATCH 022/217] Update README.md Former-commit-id: bf982801e6f7b3b82ff6ae1af98b66a29bf8b55c --- cpp/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/README.md b/cpp/README.md index 3029317810..aa62a256d9 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -93,7 +93,6 @@ please reinstall CMake with curl: ``` 2. Install [CMake 3.14](https://github.com/Kitware/CMake/releases/download/v3.14.6/cmake-3.14.6.tar.gz): - ```shell $ ./bootstrap --system-curl $ make From 5a2743946b68de2c155af4315e6b0b5c7604be74 Mon Sep 17 00:00:00 2001 From: wxyu Date: Wed, 18 Sep 2019 10:42:01 +0800 Subject: [PATCH 023/217] MS-562 Add JobMgr and TaskCreator in Scheduler Former-commit-id: bbaf2b649843e86fc7bbd28ec61e5a990fc5951f --- cpp/CHANGELOG.md | 1 + cpp/src/scheduler/JobMgr.cpp | 89 +++++++++++++++++++++++++++++++ cpp/src/scheduler/JobMgr.h | 78 +++++++++++++++++++++++++++ cpp/src/scheduler/TaskCreator.cpp | 65 ++++++++++++++++++++++ cpp/src/scheduler/TaskCreator.h | 61 +++++++++++++++++++++ cpp/src/scheduler/job/DeleteJob.h | 1 + cpp/src/scheduler/job/Job.h | 1 + cpp/src/scheduler/job/SearchJob.h | 6 +++ 8 files changed, 302 insertions(+) create mode 100644 cpp/src/scheduler/JobMgr.cpp create mode 100644 cpp/src/scheduler/JobMgr.h create mode 100644 cpp/src/scheduler/TaskCreator.cpp create mode 100644 cpp/src/scheduler/TaskCreator.h diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 1f0aedef85..c5d27e3461 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -12,6 +12,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-557 - Merge Log.h - MS-556 - Add Job Definition in Scheduler - MS-558 - Refine status code +- MS-562 - Add JobMgr and TaskCreator in Scheduler ## New Feature diff --git a/cpp/src/scheduler/JobMgr.cpp b/cpp/src/scheduler/JobMgr.cpp new file mode 100644 index 0000000000..c5c14c1b41 --- /dev/null +++ b/cpp/src/scheduler/JobMgr.cpp @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "JobMgr.h" +#include "task/Task.h" +#include "TaskCreator.h" + + +namespace zilliz { +namespace milvus { +namespace scheduler { + +using namespace engine; + +JobMgr::JobMgr(ResourceMgrPtr res_mgr) + : res_mgr_(std::move(res_mgr)) {} + +void +JobMgr::Start() { + if (not running_) { + worker_thread_ = std::thread(&JobMgr::worker_function, this); + running_ = true; + } +} + +void +JobMgr::Stop() { + if (running_) { + this->Put(nullptr); + worker_thread_.join(); + running_ = false; + } +} + +void +JobMgr::Put(const JobPtr &job) { + { + std::lock_guard lock(mutex_); + queue_.push(job); + } + cv_.notify_one(); +} + +void +JobMgr::worker_function() { + while (running_) { + std::unique_lock lock(mutex_); + cv_.wait(lock, [this] { return !queue_.empty(); }); + auto job = queue_.front(); + queue_.pop(); + lock.unlock(); + if (job == nullptr) { + break; + } + + auto tasks = build_task(job); + auto disk_list = res_mgr_->GetDiskResources(); + if (!disk_list.empty()) { + if (auto disk = disk_list[0].lock()) { + for (auto &task : tasks) { + disk->task_table().Put(task); + } + } + } + } +} + +std::vector +JobMgr::build_task(const JobPtr &job) { + return TaskCreator::Create(job); +} + +} +} +} diff --git a/cpp/src/scheduler/JobMgr.h b/cpp/src/scheduler/JobMgr.h new file mode 100644 index 0000000000..f096ab6121 --- /dev/null +++ b/cpp/src/scheduler/JobMgr.h @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "job/Job.h" +#include "task/Task.h" +#include "ResourceMgr.h" + + +namespace zilliz { +namespace milvus { +namespace scheduler { + +using engine::TaskPtr; +using engine::ResourceMgrPtr; + +class JobMgr { +public: + explicit + JobMgr(ResourceMgrPtr res_mgr); + + void + Start(); + + void + Stop(); + +public: + void + Put(const JobPtr &job); + +private: + void + worker_function(); + + std::vector + build_task(const JobPtr &job); + +private: + bool running_ = false; + std::queue queue_; + + std::thread worker_thread_; + + std::mutex mutex_; + std::condition_variable cv_; + + ResourceMgrPtr res_mgr_ = nullptr; +}; + +} +} +} diff --git a/cpp/src/scheduler/TaskCreator.cpp b/cpp/src/scheduler/TaskCreator.cpp new file mode 100644 index 0000000000..afe0d9d868 --- /dev/null +++ b/cpp/src/scheduler/TaskCreator.cpp @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "TaskCreator.h" + + +namespace zilliz { +namespace milvus { +namespace scheduler { + +std::vector +TaskCreator::Create(const JobPtr &job) { + switch (job->type()) { + case JobType::SEARCH: { + return Create(std::static_pointer_cast(job)); + } + case JobType::DELETE: { + return Create(std::static_pointer_cast(job)); + } + default: { + // TODO: error + return std::vector(); + } + } +} + +std::vector +TaskCreator::Create(const SearchJobPtr &job) { + std::vector tasks; + for (auto &index_file : job->index_files()) { + auto task = std::make_shared(index_file.second); + tasks.emplace_back(task); + } + + return tasks; +} + +std::vector +TaskCreator::Create(const DeleteJobPtr &job) { + std::vector tasks; +// auto task = std::make_shared(job); +// tasks.emplace_back(task); + + return tasks; +} + + +} +} +} + diff --git a/cpp/src/scheduler/TaskCreator.h b/cpp/src/scheduler/TaskCreator.h new file mode 100644 index 0000000000..b0e947600d --- /dev/null +++ b/cpp/src/scheduler/TaskCreator.h @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "job/Job.h" +#include "job/SearchJob.h" +#include "job/DeleteJob.h" +#include "task/Task.h" +#include "task/SearchTask.h" +#include "task/DeleteTask.h" + + +namespace zilliz { +namespace milvus { +namespace scheduler { + +using engine::TaskPtr; +using engine::XSearchTask; +using engine::XDeleteTask; + +class TaskCreator { +public: + static std::vector + Create(const JobPtr &job); + +public: + static std::vector + Create(const SearchJobPtr &job); + + static std::vector + Create(const DeleteJobPtr &job); +}; + +} +} +} diff --git a/cpp/src/scheduler/job/DeleteJob.h b/cpp/src/scheduler/job/DeleteJob.h index 7410577563..d82262b235 100644 --- a/cpp/src/scheduler/job/DeleteJob.h +++ b/cpp/src/scheduler/job/DeleteJob.h @@ -14,6 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +#pragma once #include #include diff --git a/cpp/src/scheduler/job/Job.h b/cpp/src/scheduler/job/Job.h index e79dd51e71..845a5dc165 100644 --- a/cpp/src/scheduler/job/Job.h +++ b/cpp/src/scheduler/job/Job.h @@ -14,6 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +#pragma once #include #include diff --git a/cpp/src/scheduler/job/SearchJob.h b/cpp/src/scheduler/job/SearchJob.h index 4530c6485b..ed6531767c 100644 --- a/cpp/src/scheduler/job/SearchJob.h +++ b/cpp/src/scheduler/job/SearchJob.h @@ -14,6 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +#pragma once #include #include @@ -78,6 +79,11 @@ public: return vectors_; } + Id2IndexMap & + index_files() { + return index_files_; + } + private: uint64_t topk_ = 0; uint64_t nq_ = 0; From 69ce92dcc58884af741320c38cc67e928a85fbc3 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 18 Sep 2019 12:49:29 +0800 Subject: [PATCH 024/217] MS-558 refine status code Former-commit-id: ebb5ec55a58cfce4385f39d186e86d858eab1bf0 --- cpp/coverage.sh | 2 + cpp/src/db/Utils.cpp | 18 +- cpp/src/db/insert/MemTable.cpp | 4 +- cpp/src/db/insert/MemTableFile.cpp | 2 +- .../examples/grpcsimple/src/ClientTest.cpp | 24 +- cpp/src/sdk/include/Status.h | 14 +- cpp/src/sdk/interface/Status.cpp | 56 +-- cpp/src/server/DBWrapper.cpp | 24 +- cpp/src/server/DBWrapper.h | 6 +- cpp/src/server/Server.cpp | 21 +- cpp/src/server/Server.h | 4 +- cpp/src/server/ServerConfig.cpp | 108 ++--- cpp/src/server/ServerConfig.h | 18 +- cpp/src/server/grpc_impl/GrpcMilvusServer.cpp | 7 +- cpp/src/server/grpc_impl/GrpcMilvusServer.h | 6 +- .../server/grpc_impl/GrpcRequestScheduler.cpp | 55 ++- .../server/grpc_impl/GrpcRequestScheduler.h | 20 +- cpp/src/server/grpc_impl/GrpcRequestTask.cpp | 406 +++++++++--------- cpp/src/server/grpc_impl/GrpcRequestTask.h | 30 +- cpp/src/utils/CommonUtil.cpp | 29 +- cpp/src/utils/CommonUtil.h | 7 +- cpp/src/utils/Status.cpp | 44 +- cpp/src/utils/Status.h | 17 +- cpp/src/utils/StringHelpFunctions.cpp | 24 +- cpp/src/utils/StringHelpFunctions.h | 16 +- cpp/src/utils/ValidationUtil.cpp | 148 ++++--- cpp/src/utils/ValidationUtil.h | 32 +- cpp/unittest/server/config_test.cpp | 24 +- cpp/unittest/server/rpc_test.cpp | 16 +- cpp/unittest/server/util_test.cpp | 140 +++--- 30 files changed, 665 insertions(+), 657 deletions(-) diff --git a/cpp/coverage.sh b/cpp/coverage.sh index 64f60961a8..f1c42ede61 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -104,6 +104,8 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \ "src/server/Server.cpp"\ "src/server/DBWrapper.cpp"\ "src/server/grpc_impl/GrpcMilvusServer.cpp"\ + "src/utils/easylogging++.h"\ + "src/utils/easylogging++.cc"\ # gen html report ${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/ \ No newline at end of file diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index cbc3ba45b6..cbed4c6646 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -80,17 +80,17 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id std::string db_path = options.path; std::string table_path = db_path + TABLES_FOLDER + table_id; auto status = server::CommonUtil::CreateDirectory(table_path); - if (status != 0) { - ENGINE_LOG_ERROR << "Create directory " << table_path << " Error"; - return Status(DB_ERROR, "Failed to create table path"); + if (!status.ok()) { + ENGINE_LOG_ERROR << status.message(); + return status; } for(auto& path : options.slave_paths) { table_path = path + TABLES_FOLDER + table_id; status = server::CommonUtil::CreateDirectory(table_path); - if (status != 0) { - ENGINE_LOG_ERROR << "Create directory " << table_path << " Error"; - return Status(DB_ERROR, "Failed to create table path"); + if (!status.ok()) { + ENGINE_LOG_ERROR << status.message(); + return status; } } @@ -120,9 +120,9 @@ Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& std::string parent_path = GetTableFileParentFolder(options, table_file); auto status = server::CommonUtil::CreateDirectory(parent_path); - if (status != 0) { - ENGINE_LOG_ERROR << "Create directory " << parent_path << " Error"; - return Status(DB_ERROR, "Failed to create partition directory"); + if (!status.ok()) { + ENGINE_LOG_ERROR << status.message(); + return status; } table_file.location_ = parent_path + "/" + table_file.file_id_; diff --git a/cpp/src/db/insert/MemTable.cpp b/cpp/src/db/insert/MemTable.cpp index 6bd5274d9e..7f4f497ba1 100644 --- a/cpp/src/db/insert/MemTable.cpp +++ b/cpp/src/db/insert/MemTable.cpp @@ -54,7 +54,7 @@ Status MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) { } if (!status.ok()) { - std::string err_msg = "MemTable::Add failed: " + status.ToString(); + std::string err_msg = "Insert failed: " + status.ToString(); ENGINE_LOG_ERROR << err_msg; return Status(DB_ERROR, err_msg); } @@ -74,7 +74,7 @@ Status MemTable::Serialize() { for (auto mem_table_file = mem_table_file_list_.begin(); mem_table_file != mem_table_file_list_.end();) { auto status = (*mem_table_file)->Serialize(); if (!status.ok()) { - std::string err_msg = "MemTable::Serialize failed: " + status.ToString(); + std::string err_msg = "Insert data serialize failed: " + status.ToString(); ENGINE_LOG_ERROR << err_msg; return Status(DB_ERROR, err_msg); } diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index 63a968986d..57ef39c670 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -67,7 +67,7 @@ Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) { std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " + std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_; ENGINE_LOG_ERROR << err_msg; - return Status(DB_ERROR, "not able to create table file"); + return Status(DB_ERROR, "Not able to create table file"); } size_t single_vector_mem_size = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE; diff --git a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp index f0f7cf5292..ff3c6edaf5 100644 --- a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp @@ -190,7 +190,7 @@ void DoSearch(std::shared_ptr conn, { TimeRecorder rc(phase_name); Status stat = conn->Search(TABLE_NAME, record_array, query_range_array, TOP_K, 32, topk_query_result_array); - std::cout << "SearchVector function call status: " << stat.ToString() << std::endl; + std::cout << "SearchVector function call status: " << stat.message() << std::endl; } auto finish = std::chrono::high_resolution_clock::now(); std::cout << "SEARCHVECTOR COST: " << std::chrono::duration_cast>(finish - start).count() << "s\n"; @@ -207,7 +207,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { {//connect server ConnectParam param = {address, port}; Status stat = conn->Connect(param); - std::cout << "Connect function call status: " << stat.ToString() << std::endl; + std::cout << "Connect function call status: " << stat.message() << std::endl; } {//server version @@ -223,7 +223,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { { std::vector tables; Status stat = conn->ShowTables(tables); - std::cout << "ShowTables function call status: " << stat.ToString() << std::endl; + std::cout << "ShowTables function call status: " << stat.message() << std::endl; std::cout << "All tables: " << std::endl; for(auto& table : tables) { int64_t row_count = 0; @@ -236,7 +236,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { {//create table TableSchema tb_schema = BuildTableSchema(); Status stat = conn->CreateTable(tb_schema); - std::cout << "CreateTable function call status: " << stat.ToString() << std::endl; + std::cout << "CreateTable function call status: " << stat.message() << std::endl; PrintTableSchema(tb_schema); bool has_table = conn->HasTable(tb_schema.table_name); @@ -248,7 +248,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { {//describe table TableSchema tb_schema; Status stat = conn->DescribeTable(TABLE_NAME, tb_schema); - std::cout << "DescribeTable function call status: " << stat.ToString() << std::endl; + std::cout << "DescribeTable function call status: " << stat.message() << std::endl; PrintTableSchema(tb_schema); } @@ -279,7 +279,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::cout << "InsertVector cost: " << std::chrono::duration_cast>(finish - start).count() << "s\n"; - std::cout << "InsertVector function call status: " << stat.ToString() << std::endl; + std::cout << "InsertVector function call status: " << stat.message() << std::endl; std::cout << "Returned id array count: " << record_ids.size() << std::endl; if(search_record_array.size() < NQ) { @@ -305,16 +305,16 @@ ClientTest::Test(const std::string& address, const std::string& port) { index.index_type = IndexType::gpu_ivfsq8; index.nlist = 16384; Status stat = conn->CreateIndex(index); - std::cout << "CreateIndex function call status: " << stat.ToString() << std::endl; + std::cout << "CreateIndex function call status: " << stat.message() << std::endl; IndexParam index2; stat = conn->DescribeIndex(TABLE_NAME, index2); - std::cout << "DescribeIndex function call status: " << stat.ToString() << std::endl; + std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; } {//preload table Status stat = conn->PreloadTable(TABLE_NAME); - std::cout << "PreloadTable function call status: " << stat.ToString() << std::endl; + std::cout << "PreloadTable function call status: " << stat.message() << std::endl; } {//search vectors after build index finish @@ -326,7 +326,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { {//delete index Status stat = conn->DropIndex(TABLE_NAME); - std::cout << "DropIndex function call status: " << stat.ToString() << std::endl; + std::cout << "DropIndex function call status: " << stat.message() << std::endl; int64_t row_count = 0; stat = conn->CountTable(TABLE_NAME, row_count); @@ -339,12 +339,12 @@ ClientTest::Test(const std::string& address, const std::string& port) { rg.end_value = CurrentTmDate(-3); Status stat = conn->DeleteByRange(rg, TABLE_NAME); - std::cout << "DeleteByRange function call status: " << stat.ToString() << std::endl; + std::cout << "DeleteByRange function call status: " << stat.message() << std::endl; } {//delete table Status stat = conn->DropTable(TABLE_NAME); - std::cout << "DeleteTable function call status: " << stat.ToString() << std::endl; + std::cout << "DeleteTable function call status: " << stat.message() << std::endl; } {//server status diff --git a/cpp/src/sdk/include/Status.h b/cpp/src/sdk/include/Status.h index 145d09d47a..cdb5f54ede 100644 --- a/cpp/src/sdk/include/Status.h +++ b/cpp/src/sdk/include/Status.h @@ -40,14 +40,12 @@ enum class StatusCode { ServerFailed, }; -using ErrorCode = StatusCode; - /** * @brief Status for SDK interface return */ class Status { public: - Status(ErrorCode code, const std::string &msg); + Status(StatusCode code, const std::string &msg); Status(); ~Status(); @@ -67,14 +65,14 @@ public: bool ok() const { return state_ == nullptr || code() == StatusCode::OK; } - std::string - ToString() const; - - ErrorCode + StatusCode code() const { - return (state_ == nullptr) ? StatusCode::OK : *(ErrorCode*)(state_); + return (state_ == nullptr) ? StatusCode::OK : *(StatusCode*)(state_); } + std::string + message() const; + private: inline void CopyFrom(const Status &s); diff --git a/cpp/src/sdk/interface/Status.cpp b/cpp/src/sdk/interface/Status.cpp index b04c6581fe..e6fd06c72e 100644 --- a/cpp/src/sdk/interface/Status.cpp +++ b/cpp/src/sdk/interface/Status.cpp @@ -21,9 +21,9 @@ namespace milvus { -constexpr int CODE_WIDTH = sizeof(ErrorCode); +constexpr int CODE_WIDTH = sizeof(StatusCode); -Status::Status(ErrorCode code, const std::string& msg) { +Status::Status(StatusCode code, const std::string& msg) { //4 bytes store code //4 bytes store message length //the left bytes store message string @@ -50,7 +50,8 @@ Status::Status(const Status &s) CopyFrom(s); } -Status &Status::operator=(const Status &s) { +Status& +Status::operator=(const Status &s) { CopyFrom(s); return *this; } @@ -60,12 +61,14 @@ Status::Status(Status &&s) MoveFrom(s); } -Status &Status::operator=(Status &&s) { +Status& +Status::operator=(Status &&s) { MoveFrom(s); return *this; } -void Status::CopyFrom(const Status &s) { +void +Status::CopyFrom(const Status &s) { delete state_; state_ = nullptr; if(s.state_ == nullptr) { @@ -73,58 +76,33 @@ void Status::CopyFrom(const Status &s) { } uint32_t length = 0; - std::memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length)); + memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length)); int buff_len = length + sizeof(length) + CODE_WIDTH; state_ = new char[buff_len]; memcpy((void*)state_, (void*)s.state_, buff_len); } -void Status::MoveFrom(Status &s) { +void +Status::MoveFrom(Status &s) { delete state_; state_ = s.state_; s.state_ = nullptr; } -std::string Status::ToString() const { +std::string +Status::message() const { if (state_ == nullptr) { - return "OK"; - } - - std::string result; - switch (code()) { - case StatusCode::OK: - result = "OK "; - break; - case StatusCode::UnknownError: - result = "Unknown error: "; - break; - case StatusCode::NotSupported: - result = "Not supported: "; - break; - case StatusCode::NotConnected: - result = "Not connected: "; - break; - case StatusCode::InvalidAgument: - result = "Invalid agument: "; - break; - case StatusCode::RPCFailed: - result = "Remote call failed: "; - break; - case StatusCode::ServerFailed: - result = "Service error: "; - break; - default: - result = "Error code(" + std::to_string((int)code()) + "): "; - break; + return ""; } + std::string msg; uint32_t length = 0; memcpy(&length, state_ + CODE_WIDTH, sizeof(length)); if(length > 0) { - result.append(state_ + sizeof(length) + CODE_WIDTH, length); + msg.append(state_ + sizeof(length) + CODE_WIDTH, length); } - return result; + return msg; } } diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index d4a145ff37..508d9552b1 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -33,7 +33,7 @@ DBWrapper::DBWrapper() { } -ErrorCode DBWrapper::StartService() { +Status DBWrapper::StartService() { //db config zilliz::milvus::engine::Options opt; ConfigNode& db_config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); @@ -60,7 +60,7 @@ ErrorCode DBWrapper::StartService() { opt.mode = zilliz::milvus::engine::Options::MODE::READ_ONLY; } else { - std::cout << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" << std::endl; + std::cerr << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" << std::endl; kill(0, SIGUSR1); } @@ -91,16 +91,16 @@ ErrorCode DBWrapper::StartService() { opt.meta.archive_conf.SetCriterias(criterial); //create db root folder - ErrorCode err = CommonUtil::CreateDirectory(opt.meta.path); - if(err != SERVER_SUCCESS) { - std::cout << "ERROR! Failed to create database root path: " << opt.meta.path << std::endl; + Status status = CommonUtil::CreateDirectory(opt.meta.path); + if(!status.ok()) { + std::cerr << "ERROR! Failed to create database root path: " << opt.meta.path << std::endl; kill(0, SIGUSR1); } for(auto& path : opt.meta.slave_paths) { - err = CommonUtil::CreateDirectory(path); - if(err != SERVER_SUCCESS) { - std::cout << "ERROR! Failed to create database slave path: " << path << std::endl; + status = CommonUtil::CreateDirectory(path); + if(!status.ok()) { + std::cerr << "ERROR! Failed to create database slave path: " << path << std::endl; kill(0, SIGUSR1); } } @@ -114,21 +114,21 @@ ErrorCode DBWrapper::StartService() { } if(db_ == nullptr) { - std::cout << "ERROR! Failed to open database: " << msg << std::endl; + std::cerr << "ERROR! Failed to open database: " << msg << std::endl; kill(0, SIGUSR1); } db_->Start(); - return SERVER_SUCCESS; + return Status::OK(); } -ErrorCode DBWrapper::StopService() { +Status DBWrapper::StopService() { if(db_) { db_->Stop(); } - return SERVER_SUCCESS; + return Status::OK(); } } diff --git a/cpp/src/server/DBWrapper.h b/cpp/src/server/DBWrapper.h index a321e3cdae..99b8f90500 100644 --- a/cpp/src/server/DBWrapper.h +++ b/cpp/src/server/DBWrapper.h @@ -17,7 +17,7 @@ #pragma once -#include "utils/Error.h" +#include "utils/Status.h" #include "db/DB.h" #include @@ -41,8 +41,8 @@ public: return GetInstance().EngineDB(); } - ErrorCode StartService(); - ErrorCode StopService(); + Status StartService(); + Status StopService(); engine::DBPtr EngineDB() { return db_; diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index f64ec009b9..e1e67a112f 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -138,11 +138,11 @@ Server::Daemonize() { if (!pid_filename_.empty()) { pid_fd = open(pid_filename_.c_str(), O_RDWR | O_CREAT, 0640); if (pid_fd < 0) { - std::cout << "Can't open filename: " + pid_filename_ + ", Error: " + strerror(errno); + std::cerr << "Can't open filename: " + pid_filename_ + ", Error: " + strerror(errno); exit(EXIT_FAILURE); } if (lockf(pid_fd, F_TLOCK, 0) < 0) { - std::cout << "Can't lock filename: " + pid_filename_ + ", Error: " + strerror(errno); + std::cerr << "Can't lock filename: " + pid_filename_ + ", Error: " + strerror(errno); exit(EXIT_FAILURE); } @@ -216,18 +216,18 @@ Server::Start() { void Server::Stop() { - std::cout << "Milvus server is going to shutdown ..." << std::endl; + std::cerr << "Milvus server is going to shutdown ..." << std::endl; // Unlock and close lockfile if (pid_fd != -1) { int ret = lockf(pid_fd, F_ULOCK, 0); if (ret != 0) { - std::cout << "Can't lock file: " << strerror(errno) << std::endl; + std::cerr << "Can't lock file: " << strerror(errno) << std::endl; exit(0); } ret = close(pid_fd); if (ret != 0) { - std::cout << "Can't close file: " << strerror(errno) << std::endl; + std::cerr << "Can't close file: " << strerror(errno) << std::endl; exit(0); } } @@ -236,24 +236,23 @@ Server::Stop() { if (!pid_filename_.empty()) { int ret = unlink(pid_filename_.c_str()); if (ret != 0) { - std::cout << "Can't unlink file: " << strerror(errno) << std::endl; + std::cerr << "Can't unlink file: " << strerror(errno) << std::endl; exit(0); } } - running_ = 0; - StopService(); - std::cout << "Milvus server is closed!" << std::endl; + std::cerr << "Milvus server is closed!" << std::endl; } ErrorCode Server::LoadConfig() { ServerConfig::GetInstance().LoadConfigFile(config_filename_); - ErrorCode err = ServerConfig::GetInstance().ValidateConfig(); - if (err != SERVER_SUCCESS) { + auto status = ServerConfig::GetInstance().ValidateConfig(); + if (!status.ok()) { + std::cerr << "Failed to load config file: " << config_filename_ << std::endl; exit(0); } diff --git a/cpp/src/server/Server.h b/cpp/src/server/Server.h index f66c8a314b..43e8b28028 100644 --- a/cpp/src/server/Server.h +++ b/cpp/src/server/Server.h @@ -17,7 +17,7 @@ #pragma once -#include "utils/Error.h" +#include "utils/Status.h" #include #include @@ -40,7 +40,6 @@ class Server { void Daemonize(); - static void HandleSignal(int signal); ErrorCode LoadConfig(); void StartService(); @@ -48,7 +47,6 @@ class Server { private: int64_t daemonized_ = 0; - int64_t running_ = 1; int pid_fd = -1; std::string pid_filename_; std::string config_filename_; diff --git a/cpp/src/server/ServerConfig.cpp b/cpp/src/server/ServerConfig.cpp index 988ee707b6..687490cf5e 100644 --- a/cpp/src/server/ServerConfig.cpp +++ b/cpp/src/server/ServerConfig.cpp @@ -41,17 +41,17 @@ ServerConfig::GetInstance() { return config; } -ErrorCode +Status ServerConfig::LoadConfigFile(const std::string &config_filename) { std::string filename = config_filename; if (filename.empty()) { - std::cout << "ERROR: a config file is required" << std::endl; + std::cerr << "ERROR: a config file is required" << std::endl; exit(1);//directly exit program if config file not specified } struct stat directoryStat; int statOK = stat(filename.c_str(), &directoryStat); if (statOK != 0) { - std::cout << "ERROR: " << filename << " not found!" << std::endl; + std::cerr << "ERROR: " << filename << " not found!" << std::endl; exit(1);//directly exit program if config file not found } @@ -59,43 +59,44 @@ ServerConfig::LoadConfigFile(const std::string &config_filename) { ConfigMgr *mgr = const_cast(ConfigMgr::GetInstance()); ErrorCode err = mgr->LoadConfigFile(filename); if (err != 0) { - std::cout << "Server failed to load config file" << std::endl; + std::cerr << "Server failed to load config file" << std::endl; exit(1);//directly exit program if the config file is illegal } } catch (YAML::Exception &e) { - std::cout << "Server failed to load config file: " << std::endl; - return SERVER_UNEXPECTED_ERROR; + std::cerr << "Server failed to load config file: " << std::endl; + exit(1);//directly exit program if the config file is illegal } - return SERVER_SUCCESS; + return Status::OK(); } -ErrorCode ServerConfig::ValidateConfig() { +Status +ServerConfig::ValidateConfig() { bool okay = true; - if (CheckServerConfig() != SERVER_SUCCESS) { + if (!CheckServerConfig().ok()) { okay = false; } - if (CheckDBConfig() != SERVER_SUCCESS) { + if (!CheckDBConfig().ok()) { okay = false; } - if (CheckMetricConfig() != SERVER_SUCCESS) { + if (!CheckMetricConfig().ok()) { okay = false; } - if (CheckCacheConfig() != SERVER_SUCCESS) { + if (!CheckCacheConfig().ok()) { okay = false; } - if (CheckEngineConfig() != SERVER_SUCCESS) { + if (!CheckEngineConfig().ok()) { okay = false; } - if (CheckResourceConfig() != SERVER_SUCCESS) { + if (!CheckResourceConfig().ok()) { okay = false; } - return (okay ? SERVER_SUCCESS : SERVER_INVALID_ARGUMENT); + return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Config validation not pass")); } -ErrorCode +Status ServerConfig::CheckServerConfig() { /* server_config: @@ -109,13 +110,13 @@ ServerConfig::CheckServerConfig() { ConfigNode server_config = GetConfig(CONFIG_SERVER); std::string ip_address = server_config.GetValue(CONFIG_SERVER_ADDRESS, "127.0.0.1"); - if (ValidationUtil::ValidateIpAddress(ip_address) != SERVER_SUCCESS) { + if (!ValidationUtil::ValidateIpAddress(ip_address).ok()) { std::cerr << "ERROR: invalid server IP address: " << ip_address << std::endl; okay = false; } std::string port_str = server_config.GetValue(CONFIG_SERVER_PORT, "19530"); - if (ValidationUtil::ValidateStringIsNumber(port_str) != SERVER_SUCCESS) { + if (!ValidationUtil::ValidateStringIsNumber(port_str).ok()) { std::cerr << "ERROR: port " << port_str << " is not a number" << std::endl; okay = false; } else { @@ -151,10 +152,10 @@ ServerConfig::CheckServerConfig() { okay = false; } - return (okay ? SERVER_SUCCESS : SERVER_INVALID_ARGUMENT); + return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Server config is illegal")); } -ErrorCode +Status ServerConfig::CheckDBConfig() { /* db_config: @@ -182,25 +183,25 @@ ServerConfig::CheckDBConfig() { } std::string db_backend_url = db_config.GetValue(CONFIG_DB_URL); - if (ValidationUtil::ValidateDbURI(db_backend_url) != SERVER_SUCCESS) { + if (!ValidationUtil::ValidateDbURI(db_backend_url).ok()) { std::cerr << "ERROR: invalid db_backend_url: " << db_backend_url << std::endl; okay = false; } std::string archive_disk_threshold_str = db_config.GetValue(CONFIG_DB_INSERT_BUFFER_SIZE, "0"); - if (ValidationUtil::ValidateStringIsNumber(archive_disk_threshold_str) != SERVER_SUCCESS) { + if (!ValidationUtil::ValidateStringIsNumber(archive_disk_threshold_str).ok()) { std::cerr << "ERROR: archive_disk_threshold " << archive_disk_threshold_str << " is not a number" << std::endl; okay = false; } std::string archive_days_threshold_str = db_config.GetValue(CONFIG_DB_INSERT_BUFFER_SIZE, "0"); - if (ValidationUtil::ValidateStringIsNumber(archive_days_threshold_str) != SERVER_SUCCESS) { + if (!ValidationUtil::ValidateStringIsNumber(archive_days_threshold_str).ok()) { std::cerr << "ERROR: archive_days_threshold " << archive_days_threshold_str << " is not a number" << std::endl; okay = false; } std::string insert_buffer_size_str = db_config.GetValue(CONFIG_DB_INSERT_BUFFER_SIZE, "4"); - if (ValidationUtil::ValidateStringIsNumber(insert_buffer_size_str) != SERVER_SUCCESS) { + if (!ValidationUtil::ValidateStringIsNumber(insert_buffer_size_str).ok()) { std::cerr << "ERROR: insert_buffer_size " << insert_buffer_size_str << " is not a number" << std::endl; okay = false; } @@ -216,21 +217,21 @@ ServerConfig::CheckDBConfig() { } std::string gpu_index_str = db_config.GetValue(CONFIG_DB_BUILD_INDEX_GPU, "0"); - if (ValidationUtil::ValidateStringIsNumber(gpu_index_str) != SERVER_SUCCESS) { + if (!ValidationUtil::ValidateStringIsNumber(gpu_index_str).ok()) { std::cerr << "ERROR: gpu_index " << gpu_index_str << " is not a number" << std::endl; okay = false; } else { int32_t gpu_index = std::stol(gpu_index_str); - if (ValidationUtil::ValidateGpuIndex(gpu_index) != SERVER_SUCCESS) { + if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { std::cerr << "ERROR: invalid gpu_index " << gpu_index_str << std::endl; okay = false; } } - return (okay ? SERVER_SUCCESS : SERVER_INVALID_ARGUMENT); + return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "DB config is illegal")); } -ErrorCode +Status ServerConfig::CheckMetricConfig() { /* metric_config: @@ -245,21 +246,21 @@ ServerConfig::CheckMetricConfig() { ConfigNode metric_config = GetConfig(CONFIG_METRIC); std::string is_startup_str = metric_config.GetValue(CONFIG_METRIC_IS_STARTUP, "off"); - if (ValidationUtil::ValidateStringIsBool(is_startup_str) != SERVER_SUCCESS) { + if (!ValidationUtil::ValidateStringIsBool(is_startup_str).ok()) { std::cerr << "ERROR: invalid is_startup config: " << is_startup_str << std::endl; okay = false; } std::string port_str = metric_config.GetChild(CONFIG_PROMETHEUS).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, "8080"); - if (ValidationUtil::ValidateStringIsNumber(port_str) != SERVER_SUCCESS) { + if (!ValidationUtil::ValidateStringIsNumber(port_str).ok()) { std::cerr << "ERROR: port specified in prometheus_config " << port_str << " is not a number" << std::endl; okay = false; } - return (okay ? SERVER_SUCCESS : SERVER_INVALID_ARGUMENT); + return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Metric config is illegal")); } -ErrorCode +Status ServerConfig::CheckCacheConfig() { /* cache_config: @@ -274,7 +275,7 @@ ServerConfig::CheckCacheConfig() { ConfigNode cache_config = GetConfig(CONFIG_CACHE); std::string cpu_cache_capacity_str = cache_config.GetValue(CONFIG_CPU_CACHE_CAPACITY, "16"); - if (ValidationUtil::ValidateStringIsNumber(cpu_cache_capacity_str) != SERVER_SUCCESS) { + if (!ValidationUtil::ValidateStringIsNumber(cpu_cache_capacity_str).ok()) { std::cerr << "ERROR: cpu_cache_capacity " << cpu_cache_capacity_str << " is not a number" << std::endl; okay = false; } @@ -301,7 +302,7 @@ ServerConfig::CheckCacheConfig() { std::string cpu_cache_free_percent_str = cache_config.GetValue(CACHE_FREE_PERCENT, "0.85"); double cpu_cache_free_percent; - if (ValidationUtil::ValidateStringIsDouble(cpu_cache_free_percent_str, cpu_cache_free_percent) != SERVER_SUCCESS) { + if (!ValidationUtil::ValidateStringIsDouble(cpu_cache_free_percent_str, cpu_cache_free_percent).ok()) { std::cerr << "ERROR: cpu_cache_free_percent " << cpu_cache_free_percent_str << " is not a double" << std::endl; okay = false; } @@ -311,13 +312,13 @@ ServerConfig::CheckCacheConfig() { } std::string insert_cache_immediately_str = cache_config.GetValue(CONFIG_INSERT_CACHE_IMMEDIATELY, "false"); - if (ValidationUtil::ValidateStringIsBool(insert_cache_immediately_str) != SERVER_SUCCESS) { + if (!ValidationUtil::ValidateStringIsBool(insert_cache_immediately_str).ok()) { std::cerr << "ERROR: invalid insert_cache_immediately config: " << insert_cache_immediately_str << std::endl; okay = false; } std::string gpu_cache_capacity_str = cache_config.GetValue(CONFIG_GPU_CACHE_CAPACITY, "0"); - if (ValidationUtil::ValidateStringIsNumber(gpu_cache_capacity_str) != SERVER_SUCCESS) { + if (!ValidationUtil::ValidateStringIsNumber(gpu_cache_capacity_str).ok()) { std::cerr << "ERROR: gpu_cache_capacity " << gpu_cache_capacity_str << " is not a number" << std::endl; okay = false; } @@ -326,7 +327,7 @@ ServerConfig::CheckCacheConfig() { gpu_cache_capacity *= GB; int gpu_index = GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, 0); size_t gpu_memory; - if (ValidationUtil::GetGpuMemory(gpu_index, gpu_memory) != SERVER_SUCCESS) { + if (!ValidationUtil::GetGpuMemory(gpu_index, gpu_memory).ok()) { std::cerr << "ERROR: could not get gpu memory for device " << gpu_index << std::endl; okay = false; } @@ -342,7 +343,7 @@ ServerConfig::CheckCacheConfig() { std::string gpu_cache_free_percent_str = cache_config.GetValue(GPU_CACHE_FREE_PERCENT, "0.85"); double gpu_cache_free_percent; - if (ValidationUtil::ValidateStringIsDouble(gpu_cache_free_percent_str, gpu_cache_free_percent) != SERVER_SUCCESS) { + if (!ValidationUtil::ValidateStringIsDouble(gpu_cache_free_percent_str, gpu_cache_free_percent).ok()) { std::cerr << "ERROR: gpu_cache_free_percent " << gpu_cache_free_percent_str << " is not a double" << std::endl; okay = false; } @@ -351,10 +352,10 @@ ServerConfig::CheckCacheConfig() { okay = false; } - return (okay ? SERVER_SUCCESS : SERVER_INVALID_ARGUMENT); + return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Cache config is illegal")); } -ErrorCode +Status ServerConfig::CheckEngineConfig() { /* engine_config: @@ -365,13 +366,13 @@ ServerConfig::CheckEngineConfig() { ConfigNode engine_config = GetConfig(CONFIG_ENGINE); std::string use_blas_threshold_str = engine_config.GetValue(CONFIG_DCBT, "20"); - if (ValidationUtil::ValidateStringIsNumber(use_blas_threshold_str) != SERVER_SUCCESS) { + if (!ValidationUtil::ValidateStringIsNumber(use_blas_threshold_str).ok()) { std::cerr << "ERROR: use_blas_threshold " << use_blas_threshold_str << " is not a number" << std::endl; okay = false; } std::string omp_thread_num_str = engine_config.GetValue(CONFIG_OMP_THREAD_NUM, "0"); - if (ValidationUtil::ValidateStringIsNumber(omp_thread_num_str) != SERVER_SUCCESS) { + if (!ValidationUtil::ValidateStringIsNumber(omp_thread_num_str).ok()) { std::cerr << "ERROR: omp_thread_num " << omp_thread_num_str << " is not a number" << std::endl; okay = false; } else { @@ -384,10 +385,10 @@ ServerConfig::CheckEngineConfig() { } } - return (okay ? SERVER_SUCCESS : SERVER_INVALID_ARGUMENT); + return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Engine config is illegal")); } -ErrorCode +Status ServerConfig::CheckResourceConfig() { /* resource_config: @@ -410,10 +411,10 @@ ServerConfig::CheckResourceConfig() { okay = false; } - return (okay ? SERVER_SUCCESS : SERVER_INVALID_ARGUMENT); + return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Resource config is illegal")); } -//ErrorCode +//Status //ServerConfig::CheckResourceConfig() { /* resource_config: @@ -484,7 +485,7 @@ ServerConfig::CheckResourceConfig() { // // std::string device_id_str = resource_conf.GetValue(CONFIG_RESOURCE_DEVICE_ID, "0"); // int32_t device_id = -1; -// if (ValidationUtil::ValidateStringIsNumber(device_id_str) != SERVER_SUCCESS) { +// if (!ValidationUtil::ValidateStringIsNumber(device_id_str).ok()) { // std::cerr << "ERROR: device_id " << device_id_str << " is not a number" << std::endl; // okay = false; // } else { @@ -492,7 +493,7 @@ ServerConfig::CheckResourceConfig() { // } // // std::string enable_executor_str = resource_conf.GetValue(CONFIG_RESOURCE_ENABLE_EXECUTOR, "off"); -// if (ValidationUtil::ValidateStringIsBool(enable_executor_str) != SERVER_SUCCESS) { +// if (!ValidationUtil::ValidateStringIsBool(enable_executor_str).ok()) { // std::cerr << "ERROR: invalid enable_executor config: " << enable_executor_str << std::endl; // okay = false; // } @@ -514,26 +515,26 @@ ServerConfig::CheckResourceConfig() { // hasExecutor = true; // } // std::string gpu_resource_num_str = resource_conf.GetValue(CONFIG_RESOURCE_NUM, "2"); -// if (ValidationUtil::ValidateStringIsNumber(gpu_resource_num_str) != SERVER_SUCCESS) { +// if (!ValidationUtil::ValidateStringIsNumber(gpu_resource_num_str).ok()) { // std::cerr << "ERROR: gpu_resource_num " << gpu_resource_num_str << " is not a number" << std::endl; // okay = false; // } // bool mem_valid = true; // std::string pinned_memory_str = resource_conf.GetValue(CONFIG_RESOURCE_PIN_MEMORY, "300"); -// if (ValidationUtil::ValidateStringIsNumber(pinned_memory_str) != SERVER_SUCCESS) { +// if (!ValidationUtil::ValidateStringIsNumber(pinned_memory_str).ok()) { // std::cerr << "ERROR: pinned_memory " << pinned_memory_str << " is not a number" << std::endl; // okay = false; // mem_valid = false; // } // std::string temp_memory_str = resource_conf.GetValue(CONFIG_RESOURCE_TEMP_MEMORY, "300"); -// if (ValidationUtil::ValidateStringIsNumber(temp_memory_str) != SERVER_SUCCESS) { +// if (!ValidationUtil::ValidateStringIsNumber(temp_memory_str).ok()) { // std::cerr << "ERROR: temp_memory " << temp_memory_str << " is not a number" << std::endl; // okay = false; // mem_valid = false; // } // if (mem_valid) { // size_t gpu_memory; -// if (ValidationUtil::GetGpuMemory(device_id, gpu_memory) != SERVER_SUCCESS) { +// if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) { // std::cerr << "ERROR: could not get gpu memory for device " << device_id << std::endl; // okay = false; // } @@ -592,8 +593,7 @@ ServerConfig::CheckResourceConfig() { // } // } // -// return (okay ? SERVER_SUCCESS : SERVER_INVALID_ARGUMENT); -// return SERVER_SUCCESS; +// return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Resource config is illegal")); //} void diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index d1f414b47b..d0f3e30d6b 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -17,7 +17,7 @@ #pragma once -#include "utils/Error.h" +#include "utils/Status.h" #include "config/ConfigNode.h" #include "yaml-cpp/yaml.h" @@ -78,20 +78,20 @@ class ServerConfig { public: static ServerConfig &GetInstance(); - ErrorCode LoadConfigFile(const std::string& config_filename); - ErrorCode ValidateConfig(); + Status LoadConfigFile(const std::string& config_filename); + Status ValidateConfig(); void PrintAll() const; ConfigNode GetConfig(const std::string& name) const; ConfigNode& GetConfig(const std::string& name); private: - ErrorCode CheckServerConfig(); - ErrorCode CheckDBConfig(); - ErrorCode CheckMetricConfig(); - ErrorCode CheckCacheConfig(); - ErrorCode CheckEngineConfig(); - ErrorCode CheckResourceConfig(); + Status CheckServerConfig(); + Status CheckDBConfig(); + Status CheckMetricConfig(); + Status CheckCacheConfig(); + Status CheckEngineConfig(); + Status CheckResourceConfig(); }; } diff --git a/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp b/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp index 7b17cd6e56..73f8a7573e 100644 --- a/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp @@ -58,7 +58,7 @@ class NoReusePortOption : public ::grpc::ServerBuilderOption { }; -void +Status GrpcMilvusServer::StartService() { if (server != nullptr) { std::cout << "stop service!\n"; @@ -92,13 +92,16 @@ GrpcMilvusServer::StartService() { server = builder.BuildAndStart(); server->Wait(); + return Status::OK(); } -void +Status GrpcMilvusServer::StopService() { if (server != nullptr) { server->Shutdown(); } + + return Status::OK(); } } diff --git a/cpp/src/server/grpc_impl/GrpcMilvusServer.h b/cpp/src/server/grpc_impl/GrpcMilvusServer.h index 2ddc5ee2fb..751a2cd2a3 100644 --- a/cpp/src/server/grpc_impl/GrpcMilvusServer.h +++ b/cpp/src/server/grpc_impl/GrpcMilvusServer.h @@ -17,6 +17,8 @@ #pragma once +#include "utils/Status.h" + #include #include @@ -27,10 +29,10 @@ namespace grpc { class GrpcMilvusServer { public: - static void + static Status StartService(); - static void + static Status StopService(); }; diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp index 81ef8d81ad..fb8daa3540 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp @@ -75,8 +75,7 @@ namespace { GrpcBaseTask::GrpcBaseTask(const std::string &task_group, bool async) : task_group_(task_group), async_(async), - done_(false), - error_code_(SERVER_SUCCESS) { + done_(false) { } @@ -84,10 +83,10 @@ GrpcBaseTask::~GrpcBaseTask() { WaitToFinish(); } -ErrorCode GrpcBaseTask::Execute() { - error_code_ = OnExecute(); +Status GrpcBaseTask::Execute() { + status_ = OnExecute(); Done(); - return error_code_; + return status_; } void GrpcBaseTask::Done() { @@ -95,19 +94,17 @@ void GrpcBaseTask::Done() { finish_cond_.notify_all(); } -ErrorCode GrpcBaseTask::SetError(ErrorCode error_code, const std::string &error_msg) { - error_code_ = error_code; - error_msg_ = error_msg; - - SERVER_LOG_ERROR << error_msg_; - return error_code_; +Status GrpcBaseTask::SetStatus(ErrorCode error_code, const std::string &error_msg) { + status_ = Status(error_code, error_msg); + SERVER_LOG_ERROR << error_msg; + return status_; } -ErrorCode GrpcBaseTask::WaitToFinish() { +Status GrpcBaseTask::WaitToFinish() { std::unique_lock lock(finish_mtx_); finish_cond_.wait(lock, [this] { return done_; }); - return error_code_; + return status_; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -130,10 +127,10 @@ void GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Statu if (!task_ptr->IsAsync()) { task_ptr->WaitToFinish(); - ErrorCode err = task_ptr->ErrorID(); - if (err != SERVER_SUCCESS) { - grpc_status->set_reason(task_ptr->ErrorMsg()); - grpc_status->set_error_code(ErrorMap(err)); + const Status& status = task_ptr->status(); + if (!status.ok()) { + grpc_status->set_reason(status.message()); + grpc_status->set_error_code(ErrorMap(status.code())); } } } @@ -171,19 +168,19 @@ void GrpcRequestScheduler::Stop() { SERVER_LOG_INFO << "Scheduler stopped"; } -ErrorCode GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr &task_ptr) { +Status GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr &task_ptr) { if (task_ptr == nullptr) { - return SERVER_NULL_POINTER; + return Status::OK(); } - ErrorCode err = PutTaskToQueue(task_ptr); - if (err != SERVER_SUCCESS) { - SERVER_LOG_ERROR << "Put task to queue failed with code: " << err; - return err; + auto status = PutTaskToQueue(task_ptr); + if (!status.ok()) { + SERVER_LOG_ERROR << "Put task to queue failed with code: " << status.ToString(); + return status; } if (task_ptr->IsAsync()) { - return SERVER_SUCCESS;//async execution, caller need to call WaitToFinish at somewhere + return Status::OK(); //async execution, caller need to call WaitToFinish at somewhere } return task_ptr->WaitToFinish();//sync execution @@ -203,9 +200,9 @@ void GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) { } try { - ErrorCode err = task->Execute(); - if (err != SERVER_SUCCESS) { - SERVER_LOG_ERROR << "Task failed with code: " << err; + auto status = task->Execute(); + if (!status.ok()) { + SERVER_LOG_ERROR << "Task failed with code: " << status.ToString(); } } catch (std::exception &ex) { SERVER_LOG_ERROR << "Task failed to execute: " << ex.what(); @@ -213,7 +210,7 @@ void GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) { } } -ErrorCode GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) { +Status GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) { std::lock_guard lock(queue_mtx_); std::string group_name = task_ptr->TaskGroup(); @@ -230,7 +227,7 @@ ErrorCode GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) { SERVER_LOG_INFO << "Create new thread for task group: " << group_name; } - return SERVER_SUCCESS; + return Status::OK(); } } diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.h b/cpp/src/server/grpc_impl/GrpcRequestScheduler.h index f8f287b1e6..e1217540fc 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.h +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.h @@ -17,6 +17,7 @@ #pragma once +#include "utils/Status.h" #include "utils/BlockingQueue.h" #include "status.grpc.pb.h" #include "status.pb.h" @@ -37,24 +38,22 @@ protected: virtual ~GrpcBaseTask(); public: - ErrorCode Execute(); + Status Execute(); void Done(); - ErrorCode WaitToFinish(); + Status WaitToFinish(); std::string TaskGroup() const { return task_group_; } - ErrorCode ErrorID() const { return error_code_; } - - std::string ErrorMsg() const { return error_msg_; } + const Status& status() const { return status_; } bool IsAsync() const { return async_; } protected: - virtual ErrorCode OnExecute() = 0; + virtual Status OnExecute() = 0; - ErrorCode SetError(ErrorCode error_code, const std::string &msg); + Status SetStatus(ErrorCode error_code, const std::string &msg); protected: mutable std::mutex finish_mtx_; @@ -63,8 +62,7 @@ protected: std::string task_group_; bool async_; bool done_; - ErrorCode error_code_; - std::string error_msg_; + Status status_; }; using BaseTaskPtr = std::shared_ptr; @@ -83,7 +81,7 @@ public: void Stop(); - ErrorCode ExecuteTask(const BaseTaskPtr &task_ptr); + Status ExecuteTask(const BaseTaskPtr &task_ptr); static void ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Status *grpc_status); @@ -94,7 +92,7 @@ protected: void TakeTaskToExecute(TaskQueuePtr task_queue); - ErrorCode PutTaskToQueue(const BaseTaskPtr &task_ptr); + Status PutTaskToQueue(const BaseTaskPtr &task_ptr); private: mutable std::mutex queue_mtx_; diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index d7b39bea05..e49ea48236 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -77,33 +77,26 @@ namespace { constexpr long DAY_SECONDS = 24 * 60 * 60; - void + Status ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range> &range_array, - std::vector &dates, - ErrorCode &error_code, - std::string &error_msg) { + std::vector &dates) { dates.clear(); for (auto &range : range_array) { time_t tt_start, tt_end; tm tm_start, tm_end; if (!CommonUtil::TimeStrToTime(range.start_value(), tt_start, tm_start)) { - error_code = SERVER_INVALID_TIME_RANGE; - error_msg = "Invalid time range: " + range.start_value(); - return; + return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); } if (!CommonUtil::TimeStrToTime(range.end_value(), tt_end, tm_end)) { - error_code = SERVER_INVALID_TIME_RANGE; - error_msg = "Invalid time range: " + range.start_value(); - return; + return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); } long days = (tt_end > tt_start) ? (tt_end - tt_start) / DAY_SECONDS : (tt_start - tt_end) / DAY_SECONDS; if (days == 0) { - error_code = SERVER_INVALID_TIME_RANGE; - error_msg = "Invalid time range: " + range.start_value() + " to " + range.end_value(); - return; + return Status(SERVER_INVALID_TIME_RANGE, + "Invalid time range: " + range.start_value() + " to " + range.end_value()); } //range: [start_day, end_day) @@ -117,6 +110,8 @@ namespace { dates.push_back(date); } } + + return Status::OK(); } } @@ -136,30 +131,30 @@ CreateTableTask::Create(const ::milvus::grpc::TableSchema *schema) { return std::shared_ptr(new CreateTableTask(schema)); } -ErrorCode +Status CreateTableTask::OnExecute() { TimeRecorder rc("CreateTableTask"); try { //step 1: check arguments - ErrorCode res = ValidationUtil::ValidateTableName(schema_->table_name().table_name()); - if (res != SERVER_SUCCESS) { - return SetError(res, "Invalid table name: " + schema_->table_name().table_name()); + auto status = ValidationUtil::ValidateTableName(schema_->table_name().table_name()); + if (!status.ok()) { + return status; } - res = ValidationUtil::ValidateTableDimension(schema_->dimension()); - if (res != SERVER_SUCCESS) { - return SetError(res, "Invalid table dimension: " + std::to_string(schema_->dimension())); + status = ValidationUtil::ValidateTableDimension(schema_->dimension()); + if (!status.ok()) { + return status; } - res = ValidationUtil::ValidateTableIndexFileSize(schema_->index_file_size()); - if(res != SERVER_SUCCESS) { - return SetError(res, "Invalid index file size: " + std::to_string(schema_->index_file_size())); + status = ValidationUtil::ValidateTableIndexFileSize(schema_->index_file_size()); + if (!status.ok()) { + return status; } - res = ValidationUtil::ValidateTableIndexMetricType(schema_->metric_type()); - if(res != SERVER_SUCCESS) { - return SetError(res, "Invalid index metric type: " + std::to_string(schema_->metric_type())); + status = ValidationUtil::ValidateTableIndexMetricType(schema_->metric_type()); + if (!status.ok()) { + return status; } //step 2: construct table schema @@ -170,22 +165,22 @@ CreateTableTask::OnExecute() { table_info.metric_type_ = schema_->metric_type(); //step 3: create table - auto stat = DBWrapper::DB()->CreateTable(table_info); - if (!stat.ok()) { + status = DBWrapper::DB()->CreateTable(table_info); + if (!status.ok()) { //table could exist - if(stat.code() == DB_ALREADY_EXIST) { - return SetError(SERVER_INVALID_TABLE_NAME, stat.ToString()); + if(status.code() == DB_ALREADY_EXIST) { + return Status(SERVER_INVALID_TABLE_NAME, status.message()); } - return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); + return status; } } catch (std::exception &ex) { - return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } rc.ElapseFromBegin("totally cost"); - return SERVER_SUCCESS; + return Status::OK(); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -200,23 +195,23 @@ DescribeTableTask::Create(const std::string &table_name, ::milvus::grpc::TableSc return std::shared_ptr(new DescribeTableTask(table_name, schema)); } -ErrorCode +Status DescribeTableTask::OnExecute() { TimeRecorder rc("DescribeTableTask"); try { //step 1: check arguments - ErrorCode res = ValidationUtil::ValidateTableName(table_name_); - if (res != SERVER_SUCCESS) { - return SetError(res, "Invalid table name: " + table_name_); + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; } //step 2: get table info engine::meta::TableSchema table_info; table_info.table_id_ = table_name_; - auto stat = DBWrapper::DB()->DescribeTable(table_info); - if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + return status; } schema_->mutable_table_name()->set_table_name(table_info.table_id_); @@ -225,12 +220,12 @@ DescribeTableTask::OnExecute() { schema_->set_metric_type(table_info.metric_type_); } catch (std::exception &ex) { - return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } rc.ElapseFromBegin("totally cost"); - return SERVER_SUCCESS; + return Status::OK(); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -248,54 +243,54 @@ CreateIndexTask::Create(const ::milvus::grpc::IndexParam *index_param) { return std::shared_ptr(new CreateIndexTask(index_param)); } -ErrorCode +Status CreateIndexTask::OnExecute() { try { TimeRecorder rc("CreateIndexTask"); //step 1: check arguments std::string table_name_ = index_param_->table_name().table_name(); - ErrorCode res = ValidationUtil::ValidateTableName(table_name_); - if (res != SERVER_SUCCESS) { - return SetError(res, "Invalid table name: " + table_name_); + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; } bool has_table = false; - auto stat = DBWrapper::DB()->HasTable(table_name_, has_table); - if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); + status = DBWrapper::DB()->HasTable(table_name_, has_table); + if (!status.ok()) { + return status; } if (!has_table) { - return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); } auto &grpc_index = index_param_->index(); - res = ValidationUtil::ValidateTableIndexType(grpc_index.index_type()); - if(res != SERVER_SUCCESS) { - return SetError(res, "Invalid index type: " + std::to_string(grpc_index.index_type())); + status = ValidationUtil::ValidateTableIndexType(grpc_index.index_type()); + if (!status.ok()) { + return status; } - res = ValidationUtil::ValidateTableIndexNlist(grpc_index.nlist()); - if(res != SERVER_SUCCESS) { - return SetError(res, "Invalid index nlist: " + std::to_string(grpc_index.nlist())); + status = ValidationUtil::ValidateTableIndexNlist(grpc_index.nlist()); + if (!status.ok()) { + return status; } //step 2: check table existence engine::TableIndex index; index.engine_type_ = grpc_index.index_type(); index.nlist_ = grpc_index.nlist(); - stat = DBWrapper::DB()->CreateIndex(table_name_, index); - if (!stat.ok()) { - return SetError(SERVER_BUILD_INDEX_ERROR, stat.ToString()); + status = DBWrapper::DB()->CreateIndex(table_name_, index); + if (!status.ok()) { + return status; } rc.ElapseFromBegin("totally cost"); } catch (std::exception &ex) { - return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } - return SERVER_SUCCESS; + return Status::OK(); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -311,29 +306,29 @@ HasTableTask::Create(const std::string &table_name, bool &has_table) { return std::shared_ptr(new HasTableTask(table_name, has_table)); } -ErrorCode +Status HasTableTask::OnExecute() { try { TimeRecorder rc("HasTableTask"); //step 1: check arguments - ErrorCode res = ValidationUtil::ValidateTableName(table_name_); - if (res != SERVER_SUCCESS) { - return SetError(res, "Invalid table name: " + table_name_); + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; } //step 2: check table existence - auto stat = DBWrapper::DB()->HasTable(table_name_, has_table_); - if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); + status = DBWrapper::DB()->HasTable(table_name_, has_table_); + if (!status.ok()) { + return status; } rc.ElapseFromBegin("totally cost"); } catch (std::exception &ex) { - return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } - return SERVER_SUCCESS; + return Status::OK(); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -348,26 +343,26 @@ DropTableTask::Create(const std::string &table_name) { return std::shared_ptr(new DropTableTask(table_name)); } -ErrorCode +Status DropTableTask::OnExecute() { try { TimeRecorder rc("DropTableTask"); //step 1: check arguments - ErrorCode res = ValidationUtil::ValidateTableName(table_name_); - if (res != SERVER_SUCCESS) { - return SetError(res, "Invalid table name: " + table_name_); + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; } //step 2: check table existence engine::meta::TableSchema table_info; table_info.table_id_ = table_name_; - auto stat = DBWrapper::DB()->DescribeTable(table_info); - if (!stat.ok()) { - if (stat.code() == DB_NOT_FOUND) { - return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code() == DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); } else { - return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); + return status; } } @@ -375,17 +370,17 @@ DropTableTask::OnExecute() { //step 3: Drop table std::vector dates; - stat = DBWrapper::DB()->DeleteTable(table_name_, dates); - if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); + status = DBWrapper::DB()->DeleteTable(table_name_, dates); + if (!status.ok()) { + return status; } rc.ElapseFromBegin("total cost"); } catch (std::exception &ex) { - return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } - return SERVER_SUCCESS; + return Status::OK(); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -400,22 +395,22 @@ ShowTablesTask::Create(::grpc::ServerWriter<::milvus::grpc::TableName> *writer) return std::shared_ptr(new ShowTablesTask(writer)); } -ErrorCode +Status ShowTablesTask::OnExecute() { std::vector schema_array; - auto stat = DBWrapper::DB()->AllTables(schema_array); - if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); + auto statuts = DBWrapper::DB()->AllTables(schema_array); + if (!statuts.ok()) { + return statuts; } for (auto &schema : schema_array) { ::milvus::grpc::TableName tableName; tableName.set_table_name(schema.table_id_); if (!writer_->Write(tableName)) { - return SetError(SERVER_WRITE_ERROR, "Write table name failed!"); + return Status(SERVER_WRITE_ERROR, "Write table name failed!"); } } - return SERVER_SUCCESS; + return Status::OK(); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -436,23 +431,23 @@ InsertTask::Create(const ::milvus::grpc::InsertParam *insert_param, return std::shared_ptr(new InsertTask(insert_param, record_ids)); } -ErrorCode +Status InsertTask::OnExecute() { try { TimeRecorder rc("InsertVectorTask"); //step 1: check arguments - ErrorCode res = ValidationUtil::ValidateTableName(insert_param_->table_name()); - if (res != SERVER_SUCCESS) { - return SetError(res, "Invalid table name: " + insert_param_->table_name()); + auto status = ValidationUtil::ValidateTableName(insert_param_->table_name()); + if (!status.ok()) { + return status; } if (insert_param_->row_record_array().empty()) { - return SetError(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty"); + return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty"); } if (!record_ids_->vector_id_array().empty()) { if (record_ids_->vector_id_array().size() != insert_param_->row_record_array_size()) { - return SetError(SERVER_ILLEGAL_VECTOR_ID, + return Status(SERVER_ILLEGAL_VECTOR_ID, "Size of vector ids is not equal to row record array size"); } } @@ -460,13 +455,13 @@ InsertTask::OnExecute() { //step 2: check table existence engine::meta::TableSchema table_info; table_info.table_id_ = insert_param_->table_name(); - auto stat = DBWrapper::DB()->DescribeTable(table_info); - if (!stat.ok()) { - if (stat.code() == DB_NOT_FOUND) { - return SetError(SERVER_TABLE_NOT_EXIST, + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code() == DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, "Table " + insert_param_->table_name() + " not exists"); } else { - return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); + return status; } } @@ -475,12 +470,12 @@ InsertTask::OnExecute() { bool user_provide_ids = !insert_param_->row_id_array().empty(); //user already provided id before, all insert action require user id if((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) && !user_provide_ids) { - return SetError(SERVER_ILLEGAL_VECTOR_ID, "Table vector ids are user defined, please provide id for this batch"); + return Status(SERVER_ILLEGAL_VECTOR_ID, "Table vector ids are user defined, please provide id for this batch"); } //user didn't provided id before, no need to provide user id if((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) && user_provide_ids) { - return SetError(SERVER_ILLEGAL_VECTOR_ID, "Table vector ids are auto generated, no need to provide id for this batch"); + return Status(SERVER_ILLEGAL_VECTOR_ID, "Table vector ids are auto generated, no need to provide id for this batch"); } rc.RecordSection("check validation"); @@ -496,7 +491,7 @@ InsertTask::OnExecute() { // TODO: change to one dimension array in protobuf or use multiple-thread to copy the data for (size_t i = 0; i < insert_param_->row_record_array_size(); i++) { if (insert_param_->row_record_array(i).vector_data().empty()) { - return SetError(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array data is empty"); + return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array data is empty"); } uint64_t vec_dim = insert_param_->row_record_array(i).vector_data().size(); if (vec_dim != table_info.dimension_) { @@ -504,7 +499,7 @@ InsertTask::OnExecute() { std::string error_msg = "Invalid row record dimension: " + std::to_string(vec_dim) + " vs. table dimension:" + std::to_string(table_info.dimension_); - return SetError(error_code, error_msg); + return Status(error_code, error_msg); } memcpy(&vec_f[i * table_info.dimension_], insert_param_->row_record_array(i).vector_data().data(), @@ -522,10 +517,10 @@ InsertTask::OnExecute() { memcpy(target_data, src_data, (size_t)(sizeof(int64_t)*insert_param_->row_id_array_size())); } - stat = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), vec_count, vec_f.data(), vec_ids); + status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), vec_count, vec_f.data(), vec_ids); rc.ElapseFromBegin("add vectors to engine"); - if (!stat.ok()) { - return SetError(SERVER_CACHE_ERROR, "Cache error: " + stat.ToString()); + if (!status.ok()) { + return status; } for (int64_t id : vec_ids) { record_ids_->add_vector_id_array(id); @@ -535,13 +530,13 @@ InsertTask::OnExecute() { if (ids_size != vec_count) { std::string msg = "Add " + std::to_string(vec_count) + " vectors but only return " + std::to_string(ids_size) + " id"; - return SetError(SERVER_ILLEGAL_VECTOR_ID, msg); + return Status(SERVER_ILLEGAL_VECTOR_ID, msg); } //step 6: update table flag user_provide_ids ? table_info.flag_ |= engine::meta::FLAG_MASK_HAS_USERID : table_info.flag_ |= engine::meta::FLAG_MASK_NO_USERID; - stat = DBWrapper::DB()->UpdateTableFlag(insert_param_->table_name(), table_info.flag_); + status = DBWrapper::DB()->UpdateTableFlag(insert_param_->table_name(), table_info.flag_); #ifdef MILVUS_ENABLE_PROFILING ProfilerStop(); @@ -551,10 +546,10 @@ InsertTask::OnExecute() { rc.ElapseFromBegin("total cost"); } catch (std::exception &ex) { - return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } - return SERVER_SUCCESS; + return Status::OK(); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -580,7 +575,7 @@ SearchTask::Create(const ::milvus::grpc::SearchParam *search_vector_infos, response)); } -ErrorCode +Status SearchTask::OnExecute() { try { int64_t top_k = search_param_->topk(); @@ -591,53 +586,51 @@ SearchTask::OnExecute() { //step 1: check table name std::string table_name_ = search_param_->table_name(); - ErrorCode res = ValidationUtil::ValidateTableName(table_name_); - if (res != SERVER_SUCCESS) { - return SetError(res, "Invalid table name: " + table_name_); + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; } //step 2: check table existence engine::meta::TableSchema table_info; table_info.table_id_ = table_name_; - auto stat = DBWrapper::DB()->DescribeTable(table_info); - if (!stat.ok()) { - if (stat.code() == DB_NOT_FOUND) { - return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code() == DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); } else { - return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); + return status; } } //step 3: check search parameter - res = ValidationUtil::ValidateSearchTopk(top_k, table_info); - if (res != SERVER_SUCCESS) { - return SetError(res, "Invalid topk: " + std::to_string(top_k)); + status = ValidationUtil::ValidateSearchTopk(top_k, table_info); + if (!status.ok()) { + return status; } - res = ValidationUtil::ValidateSearchNprobe(nprobe, table_info); - if (res != SERVER_SUCCESS) { - return SetError(res, "Invalid nprobe: " + std::to_string(nprobe)); + status = ValidationUtil::ValidateSearchNprobe(nprobe, table_info); + if (!status.ok()) { + return status; } if (search_param_->query_record_array().empty()) { - return SetError(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty"); + return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty"); } //step 4: check date range, and convert to db dates std::vector dates; - ErrorCode error_code = SERVER_SUCCESS; - std::string error_msg; - std::vector<::milvus::grpc::Range> range_array; for (size_t i = 0; i < search_param_->query_range_array_size(); i++) { range_array.emplace_back(search_param_->query_range_array(i)); } - ConvertTimeRangeToDBDates(range_array, dates, error_code, error_msg); - if (error_code != SERVER_SUCCESS) { - return SetError(error_code, error_msg); + + status = ConvertTimeRangeToDBDates(range_array, dates); + if (!status.ok()) { + return status; } - double span_check = rc.RecordSection("check validation"); + rc.RecordSection("check validation"); //step 5: prepare float data @@ -645,14 +638,14 @@ SearchTask::OnExecute() { std::vector vec_f(record_array_size * table_info.dimension_, 0); for (size_t i = 0; i < record_array_size; i++) { if (search_param_->query_record_array(i).vector_data().empty()) { - return SetError(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array data is empty"); + return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array data is empty"); } uint64_t query_vec_dim = search_param_->query_record_array(i).vector_data().size(); if (query_vec_dim != table_info.dimension_) { ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; std::string error_msg = "Invalid row record dimension: " + std::to_string(query_vec_dim) + " vs. table dimension:" + std::to_string(table_info.dimension_); - return SetError(error_code, error_msg); + return Status(error_code, error_msg); } memcpy(&vec_f[i * table_info.dimension_], @@ -671,11 +664,11 @@ SearchTask::OnExecute() { #endif if (file_id_array_.empty()) { - stat = DBWrapper::DB()->Query(table_name_, (size_t) top_k, record_count, nprobe, vec_f.data(), - dates, results); + status = DBWrapper::DB()->Query(table_name_, (size_t) top_k, record_count, nprobe, + vec_f.data(), dates, results); } else { - stat = DBWrapper::DB()->Query(table_name_, file_id_array_, (size_t) top_k, - record_count, nprobe, vec_f.data(), dates, results); + status = DBWrapper::DB()->Query(table_name_, file_id_array_, (size_t) top_k, + record_count, nprobe, vec_f.data(), dates, results); } #ifdef MILVUS_ENABLE_PROFILING @@ -683,18 +676,18 @@ SearchTask::OnExecute() { #endif rc.RecordSection("search vectors from engine"); - if (!stat.ok()) { - return SetError(stat.code(), stat.ToString()); + if (!status.ok()) { + return status; } if (results.empty()) { - return SERVER_SUCCESS; //empty table + return Status::OK(); //empty table } if (results.size() != record_count) { std::string msg = "Search " + std::to_string(record_count) + " vectors but only return " + std::to_string(results.size()) + " results"; - return SetError(SERVER_ILLEGAL_SEARCH_RESULT, msg); + return Status(SERVER_ILLEGAL_SEARCH_RESULT, msg); } //step 7: construct result array @@ -713,10 +706,10 @@ SearchTask::OnExecute() { } catch (std::exception &ex) { - return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } - return SERVER_SUCCESS; + return Status::OK(); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -732,23 +725,22 @@ CountTableTask::Create(const std::string &table_name, int64_t &row_count) { return std::shared_ptr(new CountTableTask(table_name, row_count)); } -ErrorCode +Status CountTableTask::OnExecute() { try { TimeRecorder rc("GetTableRowCountTask"); //step 1: check arguments - ErrorCode res = SERVER_SUCCESS; - res = ValidationUtil::ValidateTableName(table_name_); - if (res != SERVER_SUCCESS) { - return SetError(res, "Invalid table name: " + table_name_); + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; } //step 2: get row count uint64_t row_count = 0; - auto stat = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); - if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); + status = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); + if (!status.ok()) { + return status; } row_count_ = (int64_t) row_count; @@ -756,10 +748,10 @@ CountTableTask::OnExecute() { rc.ElapseFromBegin("total cost"); } catch (std::exception &ex) { - return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } - return SERVER_SUCCESS; + return Status::OK(); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -775,7 +767,7 @@ CmdTask::Create(const std::string &cmd, std::string &result) { return std::shared_ptr(new CmdTask(cmd, result)); } -ErrorCode +Status CmdTask::OnExecute() { if (cmd_ == "version") { result_ = MILVUS_VERSION; @@ -786,7 +778,7 @@ CmdTask::OnExecute() { result_ = "OK"; } - return SERVER_SUCCESS; + return Status::OK(); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -804,27 +796,27 @@ DeleteByRangeTask::Create(const ::milvus::grpc::DeleteByRangeParam *delete_by_ra return std::shared_ptr(new DeleteByRangeTask(delete_by_range_param)); } -ErrorCode +Status DeleteByRangeTask::OnExecute() { try { TimeRecorder rc("DeleteByRangeTask"); //step 1: check arguments std::string table_name = delete_by_range_param_->table_name(); - ErrorCode res = ValidationUtil::ValidateTableName(table_name); - if (res != SERVER_SUCCESS) { - return SetError(res, "Invalid table name: " + table_name); + auto status = ValidationUtil::ValidateTableName(table_name); + if (!status.ok()) { + return status; } //step 2: check table existence engine::meta::TableSchema table_info; table_info.table_id_ = table_name; - auto stat = DBWrapper::DB()->DescribeTable(table_info); - if (!stat.ok()) { - if (stat.code(), DB_NOT_FOUND) { - return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name + " not exists"); + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code(), DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name + " not exists"); } else { - return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); + return status; } } @@ -837,25 +829,25 @@ DeleteByRangeTask::OnExecute() { std::vector<::milvus::grpc::Range> range_array; range_array.emplace_back(delete_by_range_param_->range()); - ConvertTimeRangeToDBDates(range_array, dates, error_code, error_msg); - if (error_code != SERVER_SUCCESS) { - return SetError(error_code, error_msg); + status = ConvertTimeRangeToDBDates(range_array, dates); + if (!status.ok()) { + return status; } #ifdef MILVUS_ENABLE_PROFILING std::string fname = "/tmp/search_nq_" + this->delete_by_range_param_->table_name() + ".profiling"; ProfilerStart(fname.c_str()); #endif - stat = DBWrapper::DB()->DeleteTable(table_name, dates); - if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); + status = DBWrapper::DB()->DeleteTable(table_name, dates); + if (!status.ok()) { + return status; } } catch (std::exception &ex) { - return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } - return SERVER_SUCCESS; + return Status::OK(); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -870,29 +862,29 @@ PreloadTableTask::Create(const std::string &table_name){ return std::shared_ptr(new PreloadTableTask(table_name)); } -ErrorCode +Status PreloadTableTask::OnExecute() { try { TimeRecorder rc("PreloadTableTask"); //step 1: check arguments - ErrorCode res = ValidationUtil::ValidateTableName(table_name_); - if (res != SERVER_SUCCESS) { - return SetError(res, "Invalid table name: " + table_name_); + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; } //step 2: check table existence - auto stat = DBWrapper::DB()->PreloadTable(table_name_); - if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); + status = DBWrapper::DB()->PreloadTable(table_name_); + if (!status.ok()) { + return status; } rc.ElapseFromBegin("totally cost"); } catch (std::exception &ex) { - return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } - return SERVER_SUCCESS; + return Status::OK(); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -910,22 +902,22 @@ DescribeIndexTask::Create(const std::string &table_name, return std::shared_ptr(new DescribeIndexTask(table_name, index_param)); } -ErrorCode +Status DescribeIndexTask::OnExecute() { try { TimeRecorder rc("DescribeIndexTask"); //step 1: check arguments - ErrorCode res = ValidationUtil::ValidateTableName(table_name_); - if (res != SERVER_SUCCESS) { - return SetError(res, "Invalid table name: " + table_name_); + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; } //step 2: check table existence engine::TableIndex index; - auto stat = DBWrapper::DB()->DescribeIndex(table_name_, index); - if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); + status = DBWrapper::DB()->DescribeIndex(table_name_, index); + if (!status.ok()) { + return status; } index_param_->mutable_table_name()->set_table_name(table_name_); @@ -934,10 +926,10 @@ DescribeIndexTask::OnExecute() { rc.ElapseFromBegin("totally cost"); } catch (std::exception &ex) { - return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } - return SERVER_SUCCESS; + return Status::OK(); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -952,39 +944,39 @@ DropIndexTask::Create(const std::string &table_name){ return std::shared_ptr(new DropIndexTask(table_name)); } -ErrorCode +Status DropIndexTask::OnExecute() { try { TimeRecorder rc("DropIndexTask"); //step 1: check arguments - ErrorCode res = ValidationUtil::ValidateTableName(table_name_); - if (res != SERVER_SUCCESS) { - return SetError(res, "Invalid table name: " + table_name_); + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; } bool has_table = false; - auto stat = DBWrapper::DB()->HasTable(table_name_, has_table); - if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); + status = DBWrapper::DB()->HasTable(table_name_, has_table); + if (!status.ok()) { + return status; } if (!has_table) { - return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); } //step 2: check table existence - stat = DBWrapper::DB()->DropIndex(table_name_); - if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); + status = DBWrapper::DB()->DropIndex(table_name_); + if (!status.ok()) { + return status; } rc.ElapseFromBegin("totally cost"); } catch (std::exception &ex) { - return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } - return SERVER_SUCCESS; + return Status::OK(); } } diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.h b/cpp/src/server/grpc_impl/GrpcRequestTask.h index 42a1048124..b7aa4af1d8 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.h +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.h @@ -17,7 +17,7 @@ #pragma once #include "GrpcRequestScheduler.h" -#include "utils/Error.h" +#include "utils/Status.h" #include "db/Types.h" #include "milvus.grpc.pb.h" @@ -41,7 +41,7 @@ protected: explicit CreateTableTask(const ::milvus::grpc::TableSchema *request); - ErrorCode + Status OnExecute() override; private: @@ -57,7 +57,7 @@ public: protected: HasTableTask(const std::string &request, bool &has_table); - ErrorCode + Status OnExecute() override; @@ -75,7 +75,7 @@ public: protected: DescribeTableTask(const std::string &table_name, ::milvus::grpc::TableSchema *schema); - ErrorCode + Status OnExecute() override; @@ -94,7 +94,7 @@ protected: explicit DropTableTask(const std::string &table_name); - ErrorCode + Status OnExecute() override; @@ -112,7 +112,7 @@ protected: explicit CreateIndexTask(const ::milvus::grpc::IndexParam *index_Param); - ErrorCode + Status OnExecute() override; @@ -130,7 +130,7 @@ protected: explicit ShowTablesTask(::grpc::ServerWriter<::milvus::grpc::TableName> *writer); - ErrorCode + Status OnExecute() override; private: @@ -148,7 +148,7 @@ protected: InsertTask(const ::milvus::grpc::InsertParam *insert_Param, ::milvus::grpc::VectorIds *record_ids_); - ErrorCode + Status OnExecute() override; private: @@ -169,7 +169,7 @@ protected: const std::vector &file_id_array, ::milvus::grpc::TopKQueryResultList *response); - ErrorCode + Status OnExecute() override; private: @@ -187,7 +187,7 @@ public: protected: CountTableTask(const std::string &table_name, int64_t &row_count); - ErrorCode + Status OnExecute() override; private: @@ -204,7 +204,7 @@ public: protected: CmdTask(const std::string &cmd, std::string &result); - ErrorCode + Status OnExecute() override; private: @@ -221,7 +221,7 @@ public: protected: DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param); - ErrorCode + Status OnExecute() override; private: @@ -237,7 +237,7 @@ public: protected: PreloadTableTask(const std::string &table_name); - ErrorCode + Status OnExecute() override; private: @@ -255,7 +255,7 @@ protected: DescribeIndexTask(const std::string &table_name, ::milvus::grpc::IndexParam *index_param); - ErrorCode + Status OnExecute() override; private: @@ -272,7 +272,7 @@ public: protected: DropIndexTask(const std::string &table_name); - ErrorCode + Status OnExecute() override; private: diff --git a/cpp/src/utils/CommonUtil.cpp b/cpp/src/utils/CommonUtil.cpp index 0f3f7a229e..19a63006cd 100644 --- a/cpp/src/utils/CommonUtil.cpp +++ b/cpp/src/utils/CommonUtil.cpp @@ -73,35 +73,35 @@ bool CommonUtil::IsDirectoryExist(const std::string &path) { return true; } -ErrorCode CommonUtil::CreateDirectory(const std::string &path) { +Status CommonUtil::CreateDirectory(const std::string &path) { if(path.empty()) { - return SERVER_SUCCESS; + return Status::OK(); } struct stat directory_stat; int status = stat(path.c_str(), &directory_stat); if (status == 0) { - return SERVER_SUCCESS;//already exist + return Status::OK();//already exist } fs::path fs_path(path); fs::path parent_path = fs_path.parent_path(); - ErrorCode err = CreateDirectory(parent_path.string()); - if(err != SERVER_SUCCESS){ - return err; + Status err_status = CreateDirectory(parent_path.string()); + if(!err_status.ok()){ + return err_status; } status = stat(path.c_str(), &directory_stat); if (status == 0) { - return SERVER_SUCCESS;//already exist + return Status::OK();//already exist } int makeOK = mkdir(path.c_str(), S_IRWXU|S_IRGRP|S_IROTH); if (makeOK != 0) { - return SERVER_UNEXPECTED_ERROR; + return Status(SERVER_UNEXPECTED_ERROR, "failed to create directory: " + path); } - return SERVER_SUCCESS; + return Status::OK(); } namespace { @@ -134,18 +134,19 @@ namespace { } } -ErrorCode CommonUtil::DeleteDirectory(const std::string &path) { +Status CommonUtil::DeleteDirectory(const std::string &path) { if(path.empty()) { - return SERVER_SUCCESS; + return Status::OK(); } struct stat directory_stat; int statOK = stat(path.c_str(), &directory_stat); - if (statOK != 0) - return SERVER_SUCCESS; + if (statOK != 0) { + return Status::OK(); + } RemoveDirectory(path); - return SERVER_SUCCESS; + return Status::OK(); } bool CommonUtil::IsFileExist(const std::string &path) { diff --git a/cpp/src/utils/CommonUtil.h b/cpp/src/utils/CommonUtil.h index 8c825f70fe..aee57f0446 100755 --- a/cpp/src/utils/CommonUtil.h +++ b/cpp/src/utils/CommonUtil.h @@ -17,10 +17,11 @@ #pragma once +#include "utils/Status.h" + #include #include -#include "Error.h" namespace zilliz { namespace milvus { @@ -34,8 +35,8 @@ class CommonUtil { static bool IsFileExist(const std::string &path); static uint64_t GetFileSize(const std::string &path); static bool IsDirectoryExist(const std::string &path); - static ErrorCode CreateDirectory(const std::string &path); - static ErrorCode DeleteDirectory(const std::string &path); + static Status CreateDirectory(const std::string &path); + static Status DeleteDirectory(const std::string &path); static std::string GetExePath(); diff --git a/cpp/src/utils/Status.cpp b/cpp/src/utils/Status.cpp index 0232c823f0..e21a279a4c 100644 --- a/cpp/src/utils/Status.cpp +++ b/cpp/src/utils/Status.cpp @@ -21,9 +21,9 @@ namespace zilliz { namespace milvus { -constexpr int CODE_WIDTH = sizeof(ErrorCode); +constexpr int CODE_WIDTH = sizeof(StatusCode); -Status::Status(ErrorCode code, const std::string& msg) { +Status::Status(StatusCode code, const std::string& msg) { //4 bytes store code //4 bytes store message length //the left bytes store message string @@ -50,7 +50,8 @@ Status::Status(const Status &s) CopyFrom(s); } -Status &Status::operator=(const Status &s) { +Status& +Status::operator=(const Status &s) { CopyFrom(s); return *this; } @@ -60,12 +61,14 @@ Status::Status(Status &&s) MoveFrom(s); } -Status &Status::operator=(Status &&s) { +Status& +Status::operator=(Status &&s) { MoveFrom(s); return *this; } -void Status::CopyFrom(const Status &s) { +void +Status::CopyFrom(const Status &s) { delete state_; state_ = nullptr; if(s.state_ == nullptr) { @@ -73,19 +76,37 @@ void Status::CopyFrom(const Status &s) { } uint32_t length = 0; - std::memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length)); + memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length)); int buff_len = length + sizeof(length) + CODE_WIDTH; state_ = new char[buff_len]; memcpy((void*)state_, (void*)s.state_, buff_len); } -void Status::MoveFrom(Status &s) { +void +Status::MoveFrom(Status &s) { delete state_; state_ = s.state_; s.state_ = nullptr; } -std::string Status::ToString() const { +std::string +Status::message() const { + if (state_ == nullptr) { + return ""; + } + + std::string msg; + uint32_t length = 0; + memcpy(&length, state_ + CODE_WIDTH, sizeof(length)); + if(length > 0) { + msg.append(state_ + sizeof(length) + CODE_WIDTH, length); + } + + return msg; +} + +std::string +Status::ToString() const { if (state_ == nullptr) { return "OK"; } @@ -115,12 +136,7 @@ std::string Status::ToString() const { break; } - uint32_t length = 0; - memcpy(&length, state_ + CODE_WIDTH, sizeof(length)); - if(length > 0) { - result.append(state_ + sizeof(length) + CODE_WIDTH, length); - } - + result += message(); return result; } diff --git a/cpp/src/utils/Status.h b/cpp/src/utils/Status.h index 153ff61dc5..fe06c8029d 100644 --- a/cpp/src/utils/Status.h +++ b/cpp/src/utils/Status.h @@ -25,9 +25,11 @@ namespace zilliz { namespace milvus { +using StatusCode = ErrorCode; + class Status { public: - Status(ErrorCode code, const std::string &msg); + Status(StatusCode code, const std::string &msg); Status(); ~Status(); @@ -47,14 +49,17 @@ class Status { bool ok() const { return state_ == nullptr || code() == 0; } + StatusCode + code() const { + return (state_ == nullptr) ? 0 : *(StatusCode*)(state_); + } + + std::string + message() const; + std::string ToString() const; - ErrorCode - code() const { - return (state_ == nullptr) ? 0 : *(ErrorCode*)(state_); - } - private: inline void CopyFrom(const Status &s); diff --git a/cpp/src/utils/StringHelpFunctions.cpp b/cpp/src/utils/StringHelpFunctions.cpp index 9c4a6d3a3a..068c376ea0 100644 --- a/cpp/src/utils/StringHelpFunctions.cpp +++ b/cpp/src/utils/StringHelpFunctions.cpp @@ -36,11 +36,11 @@ void StringHelpFunctions::TrimStringQuote(std::string &string, const std::string } } -ErrorCode StringHelpFunctions::SplitStringByDelimeter(const std::string &str, - const std::string &delimeter, - std::vector &result) { +Status StringHelpFunctions::SplitStringByDelimeter(const std::string &str, + const std::string &delimeter, + std::vector &result) { if(str.empty()) { - return SERVER_SUCCESS; + return Status::OK(); } size_t last = 0; @@ -55,13 +55,13 @@ ErrorCode StringHelpFunctions::SplitStringByDelimeter(const std::string &str, result.emplace_back(temp); } - return SERVER_SUCCESS; + return Status::OK(); } - ErrorCode StringHelpFunctions::SplitStringByQuote(const std::string &str, - const std::string &delimeter, - const std::string "e, - std::vector &result) { +Status StringHelpFunctions::SplitStringByQuote(const std::string &str, + const std::string &delimeter, + const std::string "e, + std::vector &result) { if (quote.empty()) { return SplitStringByDelimeter(str, delimeter, result); } @@ -88,7 +88,7 @@ ErrorCode StringHelpFunctions::SplitStringByDelimeter(const std::string &str, std::string postfix = process_str.substr(last); index = postfix.find_first_of(quote, 0); if (index == std::string::npos) { - return SERVER_UNEXPECTED_ERROR; + return Status(SERVER_UNEXPECTED_ERROR, ""); } std::string quoted_text = postfix.substr(0, index); append_prefix += quoted_text; @@ -105,7 +105,7 @@ ErrorCode StringHelpFunctions::SplitStringByDelimeter(const std::string &str, result.emplace_back(append_prefix); if (last == postfix.length()) { - return SERVER_SUCCESS; + return Status::OK(); } process_str = postfix.substr(index + 1); @@ -117,7 +117,7 @@ ErrorCode StringHelpFunctions::SplitStringByDelimeter(const std::string &str, return SplitStringByDelimeter(process_str, delimeter, result); } - return SERVER_SUCCESS; + return Status::OK(); } } diff --git a/cpp/src/utils/StringHelpFunctions.h b/cpp/src/utils/StringHelpFunctions.h index 1fa45322da..d50b62d8a3 100644 --- a/cpp/src/utils/StringHelpFunctions.h +++ b/cpp/src/utils/StringHelpFunctions.h @@ -17,7 +17,7 @@ #pragma once -#include "./Error.h" +#include "utils/Status.h" #include @@ -41,9 +41,9 @@ public: // ,b, | b | // ,, | | // a a - static ErrorCode SplitStringByDelimeter(const std::string &str, - const std::string &delimeter, - std::vector &result); + static Status SplitStringByDelimeter(const std::string &str, + const std::string &delimeter, + std::vector &result); //assume the table has two columns, quote='\"', delimeter=',' // a,b a | b @@ -52,10 +52,10 @@ public: // "aa,bb" aa,bb // 55,1122\"aa,bb\",yyy,\"kkk\" 55 | 1122aa,bb | yyy | kkk // "55,1122"aa,bb",yyy,"kkk" illegal - static ErrorCode SplitStringByQuote(const std::string &str, - const std::string &delimeter, - const std::string "e, - std::vector &result); + static Status SplitStringByQuote(const std::string &str, + const std::string &delimeter, + const std::string "e, + std::vector &result); }; diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index 2572cbd8bd..c993a4ae14 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -21,9 +21,7 @@ #include "Log.h" #include - #include - #include #include @@ -36,135 +34,156 @@ constexpr size_t TABLE_NAME_SIZE_LIMIT = 255; constexpr int64_t TABLE_DIMENSION_LIMIT = 16384; constexpr int32_t INDEX_FILE_SIZE_LIMIT = 4096; //index trigger size max = 4096 MB -ErrorCode +Status ValidationUtil::ValidateTableName(const std::string &table_name) { // Table name shouldn't be empty. if (table_name.empty()) { - SERVER_LOG_ERROR << "Empty table name"; - return SERVER_INVALID_TABLE_NAME; + std::string msg = "Empty table name"; + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_TABLE_NAME, msg); } // Table name size shouldn't exceed 16384. if (table_name.size() > TABLE_NAME_SIZE_LIMIT) { - SERVER_LOG_ERROR << "Table name size exceed the limitation"; - return SERVER_INVALID_TABLE_NAME; + std::string msg = "Table name size exceed the limitation"; + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_TABLE_NAME, msg); } // Table name first character should be underscore or character. char first_char = table_name[0]; if (first_char != '_' && std::isalpha(first_char) == 0) { - SERVER_LOG_ERROR << "Table name first character isn't underscore or character: " << first_char; - return SERVER_INVALID_TABLE_NAME; + std::string msg = "Table name first character isn't underscore or character"; + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_TABLE_NAME, msg); } int64_t table_name_size = table_name.size(); for (int64_t i = 1; i < table_name_size; ++i) { char name_char = table_name[i]; if (name_char != '_' && std::isalnum(name_char) == 0) { - SERVER_LOG_ERROR << "Table name character isn't underscore or alphanumber: " << name_char; - return SERVER_INVALID_TABLE_NAME; + std::string msg = "Table name character isn't underscore or alphanumber"; + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_TABLE_NAME, msg); } } - return SERVER_SUCCESS; + return Status::OK(); } -ErrorCode +Status ValidationUtil::ValidateTableDimension(int64_t dimension) { if (dimension <= 0 || dimension > TABLE_DIMENSION_LIMIT) { - SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << TABLE_DIMENSION_LIMIT; - return SERVER_INVALID_VECTOR_DIMENSION; + std::string msg = "Table dimension excceed the limitation: " + std::to_string(TABLE_DIMENSION_LIMIT); + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_VECTOR_DIMENSION, msg); } else { - return SERVER_SUCCESS; + return Status::OK(); } } -ErrorCode +Status ValidationUtil::ValidateTableIndexType(int32_t index_type) { int engine_type = (int) engine::EngineType(index_type); if (engine_type <= 0 || engine_type > (int) engine::EngineType::MAX_VALUE) { - return SERVER_INVALID_INDEX_TYPE; + std::string msg = "Invalid index type: " + std::to_string(index_type); + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_INDEX_TYPE, msg); } - return SERVER_SUCCESS; + return Status::OK(); } -ErrorCode +Status ValidationUtil::ValidateTableIndexNlist(int32_t nlist) { if (nlist <= 0) { - return SERVER_INVALID_INDEX_NLIST; + std::string msg = "Invalid nlist value: " + std::to_string(nlist); + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_INDEX_NLIST, msg); } - return SERVER_SUCCESS; + return Status::OK(); } -ErrorCode +Status ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) { if (index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) { - return SERVER_INVALID_INDEX_FILE_SIZE; + std::string msg = "Invalid index file size: " + std::to_string(index_file_size); + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_INDEX_FILE_SIZE, msg); } - return SERVER_SUCCESS; + return Status::OK(); } -ErrorCode +Status ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { if (metric_type != (int32_t) engine::MetricType::L2 && metric_type != (int32_t) engine::MetricType::IP) { - return SERVER_INVALID_INDEX_METRIC_TYPE; + std::string msg = "Invalid metric type: " + std::to_string(metric_type); + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_INDEX_METRIC_TYPE, msg); } - return SERVER_SUCCESS; + return Status::OK(); } -ErrorCode +Status ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema &table_schema) { if (top_k <= 0 || top_k > 2048) { - return SERVER_INVALID_TOPK; + std::string msg = "Invalid top k value: " + std::to_string(top_k); + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_TOPK, msg); } - return SERVER_SUCCESS; + return Status::OK(); } -ErrorCode +Status ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema &table_schema) { if (nprobe <= 0 || nprobe > table_schema.nlist_) { - return SERVER_INVALID_NPROBE; + std::string msg = "Invalid nprobe value: " + std::to_string(nprobe); + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_NPROBE, msg); } - return SERVER_SUCCESS; + return Status::OK(); } -ErrorCode +Status ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { int num_devices = 0; auto cuda_err = cudaGetDeviceCount(&num_devices); if (cuda_err) { - SERVER_LOG_ERROR << "Failed to count video card: " << std::to_string(cuda_err); - return SERVER_UNEXPECTED_ERROR; + std::string msg = "Failed to get gpu card number, cuda error:" + std::to_string(cuda_err); + SERVER_LOG_ERROR << msg; + return Status(SERVER_UNEXPECTED_ERROR, msg); } if (gpu_index >= num_devices) { - return SERVER_INVALID_ARGUMENT; + std::string msg = "Invalid gpu index: " + std::to_string(gpu_index); + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_ARGUMENT, msg); } - return SERVER_SUCCESS; + return Status::OK(); } -ErrorCode +Status ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t &memory) { cudaDeviceProp deviceProp; auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index); if (cuda_err) { - SERVER_LOG_ERROR << "Failed to get video card properties: " << std::to_string(cuda_err); - return SERVER_UNEXPECTED_ERROR; + std::string msg = "Failed to get gpu properties, cuda error:" + std::to_string(cuda_err); + SERVER_LOG_ERROR << msg; + return Status(SERVER_UNEXPECTED_ERROR, msg); } memory = deviceProp.totalGlobalMem; - return SERVER_SUCCESS; + return Status::OK(); } -ErrorCode +Status ValidationUtil::ValidateIpAddress(const std::string &ip_address) { struct in_addr address; @@ -172,50 +191,56 @@ ValidationUtil::ValidateIpAddress(const std::string &ip_address) { int result = inet_pton(AF_INET, ip_address.c_str(), &address); switch (result) { - case 1:return SERVER_SUCCESS; - case 0:SERVER_LOG_ERROR << "Invalid IP address: " << ip_address; - return SERVER_INVALID_ARGUMENT; - default:SERVER_LOG_ERROR << "inet_pton conversion error"; - return SERVER_UNEXPECTED_ERROR; + case 1:return Status::OK(); + case 0: { + std::string msg = "Invalid IP address: " + ip_address; + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + default: { + std::string msg = "IP address conversion error: " + ip_address; + SERVER_LOG_ERROR << msg; + return Status(SERVER_UNEXPECTED_ERROR, msg); + } } } -ErrorCode +Status ValidationUtil::ValidateStringIsNumber(const std::string &string) { if (!string.empty() && std::all_of(string.begin(), string.end(), ::isdigit)) { - return SERVER_SUCCESS; + return Status::OK(); } else { - return SERVER_INVALID_ARGUMENT; + return Status(SERVER_INVALID_ARGUMENT, "Not a number"); } } -ErrorCode +Status ValidationUtil::ValidateStringIsBool(std::string &str) { std::transform(str.begin(), str.end(), str.begin(), ::tolower); if (str == "true" || str == "on" || str == "yes" || str == "1" || str == "false" || str == "off" || str == "no" || str == "0" || str.empty()) { - return SERVER_SUCCESS; + return Status::OK(); } else { - return SERVER_INVALID_ARGUMENT; + return Status(SERVER_INVALID_ARGUMENT, "Not a boolean: " + str); } } -ErrorCode +Status ValidationUtil::ValidateStringIsDouble(const std::string &str, double &val) { char *end = nullptr; val = std::strtod(str.c_str(), &end); if (end != str.c_str() && *end == '\0' && val != HUGE_VAL) { - return SERVER_SUCCESS; + return Status::OK(); } else { - return SERVER_INVALID_ARGUMENT; + return Status(SERVER_INVALID_ARGUMENT, "Not a double value: " + str); } } -ErrorCode +Status ValidationUtil::ValidateDbURI(const std::string &uri) { std::string dialectRegex = "(.*)"; std::string usernameRegex = "(.*)"; @@ -256,7 +281,8 @@ ValidationUtil::ValidateDbURI(const std::string &uri) { std::string port = pieces_match[5].str(); if (!port.empty()) { - if (ValidateStringIsNumber(port) != SERVER_SUCCESS) { + auto status = ValidateStringIsNumber(port); + if (!status.ok()) { SERVER_LOG_ERROR << "Invalid port in uri = " << port; okay = false; } @@ -267,7 +293,7 @@ ValidationUtil::ValidateDbURI(const std::string &uri) { okay = false; } - return (okay ? SERVER_SUCCESS : SERVER_INVALID_ARGUMENT); + return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid db backend uri")); } } diff --git a/cpp/src/utils/ValidationUtil.h b/cpp/src/utils/ValidationUtil.h index ed443e9165..0c109e0e25 100644 --- a/cpp/src/utils/ValidationUtil.h +++ b/cpp/src/utils/ValidationUtil.h @@ -19,7 +19,7 @@ #pragma once #include "db/meta/MetaTypes.h" -#include "Error.h" +#include "utils/Status.h" namespace zilliz { namespace milvus { @@ -27,49 +27,49 @@ namespace server { class ValidationUtil { public: - static ErrorCode + static Status ValidateTableName(const std::string &table_name); - static ErrorCode + static Status ValidateTableDimension(int64_t dimension); - static ErrorCode + static Status ValidateTableIndexType(int32_t index_type); - static ErrorCode + static Status ValidateTableIndexNlist(int32_t nlist); - static ErrorCode + static Status ValidateTableIndexFileSize(int64_t index_file_size); - static ErrorCode + static Status ValidateTableIndexMetricType(int32_t metric_type); - static ErrorCode + static Status ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema); - static ErrorCode + static Status ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema); - static ErrorCode + static Status ValidateGpuIndex(uint32_t gpu_index); - static ErrorCode + static Status GetGpuMemory(uint32_t gpu_index, size_t &memory); - static ErrorCode + static Status ValidateIpAddress(const std::string &ip_address); - static ErrorCode + static Status ValidateStringIsNumber(const std::string &str); - static ErrorCode + static Status ValidateStringIsBool(std::string &str); - static ErrorCode + static Status ValidateStringIsDouble(const std::string &str, double &val); - static ErrorCode + static Status ValidateDbURI(const std::string &uri); }; diff --git a/cpp/unittest/server/config_test.cpp b/cpp/unittest/server/config_test.cpp index d53d674b37..13089e914b 100644 --- a/cpp/unittest/server/config_test.cpp +++ b/cpp/unittest/server/config_test.cpp @@ -107,11 +107,11 @@ TEST(ConfigTest, CONFIG_TEST) { TEST(ConfigTest, SERVER_CONFIG_TEST) { server::ServerConfig& config = server::ServerConfig::GetInstance(); - ErrorCode err = config.LoadConfigFile(CONFIG_FILE_PATH); - ASSERT_EQ(err, SERVER_SUCCESS); + auto status = config.LoadConfigFile(CONFIG_FILE_PATH); + ASSERT_TRUE(status.ok()); - err = server::ServerConfig::GetInstance().ValidateConfig(); - ASSERT_EQ(err, SERVER_SUCCESS); + status = server::ServerConfig::GetInstance().ValidateConfig(); + ASSERT_TRUE(status.ok()); const server::ServerConfig& config_const = config; server::ConfigNode node1 = config_const.GetConfig("server_config"); @@ -137,23 +137,23 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) { server::ConfigNode& db_config = config.GetConfig("db_config"); server::ConfigNode& cache_config = config.GetConfig(server::CONFIG_CACHE); cache_config.SetValue(server::CACHE_FREE_PERCENT, "2.0"); - err = config.ValidateConfig(); - ASSERT_NE(err, SERVER_SUCCESS); + status = config.ValidateConfig(); + ASSERT_FALSE(status.ok()); size_t cache_cap = 16; size_t insert_buffer_size = (total_mem - cache_cap*GB + 1*GB)/GB; db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size)); cache_config.SetValue(server::CONFIG_CPU_CACHE_CAPACITY, std::to_string(cache_cap)); - err = config.ValidateConfig(); - ASSERT_NE(err, SERVER_SUCCESS); + status = config.ValidateConfig(); + ASSERT_FALSE(status.ok()); cache_cap = total_mem/GB + 2; cache_config.SetValue(server::CONFIG_CPU_CACHE_CAPACITY, std::to_string(cache_cap)); - err = config.ValidateConfig(); - ASSERT_NE(err, SERVER_SUCCESS); + status = config.ValidateConfig(); + ASSERT_FALSE(status.ok()); insert_buffer_size = total_mem/GB + 2; db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size)); - err = config.ValidateConfig(); - ASSERT_NE(err, SERVER_SUCCESS); + status = config.ValidateConfig(); + ASSERT_FALSE(status.ok()); } \ No newline at end of file diff --git a/cpp/unittest/server/rpc_test.cpp b/cpp/unittest/server/rpc_test.cpp index 09a8a1e461..b87ead099a 100644 --- a/cpp/unittest/server/rpc_test.cpp +++ b/cpp/unittest/server/rpc_test.cpp @@ -431,9 +431,9 @@ TEST_F(RpcHandlerTest, DeleteByRangeTest) { ////////////////////////////////////////////////////////////////////// class DummyTask : public GrpcBaseTask { public: - ErrorCode + Status OnExecute() override { - return 0; + return Status::OK(); } static BaseTaskPtr @@ -441,11 +441,6 @@ class DummyTask : public GrpcBaseTask { return std::shared_ptr(new DummyTask(dummy)); } - ErrorCode - DummySetError(ErrorCode error_code, const std::string &msg) { - return SetError(error_code, msg); - } - public: explicit DummyTask(std::string &dummy) : GrpcBaseTask(dummy) { @@ -464,11 +459,8 @@ class RpcSchedulerTest : public testing::Test { }; TEST_F(RpcSchedulerTest, BaseTaskTest){ - ErrorCode error_code = task_ptr->Execute(); - ASSERT_EQ(error_code, 0); - - error_code = task_ptr->DummySetError(0, "test error"); - ASSERT_EQ(error_code, 0); + auto status = task_ptr->Execute(); + ASSERT_TRUE(status.ok()); GrpcRequestScheduler::GetInstance().Start(); ::milvus::grpc::Status grpc_status; diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index cfbcbab683..00be2f3f13 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -64,19 +64,19 @@ TEST(UtilTest, COMMON_TEST) { std::string path1 = "/tmp/milvus_test/"; std::string path2 = path1 + "common_test_12345/"; std::string path3 = path2 + "abcdef"; - ErrorCode err = server::CommonUtil::CreateDirectory(path3); - ASSERT_EQ(err, SERVER_SUCCESS); + Status status = server::CommonUtil::CreateDirectory(path3); + ASSERT_TRUE(status.ok()); //test again - err = server::CommonUtil::CreateDirectory(path3); - ASSERT_EQ(err, SERVER_SUCCESS); + status = server::CommonUtil::CreateDirectory(path3); + ASSERT_TRUE(status.ok()); ASSERT_TRUE(server::CommonUtil::IsDirectoryExist(path3)); - err = server::CommonUtil::DeleteDirectory(path1); - ASSERT_EQ(err, SERVER_SUCCESS); + status = server::CommonUtil::DeleteDirectory(path1); + ASSERT_TRUE(status.ok()); //test again - err = server::CommonUtil::DeleteDirectory(path1); - ASSERT_EQ(err, SERVER_SUCCESS); + status = server::CommonUtil::DeleteDirectory(path1); + ASSERT_TRUE(status.ok()); ASSERT_FALSE(server::CommonUtil::IsDirectoryExist(path1)); ASSERT_FALSE(server::CommonUtil::IsFileExist(path1)); @@ -114,24 +114,24 @@ TEST(UtilTest, STRINGFUNCTIONS_TEST) { str = "a,b,c"; std::vector result; - ErrorCode err = server::StringHelpFunctions::SplitStringByDelimeter(str , ",", result); - ASSERT_EQ(err, SERVER_SUCCESS); + auto status = server::StringHelpFunctions::SplitStringByDelimeter(str , ",", result); + ASSERT_TRUE(status.ok()); ASSERT_EQ(result.size(), 3UL); result.clear(); - err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result); - ASSERT_EQ(err, SERVER_SUCCESS); + status = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result); + ASSERT_TRUE(status.ok()); ASSERT_EQ(result.size(), 3UL); result.clear(); - err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "", result); - ASSERT_EQ(err, SERVER_SUCCESS); + status = server::StringHelpFunctions::SplitStringByQuote(str , ",", "", result); + ASSERT_TRUE(status.ok()); ASSERT_EQ(result.size(), 3UL); str = "55,\"aa,gg,yy\",b"; result.clear(); - err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result); - ASSERT_EQ(err, SERVER_SUCCESS); + status = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result); + ASSERT_TRUE(status.ok()); ASSERT_EQ(result.size(), 3UL); @@ -227,117 +227,117 @@ TEST(UtilTest, STATUS_TEST) { TEST(ValidationUtilTest, VALIDATE_TABLENAME_TEST) { std::string table_name = "Normal123_"; - ErrorCode res = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_SUCCESS); + auto status = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_TRUE(status.ok()); table_name = "12sds"; - res = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); + status = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), SERVER_INVALID_TABLE_NAME); table_name = ""; - res = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); + status = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), SERVER_INVALID_TABLE_NAME); table_name = "_asdasd"; - res = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_SUCCESS); + status = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), SERVER_SUCCESS); table_name = "!@#!@"; - res = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); + status = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), SERVER_INVALID_TABLE_NAME); table_name = "_!@#!@"; - res = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); + status = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), SERVER_INVALID_TABLE_NAME); table_name = "中文"; - res = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); + status = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), SERVER_INVALID_TABLE_NAME); table_name = std::string(10000, 'a'); - res = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); + status = server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), SERVER_INVALID_TABLE_NAME); } TEST(ValidationUtilTest, VALIDATE_DIMENSION_TEST) { - ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(-1), SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(0), SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16385), SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16384), SERVER_SUCCESS); - ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(1), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(-1).code(), SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(0).code(), SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16385).code(), SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16384).code(), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(1).code(), SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_INDEX_TEST) { - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::INVALID), SERVER_INVALID_INDEX_TYPE); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::INVALID).code(), SERVER_INVALID_INDEX_TYPE); for(int i = 1; i <= (int)engine::EngineType::MAX_VALUE; i++) { - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType(i), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType(i).code(), SERVER_SUCCESS); } - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), SERVER_INVALID_INDEX_TYPE); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1).code(), SERVER_INVALID_INDEX_TYPE); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexNlist(0), SERVER_INVALID_INDEX_NLIST); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexNlist(100), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexNlist(0).code(), SERVER_INVALID_INDEX_NLIST); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexNlist(100).code(), SERVER_SUCCESS); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexFileSize(0), SERVER_INVALID_INDEX_FILE_SIZE); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexFileSize(100), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexFileSize(0).code(), SERVER_INVALID_INDEX_FILE_SIZE); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexFileSize(100).code(), SERVER_SUCCESS); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(0), SERVER_INVALID_INDEX_METRIC_TYPE); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(1), SERVER_SUCCESS); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(2), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(0).code(), SERVER_INVALID_INDEX_METRIC_TYPE); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(1).code(), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(2).code(), SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_TOPK_TEST) { engine::meta::TableSchema schema; - ASSERT_EQ(server::ValidationUtil::ValidateSearchTopk(10, schema), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(65536, schema), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(0, schema), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateSearchTopk(10, schema).code(), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(65536, schema).code(), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(0, schema).code(), SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_NPROBE_TEST) { engine::meta::TableSchema schema; schema.nlist_ = 100; - ASSERT_EQ(server::ValidationUtil::ValidateSearchNprobe(10, schema), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateSearchNprobe(0, schema), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateSearchNprobe(101, schema), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateSearchNprobe(10, schema).code(), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateSearchNprobe(0, schema).code(), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateSearchNprobe(101, schema).code(), SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_GPU_TEST) { - ASSERT_EQ(server::ValidationUtil::ValidateGpuIndex(0), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateGpuIndex(100), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateGpuIndex(0).code(), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateGpuIndex(100).code(), SERVER_SUCCESS); size_t memory = 0; - ASSERT_EQ(server::ValidationUtil::GetGpuMemory(0, memory), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::GetGpuMemory(100, memory), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::GetGpuMemory(0, memory).code(), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::GetGpuMemory(100, memory).code(), SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_IPADDRESS_TEST) { - ASSERT_EQ(server::ValidationUtil::ValidateIpAddress("127.0.0.1"), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateIpAddress("not ip"), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateIpAddress("127.0.0.1").code(), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateIpAddress("not ip").code(), SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_NUMBER_TEST) { - ASSERT_EQ(server::ValidationUtil::ValidateStringIsNumber("1234"), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateStringIsNumber("not number"), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateStringIsNumber("1234").code(), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateStringIsNumber("not number").code(), SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_BOOL_TEST) { std::string str = "true"; - ASSERT_EQ(server::ValidationUtil::ValidateStringIsBool(str), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateStringIsBool(str).code(), SERVER_SUCCESS); str = "not bool"; - ASSERT_NE(server::ValidationUtil::ValidateStringIsBool(str), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateStringIsBool(str).code(), SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_DOUBLE_TEST) { double ret = 0.0; - ASSERT_EQ(server::ValidationUtil::ValidateStringIsDouble("2.5", ret), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateStringIsDouble("not double", ret), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateStringIsDouble("2.5", ret).code(), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateStringIsDouble("not double", ret).code(), SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_DBURI_TEST) { - ASSERT_EQ(server::ValidationUtil::ValidateDbURI("sqlite://:@:/"), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateDbURI("xxx://:@:/"), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateDbURI("not uri"), SERVER_SUCCESS); - ASSERT_EQ(server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:3303/milvus"), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:port/milvus"), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateDbURI("sqlite://:@:/").code(), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateDbURI("xxx://:@:/").code(), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateDbURI("not uri").code(), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:3303/milvus").code(), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:port/milvus").code(), SERVER_SUCCESS); } TEST(UtilTest, ROLLOUTHANDLER_TEST){ From 8eaf03a279b1873f6df7791ec39445a640085c2f Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 18 Sep 2019 15:05:51 +0800 Subject: [PATCH 025/217] refine code Former-commit-id: 710c122428d52f56a7bf41903afa4a7f8c6db01e --- cpp/src/db/DBFactory.cpp | 6 +- cpp/src/db/DBFactory.h | 4 +- cpp/src/db/DBImpl.cpp | 8 +- cpp/src/db/DBImpl.h | 40 +++++----- cpp/src/db/Options.cpp | 3 - cpp/src/db/Options.h | 5 +- cpp/src/db/insert/MemManagerImpl.h | 4 +- cpp/src/db/insert/MemMenagerFactory.cpp | 3 +- cpp/src/db/insert/MemMenagerFactory.h | 2 +- cpp/src/db/insert/MemTable.cpp | 2 +- cpp/src/db/insert/MemTable.h | 4 +- cpp/src/db/insert/MemTableFile.cpp | 2 +- cpp/src/db/insert/MemTableFile.h | 4 +- cpp/src/db/meta/MySQLMetaImpl.cpp | 4 +- cpp/src/main.cpp | 6 +- cpp/src/server/DBWrapper.cpp | 11 ++- cpp/src/server/Server.cpp | 4 +- cpp/src/server/Server.h | 15 ++-- cpp/src/server/grpc_impl/GrpcMilvusServer.cpp | 8 +- cpp/src/utils/BlockingQueue.inl | 28 +------ cpp/src/utils/CommonUtil.cpp | 5 ++ cpp/src/utils/CommonUtil.h | 1 + cpp/src/utils/LogUtil.cpp | 40 +--------- cpp/src/utils/LogUtil.h | 10 +-- cpp/src/utils/SignalUtil.cpp | 18 ++--- cpp/src/utils/ValidationUtil.h | 3 + cpp/unittest/db/mysql_meta_test.cpp | 4 +- cpp/unittest/db/utils.cpp | 8 +- cpp/unittest/db/utils.h | 8 +- cpp/unittest/metrics/utils.cpp | 2 +- cpp/unittest/metrics/utils.h | 2 +- cpp/unittest/server/rpc_test.cpp | 80 +++++++++---------- cpp/unittest/server/util_test.cpp | 11 ++- 33 files changed, 150 insertions(+), 205 deletions(-) diff --git a/cpp/src/db/DBFactory.cpp b/cpp/src/db/DBFactory.cpp index eb13c309fc..91e03278d6 100644 --- a/cpp/src/db/DBFactory.cpp +++ b/cpp/src/db/DBFactory.cpp @@ -33,14 +33,14 @@ namespace zilliz { namespace milvus { namespace engine { -Options DBFactory::BuildOption() { +DBOptions DBFactory::BuildOption() { auto meta = MetaFactory::BuildOption(); - Options options; + DBOptions options; options.meta = meta; return options; } -DBPtr DBFactory::Build(const Options& options) { +DBPtr DBFactory::Build(const DBOptions& options) { return std::make_shared(options); } diff --git a/cpp/src/db/DBFactory.h b/cpp/src/db/DBFactory.h index 7de4396846..bb363118ed 100644 --- a/cpp/src/db/DBFactory.h +++ b/cpp/src/db/DBFactory.h @@ -29,9 +29,9 @@ namespace engine { class DBFactory { public: - static Options BuildOption(); + static DBOptions BuildOption(); - static DBPtr Build(const Options& options); + static DBPtr Build(const DBOptions& options); }; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 648038de25..0dec98b92b 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -51,7 +51,7 @@ constexpr uint64_t INDEX_ACTION_INTERVAL = 1; } -DBImpl::DBImpl(const Options& options) +DBImpl::DBImpl(const DBOptions& options) : options_(options), shutting_down_(true), compact_thread_pool_(1, 1), @@ -77,7 +77,7 @@ Status DBImpl::Start() { shutting_down_.store(false, std::memory_order_release); //for distribute version, some nodes are read only - if (options_.mode != Options::MODE::READ_ONLY) { + if (options_.mode != DBOptions::MODE::READ_ONLY) { ENGINE_LOG_TRACE << "StartTimerTasks"; bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this); } @@ -98,7 +98,7 @@ Status DBImpl::Stop() { //wait compaction/buildindex finish bg_timer_thread_.join(); - if (options_.mode != Options::MODE::READ_ONLY) { + if (options_.mode != DBOptions::MODE::READ_ONLY) { meta_ptr_->CleanUp(); } @@ -687,7 +687,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes - if (options_.mode == Options::MODE::CLUSTER) { + if (options_.mode == DBOptions::MODE::CLUSTER) { ttl = meta::D_SEC; } meta_ptr_->CleanUpFilesWithTTL(ttl); diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 85803b763f..30e9667994 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -44,7 +44,7 @@ class Meta; class DBImpl : public DB { public: - explicit DBImpl(const Options &options); + explicit DBImpl(const DBOptions &options); ~DBImpl(); Status Start() override; @@ -76,28 +76,28 @@ class DBImpl : public DB { Status DropIndex(const std::string& table_id) override; Status Query(const std::string &table_id, - uint64_t k, - uint64_t nq, - uint64_t nprobe, - const float *vectors, - QueryResults &results) override; + uint64_t k, + uint64_t nq, + uint64_t nprobe, + const float *vectors, + QueryResults &results) override; Status Query(const std::string &table_id, - uint64_t k, - uint64_t nq, - uint64_t nprobe, - const float *vectors, - const meta::DatesT &dates, - QueryResults &results) override; + uint64_t k, + uint64_t nq, + uint64_t nprobe, + const float *vectors, + const meta::DatesT &dates, + QueryResults &results) override; Status Query(const std::string &table_id, - const std::vector &file_ids, - uint64_t k, - uint64_t nq, - uint64_t nprobe, - const float *vectors, - const meta::DatesT &dates, - QueryResults &results) override; + const std::vector &file_ids, + uint64_t k, + uint64_t nq, + uint64_t nprobe, + const float *vectors, + const meta::DatesT &dates, + QueryResults &results) override; Status Size(uint64_t &result) override; @@ -130,7 +130,7 @@ class DBImpl : public DB { Status MemSerialize(); private: - const Options options_; + const DBOptions options_; std::atomic shutting_down_; diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index 9abc8e66ec..5796b781b2 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -27,9 +27,6 @@ namespace zilliz { namespace milvus { namespace engine { -Options::Options() { -} - ArchiveConf::ArchiveConf(const std::string& type, const std::string& criterias) { ParseType(type); ParseCritirias(criterias); diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index c5b4648c6d..663ac3fd09 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -58,16 +58,13 @@ struct DBMetaOptions { ArchiveConf archive_conf = ArchiveConf("delete"); }; // DBMetaOptions -struct Options { - +struct DBOptions { typedef enum { SINGLE, CLUSTER, READ_ONLY } MODE; - Options(); - uint16_t merge_trigger_number = 2; DBMetaOptions meta; int mode = MODE::SINGLE; diff --git a/cpp/src/db/insert/MemManagerImpl.h b/cpp/src/db/insert/MemManagerImpl.h index 51e8f1044e..767dc0eef0 100644 --- a/cpp/src/db/insert/MemManagerImpl.h +++ b/cpp/src/db/insert/MemManagerImpl.h @@ -38,7 +38,7 @@ class MemManagerImpl : public MemManager { public: using Ptr = std::shared_ptr; - MemManagerImpl(const meta::MetaPtr &meta, const Options &options) + MemManagerImpl(const meta::MetaPtr &meta, const DBOptions &options) : meta_(meta), options_(options) {} Status InsertVectors(const std::string &table_id, @@ -66,7 +66,7 @@ class MemManagerImpl : public MemManager { MemIdMap mem_id_map_; MemList immu_mem_list_; meta::MetaPtr meta_; - Options options_; + DBOptions options_; std::mutex mutex_; std::mutex serialization_mtx_; }; // NewMemManager diff --git a/cpp/src/db/insert/MemMenagerFactory.cpp b/cpp/src/db/insert/MemMenagerFactory.cpp index d8620e908c..d0ef51f61f 100644 --- a/cpp/src/db/insert/MemMenagerFactory.cpp +++ b/cpp/src/db/insert/MemMenagerFactory.cpp @@ -31,8 +31,7 @@ namespace zilliz { namespace milvus { namespace engine { -MemManagerPtr MemManagerFactory::Build(const std::shared_ptr& meta, - const Options& options) { +MemManagerPtr MemManagerFactory::Build(const std::shared_ptr& meta, const DBOptions& options) { return std::make_shared(meta, options); } diff --git a/cpp/src/db/insert/MemMenagerFactory.h b/cpp/src/db/insert/MemMenagerFactory.h index 4bf4267c87..d38404f5c0 100644 --- a/cpp/src/db/insert/MemMenagerFactory.h +++ b/cpp/src/db/insert/MemMenagerFactory.h @@ -26,7 +26,7 @@ namespace engine { class MemManagerFactory { public: - static MemManagerPtr Build(const std::shared_ptr &meta, const Options &options); + static MemManagerPtr Build(const std::shared_ptr &meta, const DBOptions &options); }; } diff --git a/cpp/src/db/insert/MemTable.cpp b/cpp/src/db/insert/MemTable.cpp index 7f4f497ba1..4be0695572 100644 --- a/cpp/src/db/insert/MemTable.cpp +++ b/cpp/src/db/insert/MemTable.cpp @@ -26,7 +26,7 @@ namespace engine { MemTable::MemTable(const std::string &table_id, const meta::MetaPtr &meta, - const Options &options) : + const DBOptions &options) : table_id_(table_id), meta_(meta), options_(options) { diff --git a/cpp/src/db/insert/MemTable.h b/cpp/src/db/insert/MemTable.h index def4a6dcca..c1b4125331 100644 --- a/cpp/src/db/insert/MemTable.h +++ b/cpp/src/db/insert/MemTable.h @@ -33,7 +33,7 @@ class MemTable { public: using MemTableFileList = std::vector; - MemTable(const std::string &table_id, const meta::MetaPtr &meta, const Options &options); + MemTable(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options); Status Add(VectorSourcePtr &source, IDNumbers &vector_ids); @@ -56,7 +56,7 @@ class MemTable { meta::MetaPtr meta_; - Options options_; + DBOptions options_; std::mutex mutex_; diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index 57ef39c670..12dbbcf49f 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -31,7 +31,7 @@ namespace engine { MemTableFile::MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, - const Options &options) : + const DBOptions &options) : table_id_(table_id), meta_(meta), options_(options) { diff --git a/cpp/src/db/insert/MemTableFile.h b/cpp/src/db/insert/MemTableFile.h index 5ac3e15ad2..168fabe7e9 100644 --- a/cpp/src/db/insert/MemTableFile.h +++ b/cpp/src/db/insert/MemTableFile.h @@ -31,7 +31,7 @@ namespace engine { class MemTableFile { public: - MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const Options &options); + MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options); Status Add(const VectorSourcePtr &source, IDNumbers& vector_ids); @@ -53,7 +53,7 @@ class MemTableFile { meta::MetaPtr meta_; - Options options_; + DBOptions options_; size_t current_mem_; diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index ccd7e79ada..a07d4c82cf 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -135,7 +135,7 @@ Status MySQLMetaImpl::Initialize() { ENGINE_LOG_DEBUG << "MySQL connection pool: maximum pool size = " << std::to_string(maxPoolSize); try { - if (mode_ != Options::MODE::READ_ONLY) { + if (mode_ != DBOptions::MODE::READ_ONLY) { CleanUp(); } @@ -621,7 +621,7 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { } //Scoped Connection - if (mode_ == Options::MODE::CLUSTER) { + if (mode_ == DBOptions::MODE::CLUSTER) { DeleteTableFiles(table_id); } diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index d9b160d6e0..3065e7d684 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -109,9 +109,9 @@ main(int argc, char *argv[]) { } } - server::Server* server_ptr = server::Server::Instance(); - server_ptr->Init(start_daemonized, pid_filename, config_filename, log_config_file); - return server_ptr->Start(); + server::Server& server = server::Server::Instance(); + server.Init(start_daemonized, pid_filename, config_filename, log_config_file); + return server.Start(); } void diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 508d9552b1..115a636621 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -24,6 +24,7 @@ #include "utils/StringHelpFunctions.h" #include +#include namespace zilliz { namespace milvus { @@ -35,7 +36,7 @@ DBWrapper::DBWrapper() { Status DBWrapper::StartService() { //db config - zilliz::milvus::engine::Options opt; + engine::DBOptions opt; ConfigNode& db_config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); opt.meta.backend_uri = db_config.GetValue(CONFIG_DB_URL); std::string db_path = db_config.GetValue(CONFIG_DB_PATH); @@ -51,13 +52,13 @@ Status DBWrapper::StartService() { ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); std::string mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); if (mode == "single") { - opt.mode = zilliz::milvus::engine::Options::MODE::SINGLE; + opt.mode = engine::DBOptions::MODE::SINGLE; } else if (mode == "cluster") { - opt.mode = zilliz::milvus::engine::Options::MODE::CLUSTER; + opt.mode = engine::DBOptions::MODE::CLUSTER; } else if (mode == "read_only") { - opt.mode = zilliz::milvus::engine::Options::MODE::READ_ONLY; + opt.mode = engine::DBOptions::MODE::READ_ONLY; } else { std::cerr << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" << std::endl; @@ -78,6 +79,8 @@ Status DBWrapper::StartService() { } } + faiss::distance_compute_blas_threshold = engine_config.GetInt32Value(CONFIG_DCBT, 20);//init faiss global variable + //set archive config engine::ArchiveConf::CriteriaT criterial; int64_t disk = db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK, 0); diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index e1e67a112f..252f92a5f0 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -42,10 +42,10 @@ namespace zilliz { namespace milvus { namespace server { -Server * +Server& Server::Instance() { static Server server; - return &server; + return server; } Server::Server() { diff --git a/cpp/src/server/Server.h b/cpp/src/server/Server.h index 43e8b28028..eaf19d6b15 100644 --- a/cpp/src/server/Server.h +++ b/cpp/src/server/Server.h @@ -27,15 +27,19 @@ namespace milvus { namespace server { class Server { - public: - static Server* Instance(); +public: + static Server &Instance(); + + void Init(int64_t daemonized, const std::string &pid_filename, const std::string &config_filename, + const std::string &log_config_file); - void Init(int64_t daemonized, const std::string& pid_filename, const std::string& config_filename, const std::string &log_config_file); int Start(); + void Stop(); - private: +private: Server(); + ~Server(); void Daemonize(); @@ -43,9 +47,10 @@ class Server { ErrorCode LoadConfig(); void StartService(); + void StopService(); - private: +private: int64_t daemonized_ = 0; int pid_fd = -1; std::string pid_filename_; diff --git a/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp b/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp index 73f8a7573e..6770e3111a 100644 --- a/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp @@ -17,10 +17,9 @@ #include "milvus.grpc.pb.h" #include "GrpcMilvusServer.h" -#include "../ServerConfig.h" -#include "../DBWrapper.h" +#include "server/ServerConfig.h" +#include "server/DBWrapper.h" #include "utils/Log.h" -#include "faiss/utils.h" #include "GrpcRequestHandler.h" #include @@ -47,6 +46,7 @@ static std::unique_ptr<::grpc::Server> server; constexpr long MESSAGE_SIZE = -1; +//this class is to check port occupation during server start class NoReusePortOption : public ::grpc::ServerBuilderOption { public: void UpdateArguments(::grpc::ChannelArguments *args) override { @@ -71,8 +71,6 @@ GrpcMilvusServer::StartService() { std::string address = server_config.GetValue(CONFIG_SERVER_ADDRESS, "127.0.0.1"); int32_t port = server_config.GetInt32Value(CONFIG_SERVER_PORT, 19530); - faiss::distance_compute_blas_threshold = engine_config.GetInt32Value(CONFIG_DCBT, 20); - std::string server_address(address + ":" + std::to_string(port)); ::grpc::ServerBuilder builder; diff --git a/cpp/src/utils/BlockingQueue.inl b/cpp/src/utils/BlockingQueue.inl index 8df38dd547..86237f3322 100644 --- a/cpp/src/utils/BlockingQueue.inl +++ b/cpp/src/utils/BlockingQueue.inl @@ -18,8 +18,6 @@ #pragma once -//#include "Log.h" -#include "Error.h" namespace zilliz { namespace milvus { @@ -31,14 +29,6 @@ BlockingQueue::Put(const T &task) { std::unique_lock lock(mtx); full_.wait(lock, [this] { return (queue_.size() < capacity_); }); - if (queue_.size() >= capacity_) { - std::string error_msg = - "blocking queue is full, capacity: " + std::to_string(capacity_) + " queue_size: " + - std::to_string(queue_.size()); - //SERVER_LOG_ERROR << error_msg; - throw ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg); - } - queue_.push(task); empty_.notify_all(); } @@ -49,12 +39,6 @@ BlockingQueue::Take() { std::unique_lock lock(mtx); empty_.wait(lock, [this] { return !queue_.empty(); }); - if (queue_.empty()) { - std::string error_msg = "blocking queue empty"; - //SERVER_LOG_ERROR << error_msg; - throw ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg); - } - T front(queue_.front()); queue_.pop(); full_.notify_all(); @@ -73,11 +57,7 @@ T BlockingQueue::Front() { std::unique_lock lock(mtx); empty_.wait(lock, [this] { return !queue_.empty(); }); - if (queue_.empty()) { - std::string error_msg = "blocking queue empty"; - //SERVER_LOG_ERROR << error_msg; - throw ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg); - } + T front(queue_.front()); return front; } @@ -88,12 +68,6 @@ BlockingQueue::Back() { std::unique_lock lock(mtx); empty_.wait(lock, [this] { return !queue_.empty(); }); - if (queue_.empty()) { - std::string error_msg = "blocking queue empty"; - //SERVER_LOG_ERROR << error_msg; - throw ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg); - } - T back(queue_.back()); return back; } diff --git a/cpp/src/utils/CommonUtil.cpp b/cpp/src/utils/CommonUtil.cpp index 19a63006cd..ff985dcd7d 100644 --- a/cpp/src/utils/CommonUtil.cpp +++ b/cpp/src/utils/CommonUtil.cpp @@ -162,6 +162,11 @@ uint64_t CommonUtil::GetFileSize(const std::string &path) { } } +std::string CommonUtil::GetFileName(std::string filename) { + int pos = filename.find_last_of('/'); + return filename.substr(pos + 1); +} + std::string CommonUtil::GetExePath() { const size_t buf_len = 1024; char buf[buf_len]; diff --git a/cpp/src/utils/CommonUtil.h b/cpp/src/utils/CommonUtil.h index aee57f0446..7e1349bf9e 100755 --- a/cpp/src/utils/CommonUtil.h +++ b/cpp/src/utils/CommonUtil.h @@ -38,6 +38,7 @@ class CommonUtil { static Status CreateDirectory(const std::string &path); static Status DeleteDirectory(const std::string &path); + static std::string GetFileName(std::string filename); static std::string GetExePath(); static bool TimeStrToTime(const std::string& time_str, diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp index 6224c854a5..e4af1104db 100644 --- a/cpp/src/utils/LogUtil.cpp +++ b/cpp/src/utils/LogUtil.cpp @@ -84,49 +84,15 @@ void RolloutHandler(const char *filename, std::size_t size, el::Level level) { } } -int32_t InitLog(const std::string &log_config_file) { -#if 0 - ServerConfig &config = ServerConfig::GetInstance(); - ConfigNode log_config = config.GetConfig(CONFIG_LOG); - const std::map& settings = log_config.GetChildren(); - - std::string str_config; - for(auto iter : settings) { - str_config += "* "; - str_config += iter.first; - str_config += ":"; - str_config.append("\n"); - - auto sub_configs = iter.second.GetConfig(); - for(auto it_sub : sub_configs) { - str_config += " "; - str_config += it_sub.first; - str_config += " = "; - std::string temp = it_sub.first; - std::transform(temp.begin(), temp.end(), temp.begin(), ::tolower); - bool is_text = (temp == "format" || temp == "filename"); - if(is_text){ - str_config += "\""; - } - str_config += it_sub.second; - if(is_text){ - str_config += "\""; - } - str_config.append("\n"); - } - } - - el::Configurations conf; - conf.parseFromText(str_config); -#else +Status InitLog(const std::string &log_config_file) { el::Configurations conf(log_config_file); -#endif el::Loggers::reconfigureAllLoggers(conf); el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck); el::Helpers::installPreRollOutCallback(RolloutHandler); el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog); - return 0; + + return Status::OK(); } diff --git a/cpp/src/utils/LogUtil.h b/cpp/src/utils/LogUtil.h index ebeb117aa2..d86b301d9f 100644 --- a/cpp/src/utils/LogUtil.h +++ b/cpp/src/utils/LogUtil.h @@ -17,6 +17,8 @@ #pragma once +#include "utils/Status.h" + #include #include #include "easylogging++.h" @@ -24,11 +26,9 @@ namespace zilliz { namespace milvus { namespace server { -int32_t InitLog(const std::string& log_config_file); -inline std::string GetFileName(std::string filename) { - int pos = filename.find_last_of('/'); - return filename.substr(pos + 1); -} + +Status InitLog(const std::string& log_config_file); + void RolloutHandler(const char *filename, std::size_t size, el::Level level); #define SHOW_LOCATION diff --git a/cpp/src/utils/SignalUtil.cpp b/cpp/src/utils/SignalUtil.cpp index 07c1001044..e380dc6c85 100644 --- a/cpp/src/utils/SignalUtil.cpp +++ b/cpp/src/utils/SignalUtil.cpp @@ -31,23 +31,19 @@ void SignalUtil::HandleSignal(int signum){ switch(signum){ case SIGINT: case SIGUSR2:{ - server::Server* server_ptr = server::Server::Instance(); - server_ptr->Stop(); + SERVER_LOG_INFO << "Server received signal:" << std::to_string(signum); + + server::Server& server_ptr = server::Server::Instance(); + server_ptr.Stop(); exit(0); } default:{ - SERVER_LOG_INFO << "Server received signal:" << std::to_string(signum); + SERVER_LOG_INFO << "Server received critical signal:" << std::to_string(signum); SignalUtil::PrintStacktrace(); - std::string info = "Server encounter critical signal:"; - info += std::to_string(signum); -// SendSignalMessage(signum, info); - - SERVER_LOG_INFO << info; - - server::Server* server_ptr = server::Server::Instance(); - server_ptr->Stop(); + server::Server& server_ptr = server::Server::Instance(); + server_ptr.Stop(); exit(1); } diff --git a/cpp/src/utils/ValidationUtil.h b/cpp/src/utils/ValidationUtil.h index 0c109e0e25..ad8ba5fee9 100644 --- a/cpp/src/utils/ValidationUtil.h +++ b/cpp/src/utils/ValidationUtil.h @@ -26,6 +26,9 @@ namespace milvus { namespace server { class ValidationUtil { +private: + ValidationUtil() = default; + public: static Status ValidateTableName(const std::string &table_name); diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index ab94c817fe..e99565886d 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -130,7 +130,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) { std::stringstream ss; ss << "days:" << days_num; options.archive_conf = ArchiveConf("delete", ss.str()); - int mode = Options::MODE::SINGLE; + int mode = DBOptions::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); auto table_id = "meta_test_table"; @@ -190,7 +190,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) { DBMetaOptions options = GetOptions().meta; options.archive_conf = ArchiveConf("delete", "disk:11"); - int mode = Options::MODE::SINGLE; + int mode = DBOptions::MODE::SINGLE; auto impl = meta::MySQLMetaImpl(options, mode); auto table_id = "meta_test_group"; diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 4bc9699e05..aededfd0bd 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -66,7 +66,7 @@ void BaseTest::TearDown() { zilliz::knowhere::FaissGpuResourceMgr::GetInstance().Free(); } -engine::Options BaseTest::GetOptions() { +engine::DBOptions BaseTest::GetOptions() { auto options = engine::DBFactory::BuildOption(); options.meta.path = "/tmp/milvus_test"; options.meta.backend_uri = "sqlite://:@:/"; @@ -108,7 +108,7 @@ void DBTest::TearDown() { } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// -engine::Options DBTest2::GetOptions() { +engine::DBOptions DBTest2::GetOptions() { auto options = engine::DBFactory::BuildOption(); options.meta.path = "/tmp/milvus_test"; options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1"); @@ -134,7 +134,7 @@ void MetaTest::TearDown() { } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// -engine::Options MySqlDBTest::GetOptions() { +engine::DBOptions MySqlDBTest::GetOptions() { auto options = engine::DBFactory::BuildOption(); options.meta.path = "/tmp/milvus_test"; options.meta.backend_uri = DBTestEnvironment::getURI(); @@ -163,7 +163,7 @@ void MySqlMetaTest::TearDown() { boost::filesystem::remove_all(options.meta.path); } -zilliz::milvus::engine::Options MySqlMetaTest::GetOptions() { +engine::DBOptions MySqlMetaTest::GetOptions() { auto options = engine::DBFactory::BuildOption(); options.meta.path = "/tmp/milvus_test"; options.meta.backend_uri = DBTestEnvironment::getURI(); diff --git a/cpp/unittest/db/utils.h b/cpp/unittest/db/utils.h index c1dc730089..7a1320ef6b 100644 --- a/cpp/unittest/db/utils.h +++ b/cpp/unittest/db/utils.h @@ -50,7 +50,7 @@ protected: virtual void SetUp() override; virtual void TearDown() override; - virtual zilliz::milvus::engine::Options GetOptions(); + virtual zilliz::milvus::engine::DBOptions GetOptions(); }; ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -65,7 +65,7 @@ class DBTest : public BaseTest { ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DBTest2 : public DBTest { protected: - virtual zilliz::milvus::engine::Options GetOptions() override; + virtual zilliz::milvus::engine::DBOptions GetOptions() override; }; ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -84,7 +84,7 @@ class MetaTest : public BaseTest { ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// class MySqlDBTest : public DBTest { protected: - zilliz::milvus::engine::Options GetOptions(); + zilliz::milvus::engine::DBOptions GetOptions(); }; ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -94,7 +94,7 @@ class MySqlMetaTest : public BaseTest { virtual void SetUp() override; virtual void TearDown() override; - zilliz::milvus::engine::Options GetOptions(); + zilliz::milvus::engine::DBOptions GetOptions(); }; diff --git a/cpp/unittest/metrics/utils.cpp b/cpp/unittest/metrics/utils.cpp index 4dc370c952..49ccc53d30 100644 --- a/cpp/unittest/metrics/utils.cpp +++ b/cpp/unittest/metrics/utils.cpp @@ -52,7 +52,7 @@ void MetricTest::InitLog() { el::Loggers::reconfigureLogger("default", defaultConf); } -engine::Options MetricTest::GetOptions() { +engine::DBOptions MetricTest::GetOptions() { auto options = engine::DBFactory::BuildOption(); options.meta.path = "/tmp/milvus_test"; options.meta.backend_uri = "sqlite://:@:/"; diff --git a/cpp/unittest/metrics/utils.h b/cpp/unittest/metrics/utils.h index 7f093dd091..37c21c82f9 100644 --- a/cpp/unittest/metrics/utils.h +++ b/cpp/unittest/metrics/utils.h @@ -72,5 +72,5 @@ protected: void InitLog(); virtual void SetUp() override; virtual void TearDown() override; - virtual zilliz::milvus::engine::Options GetOptions(); + virtual zilliz::milvus::engine::DBOptions GetOptions(); }; \ No newline at end of file diff --git a/cpp/unittest/server/rpc_test.cpp b/cpp/unittest/server/rpc_test.cpp index b87ead099a..f89c98eb9d 100644 --- a/cpp/unittest/server/rpc_test.cpp +++ b/cpp/unittest/server/rpc_test.cpp @@ -34,11 +34,9 @@ #include "scheduler/ResourceFactory.h" #include "utils/CommonUtil.h" +using namespace zilliz::milvus; -namespace zilliz { -namespace milvus { -namespace server { -namespace grpc { +namespace { static const char *TABLE_NAME = "test_grpc"; static constexpr int64_t TABLE_DIM = 256; @@ -65,35 +63,35 @@ class RpcHandlerTest : public testing::Test { res_mgr->Start(); engine::SchedInst::GetInstance()->Start(); - zilliz::milvus::engine::Options opt; + engine::DBOptions opt; - ConfigNode &db_config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); - db_config.SetValue(CONFIG_DB_URL, "sqlite://:@:/"); - db_config.SetValue(CONFIG_DB_PATH, "/tmp/milvus_test"); - db_config.SetValue(CONFIG_DB_SLAVE_PATH, ""); - db_config.SetValue(CONFIG_DB_ARCHIVE_DISK, ""); - db_config.SetValue(CONFIG_DB_ARCHIVE_DAYS, ""); + server::ConfigNode &db_config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_DB); + db_config.SetValue(server::CONFIG_DB_URL, "sqlite://:@:/"); + db_config.SetValue(server::CONFIG_DB_PATH, "/tmp/milvus_test"); + db_config.SetValue(server::CONFIG_DB_SLAVE_PATH, ""); + db_config.SetValue(server::CONFIG_DB_ARCHIVE_DISK, ""); + db_config.SetValue(server::CONFIG_DB_ARCHIVE_DAYS, ""); - ConfigNode &cache_config = ServerConfig::GetInstance().GetConfig(CONFIG_CACHE); - cache_config.SetValue(CONFIG_INSERT_CACHE_IMMEDIATELY, ""); + server::ConfigNode &cache_config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); + cache_config.SetValue(server::CONFIG_INSERT_CACHE_IMMEDIATELY, ""); - ConfigNode &engine_config = ServerConfig::GetInstance().GetConfig(CONFIG_ENGINE); - engine_config.SetValue(CONFIG_OMP_THREAD_NUM, ""); + server::ConfigNode &engine_config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_ENGINE); + engine_config.SetValue(server::CONFIG_OMP_THREAD_NUM, ""); - ConfigNode &serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); -// serverConfig.SetValue(CONFIG_CLUSTER_MODE, "cluster"); + server::ConfigNode &serverConfig = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_SERVER); +// serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "cluster"); // DBWrapper::GetInstance().GetInstance().StartService(); // DBWrapper::GetInstance().GetInstance().StopService(); // -// serverConfig.SetValue(CONFIG_CLUSTER_MODE, "read_only"); +// serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "read_only"); // DBWrapper::GetInstance().GetInstance().StartService(); // DBWrapper::GetInstance().GetInstance().StopService(); - serverConfig.SetValue(CONFIG_CLUSTER_MODE, "single"); - DBWrapper::GetInstance().GetInstance().StartService(); + serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "single"); + server::DBWrapper::GetInstance().StartService(); //initialize handler, create table - handler = std::make_shared(); + handler = std::make_shared(); ::grpc::ServerContext context; ::milvus::grpc::TableSchema request; ::milvus::grpc::Status status; @@ -106,16 +104,15 @@ class RpcHandlerTest : public testing::Test { void TearDown() override { - DBWrapper::GetInstance().StopService(); + server::DBWrapper::GetInstance().StopService(); engine::ResMgrInst::GetInstance()->Stop(); engine::SchedInst::GetInstance()->Stop(); boost::filesystem::remove_all("/tmp/milvus_test"); } protected: - std::shared_ptr handler; + std::shared_ptr handler; }; -namespace { void BuildVectors(int64_t from, int64_t to, std::vector> &vector_record_array) { if (to <= from) { @@ -146,6 +143,7 @@ std::string CurrentTmDate(int64_t offset_day = 0) { return str; } + } TEST_F(RpcHandlerTest, HasTableTest) { @@ -429,26 +427,27 @@ TEST_F(RpcHandlerTest, DeleteByRangeTest) { } ////////////////////////////////////////////////////////////////////// -class DummyTask : public GrpcBaseTask { - public: +namespace { +class DummyTask : public server::grpc::GrpcBaseTask { +public: Status OnExecute() override { return Status::OK(); } - static BaseTaskPtr - Create(std::string& dummy) { - return std::shared_ptr(new DummyTask(dummy)); + static server::grpc::BaseTaskPtr + Create(std::string &dummy) { + return std::shared_ptr(new DummyTask(dummy)); } - public: +public: explicit DummyTask(std::string &dummy) : GrpcBaseTask(dummy) { } }; class RpcSchedulerTest : public testing::Test { - protected: +protected: void SetUp() override { std::string dummy = "dql"; @@ -458,25 +457,22 @@ class RpcSchedulerTest : public testing::Test { std::shared_ptr task_ptr; }; +} + TEST_F(RpcSchedulerTest, BaseTaskTest){ auto status = task_ptr->Execute(); ASSERT_TRUE(status.ok()); - GrpcRequestScheduler::GetInstance().Start(); + server::grpc::GrpcRequestScheduler::GetInstance().Start(); ::milvus::grpc::Status grpc_status; std::string dummy = "dql"; - BaseTaskPtr base_task_ptr = DummyTask::Create(dummy); - GrpcRequestScheduler::GetInstance().ExecTask(base_task_ptr, &grpc_status); + server::grpc::BaseTaskPtr base_task_ptr = DummyTask::Create(dummy); + server::grpc::GrpcRequestScheduler::GetInstance().ExecTask(base_task_ptr, &grpc_status); - GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); + server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); task_ptr = nullptr; - GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); + server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); - GrpcRequestScheduler::GetInstance().Stop(); -} - -} -} -} + server::grpc::GrpcRequestScheduler::GetInstance().Stop(); } diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index 00be2f3f13..b582c9a17a 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include "utils/CommonUtil.h" #include "utils/Error.h" @@ -51,6 +52,10 @@ TEST(UtilTest, EXCEPTION_TEST) { ASSERT_EQ(msg, err_msg); } +TEST(UtilTest, SIGNAL_TEST) { + server::SignalUtil::PrintStacktrace(); +} + TEST(UtilTest, COMMON_TEST) { unsigned long total_mem = 0, free_mem = 0; server::CommonUtil::GetSystemMemInfo(total_mem, free_mem); @@ -167,13 +172,13 @@ TEST(UtilTest, BLOCKINGQUEUE_TEST) { } TEST(UtilTest, LOG_TEST) { - int32_t res = server::InitLog(LOG_FILE_PATH); - ASSERT_EQ(res, 0); + auto status = server::InitLog(LOG_FILE_PATH); + ASSERT_TRUE(status.ok()); EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::NewLineForContainer)); EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::LogDetailedCrashReason)); - std::string fname = server::GetFileName(LOG_FILE_PATH); + std::string fname = server::CommonUtil::GetFileName(LOG_FILE_PATH); ASSERT_EQ(fname, "log_config.conf"); } From 554c1808d594d6635eaab111c2f306ee9c41acad Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Wed, 18 Sep 2019 15:17:04 +0800 Subject: [PATCH 026/217] Update README.md Former-commit-id: 5843202bb1d4083ab1ed802f1aa3cd730a63d1d9 --- cpp/README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index aa62a256d9..18c1131f10 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -10,7 +10,7 @@ Milvus was developed by researchers and engineers in ZILLIZ, a tech startup that Milvus provides stable Python and C++ APIs, as well as RESTful API. -Keep up-to-date with newest releases and latest updates by reading [Releases](milvus-io/doc/master/releases). +Keep up-to-date with newest releases and latest updates by reading [Releases](https://www.milvus-io/docs/master/releases). - GPU-accelerated search engine @@ -24,13 +24,16 @@ Keep up-to-date with newest releases and latest updates by reading [Releases](mi The data is stored and computed on a distributed architecture. This lets you scale data sizes up and down without redesigning the system. +## Architecture +![Milvus_arch](https://www.milvus-io/docs/master/assets/milvus_arch.png) + ## Get started ### Install and start Milvus server #### Use Docker -Use Docker to install Milvus is a breeze. See the [Milvus install guide](doc/master/userguide/install_milvus.md) for details. +Use Docker to install Milvus is a breeze. See the [Milvus install guide](https://www.milvus-io/docs/master/userguide/install_milvus.md) for details. #### Use source code @@ -143,7 +146,7 @@ To stop Milvus server, run: $ ./stop_server.sh ``` -To edit Milvus settings in `conf/server_config.yaml` and `conf/log_config.conf`, please read [Milvus Configuration](milvus-io/milvus/doc/master/reference/milvus_config.md). +To edit Milvus settings in `conf/server_config.yaml` and `conf/log_config.conf`, please read [Milvus Configuration](https://www.milvus-io/docs/master/reference/milvus_config.md). ### Try your first Milvus program @@ -187,7 +190,7 @@ For public discussion of Milvus, please join our [discussion group]. ## Milvus Roadmap -Please read our [roadmap](milvus-io/milvus/doc/master/roadmap.md) to learn about upcoming features. +Please read our [roadmap](milvus-io/milvus/docs/master/roadmap.md) to learn about upcoming features. ## Resources @@ -199,7 +202,7 @@ Please read our [roadmap](milvus-io/milvus/doc/master/roadmap.md) to learn about [Milvus CSDN](https://mp.csdn.net/mdeditor/100041006#) -[Milvus roadmap](milvus-io/milvus/doc/master/roadmap.md) +[Milvus roadmap](https://www.milvus-io/docs/master/roadmap.md) [Milvus white paper] From e852816176407de1c7335fbf7ff2b2b4a2da2513 Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Wed, 18 Sep 2019 15:34:47 +0800 Subject: [PATCH 027/217] fix memory leak Former-commit-id: 397a13f633b1b120c516db97ed46e5c8836db1ce --- cpp/src/metrics/SystemInfo.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/src/metrics/SystemInfo.cpp b/cpp/src/metrics/SystemInfo.cpp index 61ae42c438..5db2448ee9 100644 --- a/cpp/src/metrics/SystemInfo.cpp +++ b/cpp/src/metrics/SystemInfo.cpp @@ -245,6 +245,7 @@ SystemInfo::CPUTemperature(){ float temp; fscanf(file, "%f", &temp); result.push_back(temp / 1000); + fclose(file); } } From 8464b8a206220abe0739805ad01975dae7f8b2e1 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 18 Sep 2019 15:49:47 +0800 Subject: [PATCH 028/217] refactor cmake Former-commit-id: 587cc22f501fd12cbdaf2d9950fce3f2483c7020 --- cpp/CHANGELOG.md | 1 + cpp/CMakeLists.txt | 31 +- cpp/cmake/DefineOptions.cmake | 22 +- cpp/cmake/ThirdPartyPackages.cmake | 801 +----------------- cpp/src/CMakeLists.txt | 126 ++- .../sdk/examples/grpcsimple/CMakeLists.txt | 5 - cpp/src/storage/IStorage.h | 50 -- cpp/src/storage/s3/S3ClientWrapper.cpp | 177 ---- cpp/src/storage/s3/S3ClientWrapper.h | 63 -- cpp/thirdparty/versions.txt | 8 - cpp/unittest/CMakeLists.txt | 63 +- cpp/unittest/db/CMakeLists.txt | 74 +- cpp/unittest/knowhere/CMakeLists.txt | 8 +- cpp/unittest/metrics/CMakeLists.txt | 77 +- cpp/unittest/scheduler/CMakeLists.txt | 68 +- cpp/unittest/server/CMakeLists.txt | 117 +-- cpp/unittest/storage/CMakeLists.txt | 55 -- cpp/unittest/storage/S3ClientTest.cpp | 78 -- 18 files changed, 228 insertions(+), 1596 deletions(-) delete mode 100644 cpp/src/storage/IStorage.h delete mode 100644 cpp/src/storage/s3/S3ClientWrapper.cpp delete mode 100644 cpp/src/storage/s3/S3ClientWrapper.h delete mode 100644 cpp/unittest/storage/CMakeLists.txt delete mode 100644 cpp/unittest/storage/S3ClientTest.cpp diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index c5d27e3461..6d66555590 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -13,6 +13,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-556 - Add Job Definition in Scheduler - MS-558 - Refine status code - MS-562 - Add JobMgr and TaskCreator in Scheduler +- MS-566 - Refactor cmake ## New Feature diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 69aaa9f07d..91c7b5286b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -51,13 +51,6 @@ message(STATUS "Build type = ${BUILD_TYPE}") project(milvus VERSION "${MILVUS_VERSION}") project(milvus_engine LANGUAGES CUDA CXX) -# Ensure that a default make is set -if("${MAKE}" STREQUAL "") - if(NOT MSVC) - find_program(MAKE make) - endif() -endif() - set(MILVUS_VERSION_MAJOR "${milvus_VERSION_MAJOR}") set(MILVUS_VERSION_MINOR "${milvus_VERSION_MINOR}") set(MILVUS_VERSION_PATCH "${milvus_VERSION_PATCH}") @@ -76,9 +69,6 @@ message(STATUS "Milvus version: " "${MILVUS_VERSION_MAJOR}.${MILVUS_VERSION_MINOR}.${MILVUS_VERSION_PATCH} " "(full: '${MILVUS_VERSION}')") -set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR}) -set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR}) - find_package(CUDA) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda") @@ -105,8 +95,28 @@ else() set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") endif() +# Ensure that a default make is set +if("${MAKE}" STREQUAL "") + if(NOT MSVC) + find_program(MAKE make) + endif() +endif() + +find_path(MYSQL_INCLUDE_DIR + NAMES "mysql.h" + PATH_SUFFIXES "mysql") +if (${MYSQL_INCLUDE_DIR} STREQUAL "MYSQL_INCLUDE_DIR-NOTFOUND") + message(FATAL_ERROR "Could not found MySQL include directory") +else() + include_directories(${MYSQL_INCLUDE_DIR}) +endif() + set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") +set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR}) +set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR}) +set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) + include(ExternalProject) include(DefineOptions) include(BuildUtils) @@ -114,7 +124,6 @@ include(ThirdPartyPackages) config_summary() -set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) add_subdirectory(src) if (BUILD_UNIT_TEST STREQUAL "ON") diff --git a/cpp/cmake/DefineOptions.cmake b/cpp/cmake/DefineOptions.cmake index 1484a9d25f..2b9a68e88e 100644 --- a/cpp/cmake/DefineOptions.cmake +++ b/cpp/cmake/DefineOptions.cmake @@ -55,8 +55,6 @@ define_option_string(MILVUS_DEPENDENCY_SOURCE define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD "Show output from ExternalProjects rather than just logging to files" ON) -define_option(MILVUS_WITH_ARROW "Build with ARROW" OFF) - define_option(MILVUS_BOOST_VENDORED "Use vendored Boost instead of existing Boost. \ Note that this requires linking Boost statically" ON) @@ -66,22 +64,10 @@ define_option(MILVUS_WITH_BZ2 "Build with BZ2 compression" ON) define_option(MILVUS_WITH_EASYLOGGINGPP "Build with Easylogging++ library" ON) -define_option(MILVUS_WITH_FAISS "Build with FAISS library" OFF) - -define_option(MILVUS_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" OFF) - -define_option(MILVUS_WITH_LAPACK "Build with LAPACK library" OFF) - define_option(MILVUS_WITH_LZ4 "Build with lz4 compression" ON) -define_option(MILVUS_WITH_JSONCONS "Build with JSONCONS" OFF) - -define_option(MILVUS_WITH_OPENBLAS "Build with OpenBLAS library" OFF) - define_option(MILVUS_WITH_PROMETHEUS "Build with PROMETHEUS library" ON) -define_option(MILVUS_WITH_ROCKSDB "Build with RocksDB library" OFF) - define_option(MILVUS_WITH_SNAPPY "Build with Snappy compression" ON) define_option(MILVUS_WITH_SQLITE "Build with SQLite library" ON) @@ -94,10 +80,6 @@ define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON) define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON) -define_option(MILVUS_WITH_KNOWHERE "Build with Knowhere" OFF) - -#define_option(MILVUS_ENABLE_PROFILING "Build with profiling" ON) - if(CMAKE_VERSION VERSION_LESS 3.7) set(MILVUS_WITH_ZSTD_DEFAULT OFF) else() @@ -106,13 +88,13 @@ else() endif() define_option(MILVUS_WITH_ZSTD "Build with zstd compression" ${MILVUS_WITH_ZSTD_DEFAULT}) -define_option(MILVUS_WITH_AWS "Build with AWS SDK" ON) - if (MILVUS_ENABLE_PROFILING STREQUAL "ON") define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON) define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON) endif() +define_option(MILVUS_WITH_GRPC "Build with GRPC" ON) + #---------------------------------------------------------------------- if(MSVC) set_option_category("MSVC") diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index fa61da180a..f280c48293 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -16,27 +16,18 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES - ARROW BOOST BZip2 - FAISS GTest - Knowhere - JSONCONS - LAPACK Lz4 MySQLPP - OpenBLAS Prometheus - RocksDB Snappy SQLite SQLite_ORM - Thrift yaml-cpp ZLIB ZSTD - AWS libunwind gperftools GRPC) @@ -51,30 +42,16 @@ foreach(DEPENDENCY ${MILVUS_THIRDPARTY_DEPENDENCIES}) endforeach() macro(build_dependency DEPENDENCY_NAME) - if("${DEPENDENCY_NAME}" STREQUAL "ARROW") - build_arrow() - elseif("${DEPENDENCY_NAME}" STREQUAL "BZip2") + if("${DEPENDENCY_NAME}" STREQUAL "BZip2") build_bzip2() - elseif("${DEPENDENCY_NAME}" STREQUAL "FAISS") - build_faiss() elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest") build_gtest() - elseif("${DEPENDENCY_NAME}" STREQUAL "LAPACK") - build_lapack() - elseif("${DEPENDENCY_NAME}" STREQUAL "Knowhere") - build_knowhere() elseif("${DEPENDENCY_NAME}" STREQUAL "Lz4") build_lz4() elseif ("${DEPENDENCY_NAME}" STREQUAL "MySQLPP") build_mysqlpp() - elseif ("${DEPENDENCY_NAME}" STREQUAL "JSONCONS") - build_jsoncons() - elseif ("${DEPENDENCY_NAME}" STREQUAL "OpenBLAS") - build_openblas() elseif ("${DEPENDENCY_NAME}" STREQUAL "Prometheus") build_prometheus() - elseif ("${DEPENDENCY_NAME}" STREQUAL "RocksDB") - build_rocksdb() elseif ("${DEPENDENCY_NAME}" STREQUAL "Snappy") build_snappy() elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite") @@ -87,8 +64,6 @@ macro(build_dependency DEPENDENCY_NAME) build_zlib() elseif("${DEPENDENCY_NAME}" STREQUAL "ZSTD") build_zstd() - elseif("${DEPENDENCY_NAME}" STREQUAL "AWS") - build_aws() elseif("${DEPENDENCY_NAME}" STREQUAL "libunwind") build_libunwind() elseif("${DEPENDENCY_NAME}" STREQUAL "gperftools") @@ -280,8 +255,7 @@ if(DEFINED ENV{MILVUS_BOOST_URL}) else() string(REPLACE "." "_" BOOST_VERSION_UNDERSCORES ${BOOST_VERSION}) set(BOOST_SOURCE_URL - "http://192.168.1.201/artifactory/generic-local/tools/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz") -# "https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz" + "https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz") endif() set(BOOST_MD5 "fea771fe8176828fabf9c09242ee8c26") @@ -292,22 +266,6 @@ else() endif() set(BZIP2_MD5 "00b516f4704d4a7cb50a1d97e6e8e15b") -if(DEFINED ENV{MILVUS_FAISS_URL}) - set(FAISS_SOURCE_URL "$ENV{MILVUS_FAISS_URL}") -else() - set(FAISS_SOURCE_URL "http://192.168.1.105:6060/jinhai/faiss/-/archive/${FAISS_VERSION}/faiss-${FAISS_VERSION}.tar.gz") - # set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz") -endif() - -# set(FAISS_MD5 "a589663865a8558205533c8ac414278c") -set(FAISS_MD5 "31167ecbd1903fec600dc4ac00b9be9e") - -if(DEFINED ENV{MILVUS_KNOWHERE_URL}) - set(KNOWHERE_SOURCE_URL "$ENV{MILVUS_KNOWHERE_URL}") -else() - set(KNOWHERE_SOURCE_URL "${CMAKE_SOURCE_DIR}/thirdparty/knowhere") -endif() - if (DEFINED ENV{MILVUS_GTEST_URL}) set(GTEST_SOURCE_URL "$ENV{MILVUS_GTEST_URL}") else () @@ -316,20 +274,6 @@ else () endif() set(GTEST_MD5 "2e6fbeb6a91310a16efe181886c59596") -if (DEFINED ENV{MILVUS_JSONCONS_URL}) - set(JSONCONS_SOURCE_URL "$ENV{MILVUS_JSONCONS_URL}") -else () - set(JSONCONS_SOURCE_URL - "https://github.com/danielaparker/jsoncons/archive/v${JSONCONS_VERSION}.tar.gz") -endif() - -if(DEFINED ENV{MILVUS_LAPACK_URL}) - set(LAPACK_SOURCE_URL "$ENV{MILVUS_LAPACK_URL}") -else() - set(LAPACK_SOURCE_URL "https://github.com/Reference-LAPACK/lapack/archive/${LAPACK_VERSION}.tar.gz") -endif() -set(LAPACK_MD5 "96591affdbf58c450d45c1daa540dbd2") - if(DEFINED ENV{MILVUS_LZ4_URL}) set(LZ4_SOURCE_URL "$ENV{MILVUS_LZ4_URL}") else() @@ -344,14 +288,6 @@ else() endif() set(MYSQLPP_MD5 "cda38b5ecc0117de91f7c42292dd1e79") -if (DEFINED ENV{MILVUS_OPENBLAS_URL}) - set(OPENBLAS_SOURCE_URL "$ENV{MILVUS_OPENBLAS_URL}") -else () - set(OPENBLAS_SOURCE_URL - "https://github.com/xianyi/OpenBLAS/archive/${OPENBLAS_VERSION}.tar.gz") -endif() -set(OPENBLAS_MD5 "8a110a25b819a4b94e8a9580702b6495") - if (DEFINED ENV{MILVUS_PROMETHEUS_URL}) set(PROMETHEUS_SOURCE_URL "$ENV{PROMETHEUS_OPENBLAS_URL}") else () @@ -359,15 +295,6 @@ else () https://github.com/jupp0r/prometheus-cpp.git) endif() - -if (DEFINED ENV{MILVUS_ROCKSDB_URL}) - set(ROCKSDB_SOURCE_URL "$ENV{MILVUS_ROCKSDB_URL}") -else () - set(ROCKSDB_SOURCE_URL - "https://github.com/facebook/rocksdb/archive/${ROCKSDB_VERSION}.tar.gz") -endif() -set(ROCKSDB_MD5 "a8f2f594182e97a08629bcc66dfd3fa0") - if(DEFINED ENV{MILVUS_SNAPPY_URL}) set(SNAPPY_SOURCE_URL "$ENV{MILVUS_SNAPPY_URL}") else() @@ -413,13 +340,6 @@ else() endif() set(ZSTD_MD5 "340c837db48354f8d5eafe74c6077120") -if(DEFINED ENV{MILVUS_AWS_URL}) - set(AWS_SOURCE_URL "$ENV{MILVUS_AWS_URL}") -else() - set(AWS_SOURCE_URL "https://github.com/aws/aws-sdk-cpp/archive/${AWS_VERSION}.tar.gz") -endif() -set(AWS_MD5 "9217f5bc8bf23dea04f4466521c85fd9") - if(DEFINED ENV{MILVUS_LIBUNWIND_URL}) set(LIBUNWIND_SOURCE_URL "$ENV{MILVUS_LIBUNWIND_URL}") else() @@ -445,120 +365,6 @@ endif() set(GRPC_MD5 "7ec59ad54c85a12dcbbfede09bf413a9") -# ---------------------------------------------------------------------- -# ARROW - -macro(build_arrow) - message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source") - set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp") - set(ARROW_STATIC_LIB_NAME arrow) - - set(ARROW_STATIC_LIB - "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include") - set(ARROW_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - -DARROW_BUILD_STATIC=ON - -DARROW_BUILD_SHARED=OFF - -DARROW_PARQUET=ON - -DARROW_USE_GLOG=OFF - -DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX} - "-DCMAKE_LIBRARY_PATH=${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs" - -DCMAKE_BUILD_TYPE=Release) - - if(USE_JFROG_CACHE STREQUAL "ON") - execute_process(COMMAND sh -c "git ls-remote --heads --tags ${ARROW_SOURCE_URL} ${ARROW_VERSION} | cut -f 1" OUTPUT_VARIABLE ARROW_LAST_COMMIT_ID) - if(${ARROW_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+") - string(MD5 ARROW_COMBINE_MD5 "${ARROW_LAST_COMMIT_ID}") - set(ARROW_CACHE_PACKAGE_NAME "arrow_${ARROW_COMBINE_MD5}.tar.gz") - set(ARROW_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ARROW_CACHE_PACKAGE_NAME}") - set(ARROW_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ARROW_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${ARROW_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote cache file ${ARROW_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(arrow_ep - GIT_REPOSITORY - ${ARROW_SOURCE_URL} - GIT_TAG - ${ARROW_VERSION} - GIT_SHALLOW - TRUE - SOURCE_SUBDIR - cpp - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${ARROW_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - INSTALL_COMMAND - ${MAKE} install - BUILD_BYPRODUCTS - "${ARROW_STATIC_LIB}" - ) - - ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL}) - else() - file(DOWNLOAD ${ARROW_CACHE_URL} ${ARROW_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${ARROW_CACHE_URL} TO ${ARROW_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() - message(FATAL_ERROR "The last commit ID of \"${ARROW_SOURCE_URL}\" repository don't match!") - endif() - else() - externalproject_add(arrow_ep - GIT_REPOSITORY - ${ARROW_SOURCE_URL} - GIT_TAG - ${ARROW_VERSION} - GIT_SHALLOW - TRUE - SOURCE_SUBDIR - cpp - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${ARROW_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - INSTALL_COMMAND - ${MAKE} install - BUILD_BYPRODUCTS - "${ARROW_STATIC_LIB}" - ) - endif() - - file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}") - add_library(arrow STATIC IMPORTED) - set_target_properties(arrow - PROPERTIES IMPORTED_LOCATION "${ARROW_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") - - add_dependencies(arrow arrow_ep) - - set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep") - - add_custom_command(TARGET arrow_ep POST_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory ${ARROW_PREFIX}/lib/ - COMMAND ${CMAKE_COMMAND} -E copy ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a ${ARROW_PREFIX}/lib/ - DEPENDS ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a) - -endmacro() - -if(MILVUS_WITH_ARROW) - - resolve_dependency(ARROW) - - link_directories(SYSTEM ${ARROW_PREFIX}/lib/) - include_directories(SYSTEM ${ARROW_INCLUDE_DIR}) -endif() - # ---------------------------------------------------------------------- # Add Boost dependencies (code adapted from Apache Kudu (incubating)) @@ -755,370 +561,6 @@ if(MILVUS_WITH_BZ2) include_directories(SYSTEM "${BZIP2_INCLUDE_DIR}") endif() -# ---------------------------------------------------------------------- -# Knowhere - -macro(build_knowhere) - message(STATUS "Building knowhere from source") - set(KNOWHERE_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/knowhere_ep-prefix/src/knowhere_ep") - set(KNOWHERE_INCLUDE_DIR "${KNOWHERE_PREFIX}/include") - set(KNOWHERE_STATIC_LIB - "${KNOWHERE_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}knowhere${CMAKE_STATIC_LIBRARY_SUFFIX}") - - set(KNOWHERE_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${KNOWHERE_PREFIX}" - -DCMAKE_INSTALL_LIBDIR=lib - "-DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}" - "-DCUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}" - -DCMAKE_BUILD_TYPE=Release) - - externalproject_add(knowhere_ep - URL - ${KNOWHERE_SOURCE_URL} - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${KNOWHERE_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - ${KNOWHERE_STATIC_LIB}) - - file(MAKE_DIRECTORY "${KNOWHERE_INCLUDE_DIR}") - add_library(knowhere STATIC IMPORTED) - set_target_properties( - knowhere - PROPERTIES IMPORTED_LOCATION "${KNOWHERE_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${KNOWHERE_INCLUDE_DIR}") - - add_dependencies(knowhere knowhere_ep) -endmacro() - -if(MILVUS_WITH_KNOWHERE) - resolve_dependency(Knowhere) - - get_target_property(KNOWHERE_INCLUDE_DIR knowhere INTERFACE_INCLUDE_DIRECTORIES) - link_directories(SYSTEM "${KNOWHERE_PREFIX}/lib") - include_directories(SYSTEM "${KNOWHERE_INCLUDE_DIR}") - include_directories(SYSTEM "${KNOWHERE_INCLUDE_DIR}/SPTAG/AnnService") -endif() - -# ---------------------------------------------------------------------- -# OpenBLAS - -macro(build_openblas) - message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source") - set(OPENBLAS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep") - set(OPENBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include") - set(OPENBLAS_STATIC_LIB - "${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}") - - if(USE_JFROG_CACHE STREQUAL "ON") - set(OPENBLAS_CACHE_PACKAGE_NAME "openblas_${OPENBLAS_MD5}.tar.gz") - set(OPENBLAS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${OPENBLAS_CACHE_PACKAGE_NAME}") - set(OPENBLAS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${OPENBLAS_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${OPENBLAS_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${OPENBLAS_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(openblas_ep - URL - ${OPENBLAS_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "" - BUILD_IN_SOURCE - 1 - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - INSTALL_COMMAND - ${MAKE} - PREFIX=${OPENBLAS_PREFIX} - install - BUILD_BYPRODUCTS - ${OPENBLAS_STATIC_LIB}) - - ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL}) - else() - file(DOWNLOAD ${OPENBLAS_CACHE_URL} ${OPENBLAS_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${OPENBLAS_CACHE_URL} TO ${OPENBLAS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() - externalproject_add(openblas_ep - URL - ${OPENBLAS_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "" - BUILD_IN_SOURCE - 1 - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - INSTALL_COMMAND - ${MAKE} - PREFIX=${OPENBLAS_PREFIX} - install - BUILD_BYPRODUCTS - ${OPENBLAS_STATIC_LIB}) - endif() - - file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}") - add_library(openblas STATIC IMPORTED) - set_target_properties( - openblas - PROPERTIES IMPORTED_LOCATION "${OPENBLAS_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${OPENBLAS_INCLUDE_DIR}") - - add_dependencies(openblas openblas_ep) -endmacro() - -# ---------------------------------------------------------------------- -# LAPACK - -macro(build_lapack) - message(STATUS "Building LAPACK-${LAPACK_VERSION} from source") - set(LAPACK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix/src/lapack_ep") - set(LAPACK_INCLUDE_DIR "${LAPACK_PREFIX}/include") - set(LAPACK_STATIC_LIB - "${LAPACK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lapack${CMAKE_STATIC_LIBRARY_SUFFIX}") - - set(LAPACK_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${LAPACK_PREFIX}" - -DCMAKE_INSTALL_LIBDIR=lib) - - if(USE_JFROG_CACHE STREQUAL "ON") - set(LAPACK_CACHE_PACKAGE_NAME "lapack_${LAPACK_MD5}.tar.gz") - set(LAPACK_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${LAPACK_CACHE_PACKAGE_NAME}") - set(LAPACK_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${LAPACK_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${LAPACK_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${LAPACK_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(lapack_ep - URL - ${LAPACK_SOURCE_URL} - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${LAPACK_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - ${LAPACK_STATIC_LIB}) - - ExternalProject_Create_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LAPACK_CACHE_URL}) - else() - file(DOWNLOAD ${LAPACK_CACHE_URL} ${LAPACK_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${LAPACK_CACHE_URL} TO ${LAPACK_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() - externalproject_add(lapack_ep - URL - ${LAPACK_SOURCE_URL} - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${LAPACK_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - ${LAPACK_STATIC_LIB}) - endif() - - file(MAKE_DIRECTORY "${LAPACK_INCLUDE_DIR}") - add_library(lapack STATIC IMPORTED) - set_target_properties( - lapack - PROPERTIES IMPORTED_LOCATION "${LAPACK_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${LAPACK_INCLUDE_DIR}") - - add_dependencies(lapack lapack_ep) -endmacro() - -# ---------------------------------------------------------------------- -# FAISS - -if(NOT DEFINED BUILD_FAISS_WITH_MKL) - set(BUILD_FAISS_WITH_MKL OFF) -endif() - -if(EXISTS "/proc/cpuinfo") - FILE(READ /proc/cpuinfo PROC_CPUINFO) - - SET(VENDOR_ID_RX "vendor_id[ \t]*:[ \t]*([a-zA-Z]+)\n") - STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}") - STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}") - - if(NOT ${VENDOR_ID} STREQUAL "GenuineIntel") - set(BUILD_FAISS_WITH_MKL OFF) - endif() -endif() - -macro(build_faiss) - message(STATUS "Building FAISS-${FAISS_VERSION} from source") - set(FAISS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep") - set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include") - set(FAISS_STATIC_LIB - "${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}") - - set(FAISS_CONFIGURE_ARGS - "--prefix=${FAISS_PREFIX}" - "CFLAGS=${EP_C_FLAGS}" - "CXXFLAGS=${EP_CXX_FLAGS}" - --without-python) - - set(FAISS_CFLAGS ${EP_C_FLAGS}) - set(FAISS_CXXFLAGS ${EP_CXX_FLAGS}) - - if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - message(STATUS "Build Faiss with MKL") - if(NOT DEFINED MKL_LIB_PATH) - set(MKL_LIB_PATH "/opt/intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64") - message(STATUS "MKL_LIB_PATH = ${MKL_LIB_PATH}") - endif() - - set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} - "CPPFLAGS=-DFINTEGER=long -DMKL_ILP64 -m64 -I${MKL_LIB_PATH}/../../include" - "LDFLAGS=-L${MKL_LIB_PATH}" - "LIBS=-Wl,--start-group ${MKL_LIB_PATH}/libmkl_intel_ilp64.a ${MKL_LIB_PATH}/libmkl_gnu_thread.a ${MKL_LIB_PATH}/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl") - - else() - message(STATUS "Build Faiss with OpenBlas/LAPACK") - set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} - "LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib") - endif() - - if(${MILVUS_WITH_FAISS_GPU_VERSION} STREQUAL "ON") - set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} - "--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}" - "--with-cuda-arch=-gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_75,code=sm_75" - ) - else() - set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} --without-cuda) - endif() - - if(USE_JFROG_CACHE STREQUAL "ON") - string(MD5 FAISS_COMBINE_MD5 "${FAISS_MD5}${LAPACK_MD5}${OPENBLAS_MD5}") - set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMBINE_MD5}.tar.gz") - set(FAISS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${FAISS_CACHE_PACKAGE_NAME}") - set(FAISS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${FAISS_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${FAISS_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${FAISS_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(faiss_ep - URL - ${FAISS_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "./configure" - ${FAISS_CONFIGURE_ARGS} - BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} VERBOSE=1 - BUILD_IN_SOURCE - 1 - INSTALL_COMMAND - ${MAKE} install - BUILD_BYPRODUCTS - ${FAISS_STATIC_LIB}) - - if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF") - ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) - endif() - - ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL}) - else() - file(DOWNLOAD ${FAISS_CACHE_URL} ${FAISS_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${FAISS_CACHE_URL} TO ${FAISS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() - externalproject_add(faiss_ep - URL - ${FAISS_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "./configure" - ${FAISS_CONFIGURE_ARGS} - BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} VERBOSE=1 - BUILD_IN_SOURCE - 1 - INSTALL_COMMAND - ${MAKE} install - BUILD_BYPRODUCTS - ${FAISS_STATIC_LIB}) - - if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF") - ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) - endif() - endif() - - file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}") - add_library(faiss SHARED IMPORTED) - - if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - set(MKL_LIBS ${MKL_LIB_PATH}/libmkl_intel_ilp64.a - ${MKL_LIB_PATH}/libmkl_gnu_thread.a - ${MKL_LIB_PATH}/libmkl_core.a) - - set_target_properties( - faiss - PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "${MKL_LIBS}" ) - else() - set_target_properties( - faiss - PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "openblas;lapack" ) - endif() - - add_dependencies(faiss faiss_ep) - - if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF") - add_dependencies(faiss openblas_ep) - add_dependencies(faiss lapack_ep) - endif() - -endmacro() - -if(MILVUS_WITH_FAISS) - - if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF") - resolve_dependency(OpenBLAS) - get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) - include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") - link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib) - - resolve_dependency(LAPACK) - get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) - include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") - link_directories(SYSTEM "${LAPACK_PREFIX}/lib") - endif() - - resolve_dependency(FAISS) - get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES) - include_directories(SYSTEM "${FAISS_INCLUDE_DIR}") - link_directories(SYSTEM ${FAISS_PREFIX}/lib/) -endif() - # ---------------------------------------------------------------------- # Google gtest @@ -1232,30 +674,6 @@ if (MILVUS_BUILD_TESTS) include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) endif() -# ---------------------------------------------------------------------- -# JSONCONS - -macro(build_jsoncons) - message(STATUS "Building JSONCONS-${JSONCONS_VERSION} from source") - - set(JSONCONS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jsoncons_ep-prefix") - set(JSONCONS_TAR_NAME "${JSONCONS_PREFIX}/jsoncons-${JSONCONS_VERSION}.tar.gz") - set(JSONCONS_INCLUDE_DIR "${JSONCONS_PREFIX}/jsoncons-${JSONCONS_VERSION}/include") - if (NOT EXISTS ${JSONCONS_INCLUDE_DIR}) - file(MAKE_DIRECTORY ${JSONCONS_PREFIX}) - file(DOWNLOAD ${JSONCONS_SOURCE_URL} - ${JSONCONS_TAR_NAME}) - execute_process(COMMAND ${CMAKE_COMMAND} -E tar -xf ${JSONCONS_TAR_NAME} - WORKING_DIRECTORY ${JSONCONS_PREFIX}) - - endif () -endmacro() - -if(MILVUS_WITH_JSONCONS) - resolve_dependency(JSONCONS) - include_directories(SYSTEM "${JSONCONS_INCLUDE_DIR}") -endif() - # ---------------------------------------------------------------------- # lz4 @@ -1563,95 +981,6 @@ if(MILVUS_WITH_PROMETHEUS) endif() -# ---------------------------------------------------------------------- -# RocksDB - -macro(build_rocksdb) - message(STATUS "Building RocksDB-${ROCKSDB_VERSION} from source") - set(ROCKSDB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/rocksdb_ep-prefix/src/rocksdb_ep") - set(ROCKSDB_INCLUDE_DIRS "${ROCKSDB_PREFIX}/include") - set(ROCKSDB_STATIC_LIB_NAME rocksdb) - set(ROCKSDB_STATIC_LIB - "${ROCKSDB_PREFIX}/lib/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ROCKSDB_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - - if(USE_JFROG_CACHE STREQUAL "ON") - string(MD5 ROCKSDB_COMBINE_MD5 "${ROCKSDB_LAST_COMMIT_ID}") - set(ROCKSDB_CACHE_PACKAGE_NAME "rocksdb_${ROCKSDB_MD5}.tar.gz") - set(ROCKSDB_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ROCKSDB_CACHE_PACKAGE_NAME}") - set(ROCKSDB_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ROCKSDB_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${ROCKSDB_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${ROCKSDB_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(rocksdb_ep - URL - ${ROCKSDB_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "" - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - static_lib - "prefix=${ROCKSDB_PREFIX}" - BUILD_IN_SOURCE - 1 - INSTALL_COMMAND - ${MAKE} - install-static - "INSTALL_PATH=${ROCKSDB_PREFIX}/lib" - BUILD_BYPRODUCTS - "${ROCKSDB_STATIC_LIB}") - - ExternalProject_Create_Cache(rocksdb_ep ${ROCKSDB_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/rocksdb_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ROCKSDB_CACHE_URL}) - else() - file(DOWNLOAD ${ROCKSDB_CACHE_URL} ${ROCKSDB_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${ROCKSDB_CACHE_URL} TO ${ROCKSDB_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(rocksdb_ep ${ROCKSDB_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() - externalproject_add(rocksdb_ep - URL - ${ROCKSDB_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "" - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - static_lib - "prefix=${ROCKSDB_PREFIX}" - BUILD_IN_SOURCE - 1 - INSTALL_COMMAND - ${MAKE} - install-static - "INSTALL_PATH=${ROCKSDB_PREFIX}/lib" - BUILD_BYPRODUCTS - "${ROCKSDB_STATIC_LIB}") - endif() - - file(MAKE_DIRECTORY "${ROCKSDB_PREFIX}/include") - - add_library(rocksdb STATIC IMPORTED) - set_target_properties(rocksdb - PROPERTIES IMPORTED_LOCATION "${ROCKSDB_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${ROCKSDB_INCLUDE_DIRS}") - add_dependencies(rocksdb rocksdb_ep) -endmacro() - -if(MILVUS_WITH_ROCKSDB) - - resolve_dependency(RocksDB) - - link_directories(SYSTEM ${ROCKSDB_PREFIX}/lib/lib/) - include_directories(SYSTEM ${ROCKSDB_INCLUDE_DIRS}) -endif() - # ---------------------------------------------------------------------- # Snappy @@ -2091,116 +1420,6 @@ if(MILVUS_WITH_ZSTD) include_directories(SYSTEM ${ZSTD_INCLUDE_DIR}) endif() -# ---------------------------------------------------------------------- -# aws -macro(build_aws) - message(STATUS "Building aws-${AWS_VERSION} from source") - set(AWS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/aws_ep-prefix/src/aws_ep") - - set(AWS_CMAKE_ARGS - ${EP_COMMON_TOOLCHAIN} - "-DCMAKE_INSTALL_PREFIX=${AWS_PREFIX}" - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_INSTALL_LIBDIR=lib - -DBUILD_ONLY=s3 - -DBUILD_SHARED_LIBS=off - -DENABLE_TESTING=off - -DENABLE_UNITY_BUILD=on - -DNO_ENCRYPTION=off) - - set(AWS_CPP_SDK_CORE_STATIC_LIB - "${AWS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-core${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(AWS_CPP_SDK_S3_STATIC_LIB - "${AWS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-s3${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(AWS_INCLUDE_DIR "${AWS_PREFIX}/include") - set(AWS_CMAKE_ARGS - ${AWS_CMAKE_ARGS} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_C_FLAGS=${EP_C_FLAGS} - -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}) - - if(USE_JFROG_CACHE STREQUAL "ON") - set(AWS_CACHE_PACKAGE_NAME "aws_${AWS_MD5}.tar.gz") - set(AWS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${AWS_CACHE_PACKAGE_NAME}") - set(AWS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${AWS_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${AWS_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${AWS_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(aws_ep - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${AWS_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - INSTALL_DIR - ${AWS_PREFIX} - URL - ${AWS_SOURCE_URL} - BUILD_BYPRODUCTS - "${AWS_CPP_SDK_S3_STATIC_LIB}" - "${AWS_CPP_SDK_CORE_STATIC_LIB}") - - ExternalProject_Create_Cache(aws_ep ${AWS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/aws_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${AWS_CACHE_URL}) - else() - file(DOWNLOAD ${AWS_CACHE_URL} ${AWS_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${AWS_CACHE_URL} TO ${AWS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(aws_ep ${AWS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() - externalproject_add(aws_ep - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${AWS_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - INSTALL_DIR - ${AWS_PREFIX} - URL - ${AWS_SOURCE_URL} - BUILD_BYPRODUCTS - "${AWS_CPP_SDK_S3_STATIC_LIB}" - "${AWS_CPP_SDK_CORE_STATIC_LIB}") - endif() - - file(MAKE_DIRECTORY "${AWS_INCLUDE_DIR}") - add_library(aws-cpp-sdk-s3 STATIC IMPORTED) - set_target_properties(aws-cpp-sdk-s3 - PROPERTIES - IMPORTED_LOCATION "${AWS_CPP_SDK_S3_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${AWS_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "${AWS_PREFIX}/lib/libaws-c-event-stream.a;${AWS_PREFIX}/lib/libaws-checksums.a;${AWS_PREFIX}/lib/libaws-c-common.a") - - add_library(aws-cpp-sdk-core STATIC IMPORTED) - set_target_properties(aws-cpp-sdk-core - PROPERTIES IMPORTED_LOCATION "${AWS_CPP_SDK_CORE_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${AWS_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "${AWS_PREFIX}/lib/libaws-c-event-stream.a;${AWS_PREFIX}/lib/libaws-checksums.a;${AWS_PREFIX}/lib/libaws-c-common.a") - - add_dependencies(aws-cpp-sdk-s3 aws_ep) - add_dependencies(aws-cpp-sdk-core aws_ep) - -endmacro() - -if(MILVUS_WITH_AWS) - resolve_dependency(AWS) - - link_directories(SYSTEM ${AWS_PREFIX}/lib) - - get_target_property(AWS_CPP_SDK_S3_INCLUDE_DIR aws-cpp-sdk-s3 INTERFACE_INCLUDE_DIRECTORIES) - include_directories(SYSTEM ${AWS_CPP_SDK_S3_INCLUDE_DIR}) - - get_target_property(AWS_CPP_SDK_CORE_INCLUDE_DIR aws-cpp-sdk-core INTERFACE_INCLUDE_DIRECTORIES) - include_directories(SYSTEM ${AWS_CPP_SDK_CORE_INCLUDE_DIR}) - -endif() - # ---------------------------------------------------------------------- # libunwind @@ -2465,12 +1684,14 @@ macro(build_grpc) add_dependencies(grpc_protoc grpc_ep) endmacro() -resolve_dependency(GRPC) +if(MILVUS_WITH_GRPC) + resolve_dependency(GRPC) -get_target_property(GRPC_INCLUDE_DIR grpc INTERFACE_INCLUDE_DIRECTORIES) -include_directories(SYSTEM ${GRPC_INCLUDE_DIR}) -link_directories(SYSTEM ${GRPC_PREFIX}/lib) + get_target_property(GRPC_INCLUDE_DIR grpc INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${GRPC_INCLUDE_DIR}) + link_directories(SYSTEM ${GRPC_PREFIX}/lib) -set(GRPC_THIRD_PARTY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep/third_party) -include_directories(SYSTEM ${GRPC_THIRD_PARTY_DIR}/protobuf/src) -link_directories(SYSTEM ${GRPC_PROTOBUF_LIB_DIR}) + set(GRPC_THIRD_PARTY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep/third_party) + include_directories(SYSTEM ${GRPC_THIRD_PARTY_DIR}/protobuf/src) + link_directories(SYSTEM ${GRPC_PROTOBUF_LIB_DIR}) +endif() diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index c8cba4d9aa..b7e719666b 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -17,41 +17,24 @@ # under the License. #------------------------------------------------------------------------------- +include_directories(${MILVUS_SOURCE_DIR}) +include_directories(${MILVUS_ENGINE_SRC}) add_subdirectory(core) set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) -foreach(dir ${CORE_INCLUDE_DIRS}) +foreach (dir ${CORE_INCLUDE_DIRS}) include_directories(${dir}) -endforeach() +endforeach () aux_source_directory(cache cache_files) + aux_source_directory(config config_files) -aux_source_directory(server server_files) -aux_source_directory(server/grpc_impl grpcserver_files) -aux_source_directory(utils utils_files) + aux_source_directory(db db_main_files) aux_source_directory(db/engine db_engine_files) aux_source_directory(db/insert db_insert_files) aux_source_directory(db/meta db_meta_files) -aux_source_directory(metrics metrics_files) -aux_source_directory(wrapper/knowhere knowhere_files) - -aux_source_directory(scheduler/action scheduler_action_files) -aux_source_directory(scheduler/event scheduler_event_files) -aux_source_directory(scheduler/job scheduler_job_files) -aux_source_directory(scheduler/resource scheduler_resource_files) -aux_source_directory(scheduler/task scheduler_task_files) -aux_source_directory(scheduler scheduler_root_files) -set(scheduler_srcs - ${scheduler_action_files} - ${scheduler_event_files} - ${scheduler_job_files} - ${scheduler_resource_files} - ${scheduler_task_files} - ${scheduler_root_files} - ) - aux_source_directory(db/scheduler scheduler_files) aux_source_directory(db/scheduler/context scheduler_context_files) aux_source_directory(db/scheduler/task scheduler_task_files) @@ -60,24 +43,37 @@ set(db_scheduler_files ${scheduler_context_files} ${scheduler_task_files} ) -set(license_check_files - license/LicenseLibrary.cpp - license/LicenseCheck.cpp - ) - -set(license_generator_files - license/LicenseGenerator.cpp - license/LicenseLibrary.cpp - ) set(grpc_service_files - metrics/SystemInfo.cpp - metrics/SystemInfo.h grpc/gen-milvus/milvus.grpc.pb.cc grpc/gen-milvus/milvus.pb.cc grpc/gen-status/status.grpc.pb.cc grpc/gen-status/status.pb.cc - scheduler/Utils.h) + ) + +aux_source_directory(metrics metrics_files) + +aux_source_directory(scheduler/action scheduler_action_files) +aux_source_directory(scheduler/event scheduler_event_files) +aux_source_directory(scheduler/job scheduler_job_files) +aux_source_directory(scheduler/resource scheduler_resource_files) +aux_source_directory(scheduler/task scheduler_task_files) +aux_source_directory(scheduler scheduler_root_files) +set(scheduler_files + ${scheduler_action_files} + ${scheduler_event_files} + ${scheduler_job_files} + ${scheduler_resource_files} + ${scheduler_task_files} + ${scheduler_root_files} + ) + +aux_source_directory(server server_files) +aux_source_directory(server/grpc_impl grpc_server_files) + +aux_source_directory(utils utils_files) + +aux_source_directory(wrapper/knowhere knowhere_files) set(db_files ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp @@ -88,17 +84,12 @@ set(db_files ${db_meta_files} ${db_scheduler_files} ${metrics_files} - ${knowhere_files} ${utils_files} + ${knowhere_files} ) -set(s3_client_files - storage/s3/S3ClientWrapper.cpp - storage/s3/S3ClientWrapper.h) - include_directories(/usr/include) include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") -include_directories(/usr/include/mysql) include_directories(grpc/gen-status) include_directories(grpc/gen-milvus) @@ -109,17 +100,22 @@ set(client_grpc_lib grpc_protobuf grpc_protoc) +set(prometheus_lib + prometheus-cpp-push + prometheus-cpp-pull + prometheus-cpp-core) + +set(boost_lib + boost_system_static + boost_filesystem_static + boost_serialization_static) + set(third_party_libs - knowhere sqlite ${client_grpc_lib} yaml-cpp - prometheus-cpp-push - prometheus-cpp-pull - prometheus-cpp-core - boost_system_static - boost_filesystem_static - boost_serialization_static + ${prometheus_lib} + ${boost_lib} bzip2 lz4 snappy @@ -131,13 +127,12 @@ set(third_party_libs ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so cudart ) - if (MILVUS_ENABLE_PROFILING STREQUAL "ON") set(third_party_libs ${third_party_libs} gperftools - libunwind) -endif() - + libunwind + ) +endif () link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") set(engine_libs @@ -149,12 +144,11 @@ set(engine_libs if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") set(engine_libs - ${engine_libs} - libquadmath.a - ) + ${engine_libs} + libquadmath.a + ) endif () - cuda_add_library(milvus_engine STATIC ${db_files}) target_link_libraries(milvus_engine ${engine_libs} ${third_party_libs}) @@ -162,9 +156,7 @@ add_library(metrics STATIC ${metrics_files}) set(metrics_lib yaml-cpp - prometheus-cpp-push - prometheus-cpp-pull - prometheus-cpp-core + ${prometheus_lib} ) target_link_libraries(metrics ${metrics_lib}) @@ -176,21 +168,17 @@ set(server_libs metrics ) -set(knowhere_libs - knowhere - ) - add_executable(milvus_server ${config_files} - ${server_files} - ${grpcserver_files} - ${utils_files} - ${grpc_service_files} ${metrics_files} - ${scheduler_srcs} + ${scheduler_files} + ${server_files} + ${grpc_server_files} + ${grpc_service_files} + ${utils_files} ) -target_link_libraries(milvus_server ${server_libs} ${knowhere_libs} ${third_party_libs}) +target_link_libraries(milvus_server ${server_libs} knowhere ${third_party_libs}) install(TARGETS milvus_server DESTINATION bin) diff --git a/cpp/src/sdk/examples/grpcsimple/CMakeLists.txt b/cpp/src/sdk/examples/grpcsimple/CMakeLists.txt index b6c67061be..77542ed2a7 100644 --- a/cpp/src/sdk/examples/grpcsimple/CMakeLists.txt +++ b/cpp/src/sdk/examples/grpcsimple/CMakeLists.txt @@ -20,11 +20,6 @@ aux_source_directory(src src_files) -include_directories(src) -include_directories(../../include) - -link_directories(${CMAKE_BINARY_DIR}) - add_executable(sdk_simple main.cpp ${src_files} diff --git a/cpp/src/storage/IStorage.h b/cpp/src/storage/IStorage.h deleted file mode 100644 index 431dd45caf..0000000000 --- a/cpp/src/storage/IStorage.h +++ /dev/null @@ -1,50 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -#pragma once - -#include "db/Status.h" - -#include - - -namespace zilliz { -namespace milvus { -namespace engine { -namespace storage { - -class IStorage { - public: - virtual Status Create(const std::string &ip_address, - const std::string &port, - const std::string &access_key, - const std::string &secret_key) = 0; - - virtual Status Close() = 0; - - virtual Status CreateBucket(std::string& bucket_name) = 0; - virtual Status DeleteBucket(std::string& bucket_name) = 0; - virtual Status UploadFile(std::string &bucket_name, std::string &object_key, std::string &path_key) = 0; - virtual Status DownloadFile(std::string &bucket_name, std::string &object_key, std::string &path_key) = 0; - virtual Status DeleteFile(std::string &bucket_name, std::string &object_key) = 0; -}; - -} -} -} -} \ No newline at end of file diff --git a/cpp/src/storage/s3/S3ClientWrapper.cpp b/cpp/src/storage/s3/S3ClientWrapper.cpp deleted file mode 100644 index 2dca5b3eca..0000000000 --- a/cpp/src/storage/s3/S3ClientWrapper.cpp +++ /dev/null @@ -1,177 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -//#include "S3ClientWrapper.h" -// -//#include -//#include -//#include -//#include -//#include -// -//#include -//#include -// -// -//namespace zilliz { -//namespace milvus { -//namespace engine { -//namespace storage { -// -//Status -//S3ClientWrapper::Create(const std::string &ip_address, -// const std::string &port, -// const std::string &access_key, -// const std::string &secret_key) { -// Aws::InitAPI(options_); -// Aws::Client::ClientConfiguration cfg; -// -// // TODO: ip_address need to be validated. -// -// cfg.endpointOverride = ip_address + ":" + port; // S3 server ip address and port -// cfg.scheme = Aws::Http::Scheme::HTTP; -// cfg.verifySSL = -// false; //Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key -// client_ = -// new S3Client(Aws::Auth::AWSCredentials(access_key, secret_key), -// cfg, -// Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, -// false); -// if (client_ == nullptr) { -// std::string error = "Can't connect server."; -// return Status::Error(error); -// } else { -// return Status::OK(); -// } -//} -// -// -//Status -//S3ClientWrapper::Close() { -// if (client_ != nullptr) { -// delete client_; -// client_ = nullptr; -// } -// Aws::ShutdownAPI(options_); -// return Status::OK(); -//} -// -//Status -//S3ClientWrapper::CreateBucket(std::string& bucket_name) { -// Aws::S3::Model::CreateBucketRequest request; -// request.SetBucket(bucket_name); -// -// auto outcome = client_->CreateBucket(request); -// -// if (outcome.IsSuccess()) -// { -// return Status::OK(); -// } -// else -// { -// std::cout << "CreateBucket error: " -// << outcome.GetError().GetExceptionName() << std::endl -// << outcome.GetError().GetMessage() << std::endl; -// switch(outcome.GetError().GetErrorType()) { -// case Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS: -// case Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU: -// return Status::AlreadyExist(outcome.GetError().GetMessage()); -// default: -// return Status::Error(outcome.GetError().GetMessage()); -// } -// } -//} -// -//Status -//S3ClientWrapper::DeleteBucket(std::string& bucket_name) { -// Aws::S3::Model::DeleteBucketRequest bucket_request; -// bucket_request.SetBucket(bucket_name); -// -// auto outcome = client_->DeleteBucket(bucket_request); -// -// if (outcome.IsSuccess()) -// { -// return Status::OK(); -// } -// else -// { -// std::cout << "DeleteBucket error: " -// << outcome.GetError().GetExceptionName() << " - " -// << outcome.GetError().GetMessage() << std::endl; -// return Status::Error(outcome.GetError().GetMessage()); -// } -//} -// -//Status -//S3ClientWrapper::UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) { -// -// PutObjectRequest putObjectRequest; -// putObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str()); -// -// auto input_data = Aws::MakeShared("PutObjectInputStream", -// pathkey.c_str(), -// std::ios_base::in | std::ios_base::binary); -// putObjectRequest.SetBody(input_data); -// auto put_object_result = client_->PutObject(putObjectRequest); -// if (put_object_result.IsSuccess()) { -// return Status::OK(); -// } else { -// std::cout << "PutObject error: " << put_object_result.GetError().GetExceptionName() << " " -// << put_object_result.GetError().GetMessage() << std::endl; -// return Status::Error(put_object_result.GetError().GetMessage()); -// } -//} -// -//Status -//S3ClientWrapper::DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) { -// GetObjectRequest object_request; -// object_request.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str()); -// auto get_object_outcome = client_->GetObject(object_request); -// if (get_object_outcome.IsSuccess()) { -// Aws::OFStream local_file(pathkey.c_str(), std::ios::out | std::ios::binary); -// local_file << get_object_outcome.GetResult().GetBody().rdbuf(); -// return Status::OK(); -// } else { -// std::cout << "GetObject error: " << get_object_outcome.GetError().GetExceptionName() << " " -// << get_object_outcome.GetError().GetMessage() << std::endl; -// return Status::Error(get_object_outcome.GetError().GetMessage()); -// } -//} -// -//Status -//S3ClientWrapper::DeleteFile(std::string &bucket_name, std::string &object_key) { -// Aws::S3::Model::DeleteObjectRequest object_request; -// object_request.WithBucket(bucket_name).WithKey(object_key); -// -// auto delete_object_outcome = client_->DeleteObject(object_request); -// -// if (delete_object_outcome.IsSuccess()) { -// return Status::OK(); -// } else { -// std::cout << "DeleteObject error: " << -// delete_object_outcome.GetError().GetExceptionName() << " " << -// delete_object_outcome.GetError().GetMessage() << std::endl; -// -// return Status::Error(delete_object_outcome.GetError().GetMessage()); -// } -//} -// -//} -//} -//} -//} \ No newline at end of file diff --git a/cpp/src/storage/s3/S3ClientWrapper.h b/cpp/src/storage/s3/S3ClientWrapper.h deleted file mode 100644 index 95a940655e..0000000000 --- a/cpp/src/storage/s3/S3ClientWrapper.h +++ /dev/null @@ -1,63 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -//#pragma once -// -//#include "storage/IStorage.h" -// -// -//#include -//#include -//#include -// -// -//using namespace Aws::S3; -//using namespace Aws::S3::Model; -// -//namespace zilliz { -//namespace milvus { -//namespace engine { -//namespace storage { -// -//class S3ClientWrapper : public IStorage { -// public: -// -// S3ClientWrapper() = default; -// ~S3ClientWrapper() = default; -// -// Status Create(const std::string &ip_address, -// const std::string &port, -// const std::string &access_key, -// const std::string &secret_key) override; -// Status Close() override; -// -// Status CreateBucket(std::string& bucket_name) override; -// Status DeleteBucket(std::string& bucket_name) override; -// Status UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override; -// Status DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override; -// Status DeleteFile(std::string &bucket_name, std::string &object_key) override; -// -// private: -// S3Client *client_ = nullptr; -// Aws::SDKOptions options_; -//}; -// -//} -//} -//} -//} \ No newline at end of file diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 3b02016c55..ec270c0670 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -1,24 +1,16 @@ -ARROW_VERSION=zilliz BOOST_VERSION=1.70.0 BZIP2_VERSION=1.0.6 EASYLOGGINGPP_VERSION=v9.96.7 -FAISS_VERSION=branch-0.1.0 -MKL_VERSION=2019.4.243 GTEST_VERSION=1.8.1 -JSONCONS_VERSION=0.126.0 -LAPACK_VERSION=v3.8.0 LZ4_VERSION=v1.9.1 MYSQLPP_VERSION=3.2.4 -OPENBLAS_VERSION=v0.3.6 PROMETHEUS_VERSION=v0.7.0 -ROCKSDB_VERSION=v6.0.2 SNAPPY_VERSION=1.1.7 SQLITE_VERSION=3280000 SQLITE_ORM_VERSION=master YAMLCPP_VERSION=0.6.2 ZLIB_VERSION=v1.2.11 ZSTD_VERSION=v1.4.0 -AWS_VERSION=1.7.125 LIBUNWIND_VERSION=1.3.1 GPERFTOOLS_VERSION=2.7 GRPC_VERSION=master diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 03729071b9..40db37a035 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -17,20 +17,70 @@ # under the License. #------------------------------------------------------------------------------- -link_directories( - "${CMAKE_BINARY_DIR}/lib" -) +include_directories(${MILVUS_SOURCE_DIR}) +include_directories(${MILVUS_ENGINE_SRC}) + +aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files) + +aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) + +aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files) +set(db_scheduler_files + ${scheduler_files} + ${scheduler_context_files} + ${scheduler_task_files} + ) + +set(grpc_service_files + grpc/gen-milvus/milvus.grpc.pb.cc + grpc/gen-milvus/milvus.pb.cc + grpc/gen-status/status.grpc.pb.cc + grpc/gen-status/status.pb.cc + ) + +aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files) + +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/job scheduler_job_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_root_files) +set(scheduler_files + ${scheduler_action_files} + ${scheduler_event_files} + ${scheduler_job_files} + ${scheduler_resource_files} + ${scheduler_task_files} + ${scheduler_root_files} + ) + +aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_files) + +aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files) + +aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) -set(unittest_srcs +set(unittest_files ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp) -set(require_files +set(helper_files ${MILVUS_ENGINE_SRC}/server/ServerConfig.cpp ${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp ${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp + ${MILVUS_ENGINE_SRC}/utils/Status.cpp + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp + ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc ) set(unittest_libs @@ -49,15 +99,16 @@ set(unittest_libs z ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so cudart + cublas ) foreach(dir ${CORE_INCLUDE_DIRS}) include_directories(${dir}) endforeach() -add_subdirectory(server) add_subdirectory(db) add_subdirectory(knowhere) add_subdirectory(metrics) add_subdirectory(scheduler) +add_subdirectory(server) #add_subdirectory(storage) \ No newline at end of file diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 29440dcf1a..3ab33d3cc7 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -17,61 +17,27 @@ # under the License. #------------------------------------------------------------------------------- -aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs) +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files) -aux_source_directory(./ test_srcs) +include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") +link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") -set(util_files - ${MILVUS_ENGINE_SRC}/utils/Status.cpp - ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp - ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc) - -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files) -set(db_scheduler_srcs +set(db_test_files + ${cache_files} + ${config_files} + ${db_main_files} + ${db_engine_files} + ${db_insert_files} + ${db_meta_files} + ${db_scheduler_files} + ${metrics_files} ${scheduler_files} - ${scheduler_context_files} - ${scheduler_task_files} - ) - -include_directories(/usr/local/cuda/include) -link_directories("/usr/local/cuda/lib64") - -include_directories(/usr/include/mysql) - -set(db_test_src - ${config_files} - ${cache_srcs} - ${db_main_files} - ${db_engine_files} - ${db_insert_files} - ${db_meta_files} - ${db_scheduler_srcs} - ${wrapper_src} - ${scheduler_action_srcs} - ${scheduler_event_srcs} - ${scheduler_resource_srcs} - ${scheduler_task_srcs} - ${scheduler_srcs} - ${knowhere_src} - ${util_files} - ${require_files} - ${test_srcs} + ${knowhere_files} + ${helper_files} + ${test_files} ) -cuda_add_executable(db_test ${db_test_src}) +cuda_add_executable(db_test ${db_test_files}) set(db_libs sqlite @@ -81,13 +47,7 @@ set(db_libs mysqlpp ) -set(knowhere_libs - knowhere - cudart - cublas - ) - -target_link_libraries(db_test ${db_libs} ${knowhere_libs} ${unittest_libs}) +target_link_libraries(db_test ${db_libs} knowhere ${unittest_libs}) install(TARGETS db_test DESTINATION unittest) diff --git a/cpp/unittest/knowhere/CMakeLists.txt b/cpp/unittest/knowhere/CMakeLists.txt index 27e2612ae7..5606b3820c 100644 --- a/cpp/unittest/knowhere/CMakeLists.txt +++ b/cpp/unittest/knowhere/CMakeLists.txt @@ -20,15 +20,15 @@ include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") -set(knowhere_src +set(knowhere_files ${MILVUS_ENGINE_SRC}/wrapper/knowhere/data_transfer.cpp ${MILVUS_ENGINE_SRC}/wrapper/knowhere/vec_impl.cpp ${MILVUS_ENGINE_SRC}/wrapper/knowhere/vec_index.cpp) -set(helper +set(helper_files utils.cpp ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc - ${MILVUS_ENGINE_SRC}/utils/easylogging++.h) + ) set(knowhere_libs knowhere @@ -36,7 +36,7 @@ set(knowhere_libs cublas ) -add_executable(knowhere_test knowhere_test.cpp ${knowhere_src} ${helper}) +add_executable(knowhere_test knowhere_test.cpp ${knowhere_files} ${helper_files}) target_link_libraries(knowhere_test ${knowhere_libs} ${unittest_libs}) install(TARGETS knowhere_test DESTINATION unittest) \ No newline at end of file diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index dddba2906a..9df30c4340 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -17,86 +17,35 @@ # under the License. #------------------------------------------------------------------------------- -include_directories(../../src) - - -aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src) -aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/src/metrics metrics_src) -aux_source_directory(./ test_srcs) - -set(util_files - ${MILVUS_ENGINE_SRC}/utils/Status.cpp - ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp - ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc) - -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files) -set(db_scheduler_srcs - ${scheduler_files} - ${scheduler_context_files} - ${scheduler_task_files} - ) +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files) include_directories(/usr/include) -include_directories(../../third_party/build/include) -link_directories(../../third_party/build/lib) -include_directories(/usr/local/cuda/include) -link_directories("/usr/local/cuda/lib64") -include_directories(../../src/metrics) +include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") +link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") -include_directories(/usr/include/mysql) - -set(count_test_src +set(metrics_test_files + ${cache_files} ${config_files} - ${cache_srcs} ${db_main_files} ${db_engine_files} ${db_insert_files} ${db_meta_files} - ${db_scheduler_srcs} - ${wrapper_src} - ${scheduler_action_srcs} - ${scheduler_event_srcs} - ${scheduler_resource_srcs} - ${scheduler_task_srcs} - ${scheduler_srcs} - ${knowhere_src} - ${metrics_src} - ${test_srcs} - ${util_files} + ${db_scheduler_files} + ${metrics_files} + ${scheduler_files} + ${knowhere_files} + ${helper_files} + ${test_files} ) -add_executable(metrics_test ${count_test_src} ${require_files} ) - -set(knowhere_libs - knowhere - ) +add_executable(metrics_test ${metrics_test_files}) target_link_libraries(metrics_test - ${knowhere_libs} - cudart - cublas + knowhere sqlite boost_system_static boost_filesystem_static lz4 - metrics - gtest - pthread - z mysqlpp ${unittest_libs} ) diff --git a/cpp/unittest/scheduler/CMakeLists.txt b/cpp/unittest/scheduler/CMakeLists.txt index 7359420e16..9386596da0 100644 --- a/cpp/unittest/scheduler/CMakeLists.txt +++ b/cpp/unittest/scheduler/CMakeLists.txt @@ -17,62 +17,27 @@ # under the License. #------------------------------------------------------------------------------- -aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files) +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs) -aux_source_directory(./ test_srcs) - -set(util_files - ${MILVUS_ENGINE_SRC}/utils/Status.cpp - ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp - ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc) - -set(db_scheduler_srcs - ${scheduler_files} - ${scheduler_context_files} - ${scheduler_task_files}) - -include_directories(/usr/local/cuda/include) -link_directories("/usr/local/cuda/lib64") - -include_directories(/usr/include/mysql) - -set(scheduler_test_src - ${unittest_srcs} - ${test_srcs} - ${scheduler_action_srcs} - ${scheduler_event_srcs} - ${scheduler_resource_srcs} - ${scheduler_task_srcs} - ${scheduler_srcs} +include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") +link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") +set(scheduler_test_files + ${cache_files} ${config_files} - ${cache_srcs} ${db_main_files} ${db_engine_files} ${db_insert_files} ${db_meta_files} - ${db_scheduler_srcs} - ${wrapper_src} - ${knowhere_src} - ${util_files} - ${require_files} + ${db_scheduler_files} + ${scheduler_files} + ${knowhere_files} + ${helper_files} + ${unittest_files} + ${test_files} ) -cuda_add_executable(scheduler_test ${scheduler_test_src}) +cuda_add_executable(scheduler_test ${scheduler_test_files}) set(scheduler_libs sqlite @@ -82,14 +47,7 @@ set(scheduler_libs mysqlpp ) -set(knowhere_libs - knowhere - cudart - cublas - ) - - -target_link_libraries(scheduler_test ${scheduler_libs} ${knowhere_libs} ${unittest_libs}) +target_link_libraries(scheduler_test ${scheduler_libs} knowhere ${unittest_libs}) install(TARGETS scheduler_test DESTINATION unittest) diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index 8e06859929..395f2e40ec 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -17,75 +17,19 @@ # under the License. #------------------------------------------------------------------------------- -include_directories(${MILVUS_ENGINE_SRC}/) +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files) + include_directories(/usr/include) +include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") +link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") -include_directories(/usr/include/mysql) +include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status) +include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus) -include_directories(/usr/local/cuda/include) -link_directories(/usr/local/cuda/lib64) - -aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs) - -aux_source_directory(${MILVUS_ENGINE_SRC}/server server_src) -aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_src) - -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files) - -aux_source_directory(./ test_srcs) - -set(db_scheduler_srcs - ${scheduler_files} - ${scheduler_context_files} - ${scheduler_task_files} - ) - -set(util_files - ${MILVUS_ENGINE_SRC}/utils/Status.cpp - ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp - ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc) - -set(db_src - ${config_files} - ${cache_srcs} - ${db_main_files} - ${db_engine_files} - ${db_insert_files} - ${db_meta_files} - ${db_scheduler_srcs} - ${wrapper_src} - ${scheduler_action_srcs} - ${scheduler_event_srcs} - ${scheduler_resource_srcs} - ${scheduler_task_srcs} - ${scheduler_srcs} - ${knowhere_src} - ${util_files} - ${require_files} - ${test_srcs} - ) - -set(utils_srcs +set(helper_files ${helper_files} ${MILVUS_ENGINE_SRC}/utils/StringHelpFunctions.cpp - ${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp - ${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp ${MILVUS_ENGINE_SRC}/utils/LogUtil.cpp - ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp - ${MILVUS_ENGINE_SRC}/utils/SignalUtil.cpp - ) + ${MILVUS_ENGINE_SRC}/utils/SignalUtil.cpp) set(grpc_service_files ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.grpc.pb.cc @@ -94,44 +38,49 @@ set(grpc_service_files ${MILVUS_ENGINE_SRC}/grpc/gen-status/status.pb.cc ) -include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status) -include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus) - -cuda_add_executable(server_test - ${db_src} - ${unittest_srcs} - ${grpc_server_src} - ${server_src} - ${utils_srcs} +set(server_test_files + ${cache_files} + ${config_files} + ${db_main_files} + ${db_engine_files} + ${db_insert_files} + ${db_meta_files} + ${db_scheduler_files} + ${scheduler_files} + ${server_files} + ${grpc_server_files} ${grpc_service_files} - ${require_files} + ${knowhere_files} + ${helper_files} + ${unittest_files} + ${test_files} ) +cuda_add_executable(server_test ${server_test_files}) + +set(client_grpc_lib + grpcpp_channelz + grpc++ + grpc + grpc_protobuf + grpc_protoc) + set(require_libs knowhere stdc++ - cudart - cublas sqlite mysqlpp boost_system_static boost_filesystem_static snappy - z bz2 zstd lz4 - pthread - grpcpp_channelz - grpc++ - grpc - grpc_protobuf - grpc_protoc + ${client_grpc_lib} ) target_link_libraries(server_test ${require_libs} - ${cuda_library} ${unittest_libs} ) diff --git a/cpp/unittest/storage/CMakeLists.txt b/cpp/unittest/storage/CMakeLists.txt deleted file mode 100644 index 79819b2deb..0000000000 --- a/cpp/unittest/storage/CMakeLists.txt +++ /dev/null @@ -1,55 +0,0 @@ -#------------------------------------------------------------------------------- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -#------------------------------------------------------------------------------- - -aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 s3_client_src) - -set(util_files - ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp) - -# Make sure that your call to link_directories takes place before your call to the relevant add_executable. -include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") -link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") - -set(s3_client_test_src - ${unittest_srcs} - ${s3_client_src} - ${require_files} - S3ClientTest.cpp - ${MILVUS_ENGINE_SRC}/db/Status.cpp - ) - -add_executable(s3_test - ${s3_client_test_src} - ${config_files} - ${util_files} - ) - -set(s3_client_libs - stdc++ - aws-cpp-sdk-s3 - aws-cpp-sdk-core - boost_filesystem_static - ) -target_link_libraries(s3_test - ${s3_client_libs} - ${unittest_libs} - curl - crypto) - -install(TARGETS s3_test DESTINATION unittest) \ No newline at end of file diff --git a/cpp/unittest/storage/S3ClientTest.cpp b/cpp/unittest/storage/S3ClientTest.cpp deleted file mode 100644 index 540d9ab75e..0000000000 --- a/cpp/unittest/storage/S3ClientTest.cpp +++ /dev/null @@ -1,78 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// -//#include "storage/IStorage.h" -//#include "storage/s3/S3ClientWrapper.h" -//#include -//#include -//#include -// -// -//using namespace zilliz::milvus::engine; -// -//TEST(s3_client_wrapper, CLIENT_WRAPPER_TEST) { -// -// std::shared_ptr storage_ptr = std::make_shared(); -// -// std::string ip_address = "127.0.0.1"; -// std::string port = "9000"; -// std::string access_key = "AKIAIOSFODNN7EXAMPLE"; -// std::string secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; -// -// Status status = storage_ptr->Create(ip_address, port, access_key, secret_key); -// ASSERT_TRUE(status.ok()); -// -// std::string filename = "/tmp/s3_test_file"; -// std::string bucket_name = "bucktname"; -// std::string object_name = "test_file"; -// -// status = storage_ptr->CreateBucket(bucket_name); -// std::cout << status.IsAlreadyExist() << std::endl; -// if (status.IsAlreadyExist()) { -// status = storage_ptr->DeleteBucket(bucket_name); -// status = storage_ptr->CreateBucket(bucket_name); -// } -// -// ASSERT_TRUE(status.ok()); -// -// std::ofstream ofile(filename); -// std::stringstream ss; -// for (int i = 0; i < 1024; ++i) { -// ss << i; -// } -// ofile << ss.str() << std::endl; -// ofile.close(); -// status = storage_ptr->UploadFile(bucket_name, object_name, filename); -// ASSERT_TRUE(status.ok()); -// -// status = storage_ptr->DownloadFile(bucket_name, object_name, filename); -// std::ifstream infile(filename); -// std::string in_buffer; -// infile >> in_buffer; -// ASSERT_STREQ(in_buffer.c_str(), ss.str().c_str()); -// -// status = storage_ptr->DeleteFile(bucket_name, object_name); -// ASSERT_TRUE(status.ok()); -// -// status = storage_ptr->DeleteBucket(bucket_name); -// ASSERT_TRUE(status.ok()); -// -// status = storage_ptr->Close(); -// ASSERT_TRUE(status.ok()); -//} -// From b364c8e3a7ac515248d2a0ff2da637806fb3ac32 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 18 Sep 2019 15:51:17 +0800 Subject: [PATCH 029/217] update Former-commit-id: 034185feb37416994a52ddf14a30daa06c886f14 --- cpp/unittest/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 40db37a035..37dec9b0ac 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -110,5 +110,4 @@ add_subdirectory(db) add_subdirectory(knowhere) add_subdirectory(metrics) add_subdirectory(scheduler) -add_subdirectory(server) -#add_subdirectory(storage) \ No newline at end of file +add_subdirectory(server) \ No newline at end of file From 8e5197a53be5e825044a777a5f6e2135ca2c682a Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 18 Sep 2019 16:41:42 +0800 Subject: [PATCH 030/217] add log for table file miss issue Former-commit-id: 641bc002a5f760e9bd804c99897d0c6c84f07919 --- cpp/src/db/Utils.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index cbed4c6646..26f93563dc 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -148,7 +148,9 @@ Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& tab } std::string msg = "Table file doesn't exist: " + file_path; - ENGINE_LOG_ERROR << msg; + ENGINE_LOG_ERROR << msg << " in path: " << options.path + << " for table: " << table_file.table_id_; + return Status(DB_ERROR, msg); } From ed105628fcce90b3749b77342465f9a68fb94219 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Wed, 18 Sep 2019 17:27:18 +0800 Subject: [PATCH 031/217] MS-565 1. refactor dir structure 2. add license 3. del swig Former-commit-id: 580d35b9fe3ea71ebc7d6096fbf5d905351f34f5 --- cpp/src/core/CMakeLists.txt | 2 +- cpp/src/core/include/knowhere/adapter/arrow.h | 18 ----- cpp/src/core/include/knowhere/adapter/sptag.h | 24 ------- cpp/src/core/include/knowhere/common/config.h | 14 ---- .../include/knowhere/common/device_type.h | 10 --- cpp/src/core/include/knowhere/common/error.h | 40 ----------- cpp/src/core/include/knowhere/common/id.h | 42 ------------ cpp/src/core/include/knowhere/common/schema.h | 21 ------ cpp/src/core/include/knowhere/common/tensor.h | 16 ----- cpp/src/core/include/knowhere/index/index.h | 49 ------------- .../core/include/knowhere/index/index_model.h | 24 ------- .../core/include/knowhere/index/index_type.h | 17 ----- .../index/preprocessor/preprocessor.h | 22 ------ .../knowhere/index/vector_index/definitions.h | 14 ---- .../index/vector_index/kdt_parameters.h | 34 ---------- .../knowhere/index/vector_index/nsg/nsg.i | 15 ---- cpp/src/core/{src => knowhere}/CMakeLists.txt | 4 +- .../knowhere/adapter/arrow.cpp | 19 +++++- .../core/knowhere/knowhere/adapter/arrow.h | 36 ++++++++++ .../knowhere/adapter/faiss_adopt.h | 0 .../knowhere/adapter/sptag.cpp | 20 +++++- .../core/knowhere/knowhere/adapter/sptag.h | 43 ++++++++++++ .../knowhere/adapter/structure.cpp | 2 +- .../knowhere/adapter/structure.h | 2 +- .../knowhere/common/array.h | 20 +++++- .../knowhere/common/binary_set.h | 20 +++++- .../knowhere/common/buffer.h | 19 ++++++ .../core/knowhere/knowhere/common/config.h | 32 +++++++++ .../knowhere/common/dataset.h | 29 ++++++-- .../knowhere/common/exception.cpp | 3 +- .../knowhere/common/exception.h | 0 cpp/src/core/knowhere/knowhere/common/id.h | 60 ++++++++++++++++ .../core/knowhere/knowhere/common/schema.h | 40 +++++++++++ .../core/knowhere/knowhere/common/tensor.h | 35 ++++++++++ .../knowhere/common/timer.cpp | 2 +- .../knowhere/common/timer.h | 0 cpp/src/core/knowhere/knowhere/index/index.h | 68 +++++++++++++++++++ .../knowhere/knowhere/index/index_model.h | 42 ++++++++++++ .../core/knowhere/knowhere/index/index_type.h | 35 ++++++++++ .../knowhere/index/preprocessor/normalize.cpp | 0 .../knowhere/index/preprocessor/normalize.h | 0 .../index/preprocessor/preprocessor.h | 41 +++++++++++ .../knowhere/index/vector_index/cloner.cpp | 8 +-- .../knowhere/index/vector_index/cloner.h | 0 .../index/vector_index/cpu_kdt_rng.cpp | 23 ++++++- .../knowhere/index/vector_index/cpu_kdt_rng.h | 20 +++++- .../knowhere/index/vector_index/definitions.h | 32 +++++++++ .../knowhere/index/vector_index/gpu_ivf.cpp | 4 +- .../knowhere/index/vector_index/gpu_ivf.h | 18 +++++ .../knowhere/index/vector_index/idmap.cpp | 2 +- .../knowhere/index/vector_index/idmap.h | 0 .../knowhere/index/vector_index/ivf.cpp | 5 +- .../knowhere/index/vector_index/ivf.h | 2 +- .../index/vector_index/kdt_parameters.cpp | 20 +++++- .../index/vector_index/kdt_parameters.h | 52 ++++++++++++++ .../index/vector_index/nsg/index_io.h | 2 +- .../index/vector_index/nsg/neighbor.h | 0 .../knowhere/index/vector_index/nsg/nsg.cpp | 2 +- .../knowhere/index/vector_index/nsg/nsg.h | 0 .../index/vector_index/nsg/nsg_io.cpp | 2 +- .../knowhere/index/vector_index/nsg/nsg_io.h | 0 .../knowhere/index/vector_index/nsg/utils.cpp | 0 .../knowhere/index/vector_index/nsg/utils.h | 2 +- .../knowhere/index/vector_index/nsg_index.cpp | 8 +-- .../knowhere/index/vector_index/nsg_index.h | 2 +- .../index/vector_index/vector_index.h | 19 ++++++ cpp/src/core/test/CMakeLists.txt | 46 ++++++------- 67 files changed, 778 insertions(+), 425 deletions(-) delete mode 100644 cpp/src/core/include/knowhere/adapter/arrow.h delete mode 100644 cpp/src/core/include/knowhere/adapter/sptag.h delete mode 100644 cpp/src/core/include/knowhere/common/config.h delete mode 100644 cpp/src/core/include/knowhere/common/device_type.h delete mode 100644 cpp/src/core/include/knowhere/common/error.h delete mode 100644 cpp/src/core/include/knowhere/common/id.h delete mode 100644 cpp/src/core/include/knowhere/common/schema.h delete mode 100644 cpp/src/core/include/knowhere/common/tensor.h delete mode 100644 cpp/src/core/include/knowhere/index/index.h delete mode 100644 cpp/src/core/include/knowhere/index/index_model.h delete mode 100644 cpp/src/core/include/knowhere/index/index_type.h delete mode 100644 cpp/src/core/include/knowhere/index/preprocessor/preprocessor.h delete mode 100644 cpp/src/core/include/knowhere/index/vector_index/definitions.h delete mode 100644 cpp/src/core/include/knowhere/index/vector_index/kdt_parameters.h delete mode 100644 cpp/src/core/include/knowhere/index/vector_index/nsg/nsg.i rename cpp/src/core/{src => knowhere}/CMakeLists.txt (97%) rename cpp/src/core/{src => knowhere}/knowhere/adapter/arrow.cpp (58%) create mode 100644 cpp/src/core/knowhere/knowhere/adapter/arrow.h rename cpp/src/core/{include => knowhere}/knowhere/adapter/faiss_adopt.h (100%) rename cpp/src/core/{src => knowhere}/knowhere/adapter/sptag.cpp (83%) create mode 100644 cpp/src/core/knowhere/knowhere/adapter/sptag.h rename cpp/src/core/{src => knowhere}/knowhere/adapter/structure.cpp (98%) rename cpp/src/core/{include => knowhere}/knowhere/adapter/structure.h (97%) rename cpp/src/core/{include => knowhere}/knowhere/common/array.h (52%) rename cpp/src/core/{include => knowhere}/knowhere/common/binary_set.h (58%) rename cpp/src/core/{include => knowhere}/knowhere/common/buffer.h (56%) create mode 100644 cpp/src/core/knowhere/knowhere/common/config.h rename cpp/src/core/{include => knowhere}/knowhere/common/dataset.h (77%) rename cpp/src/core/{src => knowhere}/knowhere/common/exception.cpp (97%) rename cpp/src/core/{include => knowhere}/knowhere/common/exception.h (100%) create mode 100644 cpp/src/core/knowhere/knowhere/common/id.h create mode 100644 cpp/src/core/knowhere/knowhere/common/schema.h create mode 100644 cpp/src/core/knowhere/knowhere/common/tensor.h rename cpp/src/core/{src => knowhere}/knowhere/common/timer.cpp (98%) rename cpp/src/core/{include => knowhere}/knowhere/common/timer.h (100%) create mode 100644 cpp/src/core/knowhere/knowhere/index/index.h create mode 100644 cpp/src/core/knowhere/knowhere/index/index_model.h create mode 100644 cpp/src/core/knowhere/knowhere/index/index_type.h rename cpp/src/core/{src => knowhere}/knowhere/index/preprocessor/normalize.cpp (100%) rename cpp/src/core/{include => knowhere}/knowhere/index/preprocessor/normalize.h (100%) create mode 100644 cpp/src/core/knowhere/knowhere/index/preprocessor/preprocessor.h rename cpp/src/core/{src => knowhere}/knowhere/index/vector_index/cloner.cpp (91%) rename cpp/src/core/{include => knowhere}/knowhere/index/vector_index/cloner.h (100%) rename cpp/src/core/{src => knowhere}/knowhere/index/vector_index/cpu_kdt_rng.cpp (83%) rename cpp/src/core/{include => knowhere}/knowhere/index/vector_index/cpu_kdt_rng.h (66%) create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/definitions.h rename cpp/src/core/{src => knowhere}/knowhere/index/vector_index/gpu_ivf.cpp (99%) rename cpp/src/core/{include => knowhere}/knowhere/index/vector_index/gpu_ivf.h (87%) rename cpp/src/core/{src => knowhere}/knowhere/index/vector_index/idmap.cpp (99%) rename cpp/src/core/{include => knowhere}/knowhere/index/vector_index/idmap.h (100%) rename cpp/src/core/{src => knowhere}/knowhere/index/vector_index/ivf.cpp (98%) rename cpp/src/core/{include => knowhere}/knowhere/index/vector_index/ivf.h (98%) rename cpp/src/core/{src => knowhere}/knowhere/index/vector_index/kdt_parameters.cpp (54%) create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.h rename cpp/src/core/{src => knowhere}/knowhere/index/vector_index/nsg/index_io.h (95%) rename cpp/src/core/{include => knowhere}/knowhere/index/vector_index/nsg/neighbor.h (100%) rename cpp/src/core/{src => knowhere}/knowhere/index/vector_index/nsg/nsg.cpp (99%) rename cpp/src/core/{include => knowhere}/knowhere/index/vector_index/nsg/nsg.h (100%) rename cpp/src/core/{src => knowhere}/knowhere/index/vector_index/nsg/nsg_io.cpp (97%) rename cpp/src/core/{include => knowhere}/knowhere/index/vector_index/nsg/nsg_io.h (100%) rename cpp/src/core/{src => knowhere}/knowhere/index/vector_index/nsg/utils.cpp (100%) rename cpp/src/core/{src => knowhere}/knowhere/index/vector_index/nsg/utils.h (95%) rename cpp/src/core/{src => knowhere}/knowhere/index/vector_index/nsg_index.cpp (96%) rename cpp/src/core/{include => knowhere}/knowhere/index/vector_index/nsg_index.h (96%) rename cpp/src/core/{include => knowhere}/knowhere/index/vector_index/vector_index.h (52%) diff --git a/cpp/src/core/CMakeLists.txt b/cpp/src/core/CMakeLists.txt index b18af23f1e..bef3f5e2f6 100644 --- a/cpp/src/core/CMakeLists.txt +++ b/cpp/src/core/CMakeLists.txt @@ -84,7 +84,7 @@ include(DefineOptionsCore) include(BuildUtilsCore) include(ThirdPartyPackagesCore) -add_subdirectory(src) +add_subdirectory(knowhere) if (BUILD_COVERAGE STREQUAL "ON") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") diff --git a/cpp/src/core/include/knowhere/adapter/arrow.h b/cpp/src/core/include/knowhere/adapter/arrow.h deleted file mode 100644 index 338ab691df..0000000000 --- a/cpp/src/core/include/knowhere/adapter/arrow.h +++ /dev/null @@ -1,18 +0,0 @@ - -#pragma once - -#include -#include - - -namespace zilliz { -namespace knowhere { - -ArrayPtr -CopyArray(const ArrayPtr &origin); - -SchemaPtr -CopySchema(const SchemaPtr &origin); - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/adapter/sptag.h b/cpp/src/core/include/knowhere/adapter/sptag.h deleted file mode 100644 index 08c4a625ee..0000000000 --- a/cpp/src/core/include/knowhere/adapter/sptag.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include -#include -#include - - -namespace zilliz { -namespace knowhere { - -std::shared_ptr -ConvertToVectorSet(const DatasetPtr &dataset); - -std::shared_ptr -ConvertToMetadataSet(const DatasetPtr &dataset); - -std::vector -ConvertToQueryResult(const DatasetPtr &dataset, const Config &config); - -DatasetPtr -ConvertToDataset(std::vector query_results); - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/common/config.h b/cpp/src/core/include/knowhere/common/config.h deleted file mode 100644 index 3a3fee1219..0000000000 --- a/cpp/src/core/include/knowhere/common/config.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include - - -namespace zilliz { -namespace knowhere { - - -using Config = jsoncons::json; - - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/common/device_type.h b/cpp/src/core/include/knowhere/common/device_type.h deleted file mode 100644 index 09fa362643..0000000000 --- a/cpp/src/core/include/knowhere/common/device_type.h +++ /dev/null @@ -1,10 +0,0 @@ -#pragma once - - -namespace zilliz { -namespace sched { -namespace master { - -} // namespace master -} // namespace sched -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/common/error.h b/cpp/src/core/include/knowhere/common/error.h deleted file mode 100644 index ba7b5cf315..0000000000 --- a/cpp/src/core/include/knowhere/common/error.h +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once - -#include -#include "zlibrary/error/error.h" - - -namespace zilliz { -namespace knowhere { - -using Error = zilliz::lib::ErrorCode; - -constexpr Error STORE_SUCCESS = zilliz::lib::SUCCESS_CODE; - -constexpr Error ERROR_CODE_BASE = 0x36000; -constexpr Error ERROR_CODE_END = 0x37000; - -constexpr Error -ToGlobalErrorCode(const Error error_code) { - return zilliz::lib::ToGlobalErrorCode(error_code, ERROR_CODE_BASE); -} - -class Exception : public zilliz::lib::Exception { - public: - Exception(const Error error_code, - const std::string &message = nullptr) - : zilliz::lib::Exception(error_code, "KNOWHERE", message) {} -}; - -constexpr Error UNEXPECTED = ToGlobalErrorCode(0x001); -constexpr Error UNSUPPORTED = ToGlobalErrorCode(0x002); -constexpr Error NULL_POINTER = ToGlobalErrorCode(0x003); -constexpr Error OVERFLOW = ToGlobalErrorCode(0x004); -constexpr Error INVALID_ARGUMENT = ToGlobalErrorCode(0x005); -constexpr Error UNSUPPORTED_TYPE = ToGlobalErrorCode(0x006); - - -} // namespace store -} // namespace zilliz - -using Error = zilliz::store::Error; diff --git a/cpp/src/core/include/knowhere/common/id.h b/cpp/src/core/include/knowhere/common/id.h deleted file mode 100644 index dc823e6e22..0000000000 --- a/cpp/src/core/include/knowhere/common/id.h +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once - -//#include "zcommon/id/id.h" -//using ID = zilliz::common::ID; - -#include -#include - -namespace zilliz { -namespace knowhere { - - - -class ID { - public: - constexpr static int64_t kIDSize = 20; - - public: - const int32_t * - data() const { return content_; } - - int32_t * - mutable_data() { return content_; } - - bool - IsValid() const; - - std::string - ToString() const; - - bool - operator==(const ID &that) const; - - bool - operator<(const ID &that) const; - - protected: - int32_t content_[5] = {}; -}; - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/common/schema.h b/cpp/src/core/include/knowhere/common/schema.h deleted file mode 100644 index fe87721470..0000000000 --- a/cpp/src/core/include/knowhere/common/schema.h +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once - -#include -#include "arrow/type.h" - - -namespace zilliz { -namespace knowhere { - - -using DataType = arrow::DataType; -using Field = arrow::Field; -using FieldPtr = std::shared_ptr; -using Schema = arrow::Schema; -using SchemaPtr = std::shared_ptr; -using SchemaConstPtr = std::shared_ptr; - - - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/common/tensor.h b/cpp/src/core/include/knowhere/common/tensor.h deleted file mode 100644 index 93b7642cd1..0000000000 --- a/cpp/src/core/include/knowhere/common/tensor.h +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -#include -#include "arrow/tensor.h" - - -namespace zilliz { -namespace knowhere { - - -using Tensor = arrow::Tensor; -using TensorPtr = std::shared_ptr; - - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/index/index.h b/cpp/src/core/include/knowhere/index/index.h deleted file mode 100644 index 40f3a8d5c0..0000000000 --- a/cpp/src/core/include/knowhere/index/index.h +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once - -#include -#include "knowhere/common/binary_set.h" -#include "knowhere/common/dataset.h" -#include "knowhere/index/index_type.h" -#include "knowhere/index/index_model.h" -#include "knowhere/index/preprocessor/preprocessor.h" - - -namespace zilliz { -namespace knowhere { - - -class Index { - public: - virtual BinarySet - Serialize() = 0; - - virtual void - Load(const BinarySet &index_binary) = 0; - - // @throw - virtual DatasetPtr - Search(const DatasetPtr &dataset, const Config &config) = 0; - - public: - IndexType - idx_type() const { return idx_type_; } - - void - set_idx_type(IndexType idx_type) { idx_type_ = idx_type; } - - virtual void - set_preprocessor(PreprocessorPtr preprocessor) {} - - virtual void - set_index_model(IndexModelPtr model) {} - - private: - IndexType idx_type_; -}; - - -using IndexPtr = std::shared_ptr; - - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/index/index_model.h b/cpp/src/core/include/knowhere/index/index_model.h deleted file mode 100644 index 4b3a4e439c..0000000000 --- a/cpp/src/core/include/knowhere/index/index_model.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include -#include "knowhere/common/binary_set.h" - -namespace zilliz { -namespace knowhere { - - -class IndexModel { - public: - virtual BinarySet - Serialize() = 0; - - virtual void - Load(const BinarySet &binary) = 0; -}; - -using IndexModelPtr = std::shared_ptr; - - - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/index/index_type.h b/cpp/src/core/include/knowhere/index/index_type.h deleted file mode 100644 index c7f9e39057..0000000000 --- a/cpp/src/core/include/knowhere/index/index_type.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - - -namespace zilliz { -namespace knowhere { - - -enum class IndexType { - kUnknown = 0, - kVecIdxBegin = 100, - kVecIVFFlat = kVecIdxBegin, - kVecIdxEnd, -}; - - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/index/preprocessor/preprocessor.h b/cpp/src/core/include/knowhere/index/preprocessor/preprocessor.h deleted file mode 100644 index 368276e513..0000000000 --- a/cpp/src/core/include/knowhere/index/preprocessor/preprocessor.h +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include -#include "knowhere/common/dataset.h" - - -namespace zilliz { -namespace knowhere { - - -class Preprocessor { - public: - virtual DatasetPtr - Preprocess(const DatasetPtr &input) = 0; -}; - - -using PreprocessorPtr = std::shared_ptr; - - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/index/vector_index/definitions.h b/cpp/src/core/include/knowhere/index/vector_index/definitions.h deleted file mode 100644 index 9904597584..0000000000 --- a/cpp/src/core/include/knowhere/index/vector_index/definitions.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include - - -namespace zilliz { -namespace knowhere { - -#define META_ROWS ("rows") -#define META_DIM ("dimension") -#define META_K ("k") - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/index/vector_index/kdt_parameters.h b/cpp/src/core/include/knowhere/index/vector_index/kdt_parameters.h deleted file mode 100644 index 6f2d631d42..0000000000 --- a/cpp/src/core/include/knowhere/index/vector_index/kdt_parameters.h +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once - -#include -#include - - -namespace zilliz { -namespace knowhere { - -using KDTParameter = std::pair; - -class KDTParameterManagement { - public: - const std::vector & - GetKDTParameters(); - - public: - static KDTParameterManagement & - GetInstance() { - static KDTParameterManagement instance; - return instance; - } - - KDTParameterManagement(const KDTParameterManagement &) = delete; - KDTParameterManagement &operator=(const KDTParameterManagement &) = delete; - private: - KDTParameterManagement(); - - private: - std::vector kdt_parameters_; -}; - -} // namespace knowhere -} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/index/vector_index/nsg/nsg.i b/cpp/src/core/include/knowhere/index/vector_index/nsg/nsg.i deleted file mode 100644 index d66dc49029..0000000000 --- a/cpp/src/core/include/knowhere/index/vector_index/nsg/nsg.i +++ /dev/null @@ -1,15 +0,0 @@ -%module nsg -%{ -#define SWIG_FILE_WITH_INIT -#include - -/* Include the header in the wrapper code */ -#include "nsg.h" - - -%} - - -/* Parse the header file */ -%include "index.h" - diff --git a/cpp/src/core/src/CMakeLists.txt b/cpp/src/core/knowhere/CMakeLists.txt similarity index 97% rename from cpp/src/core/src/CMakeLists.txt rename to cpp/src/core/knowhere/CMakeLists.txt index a51a62b44a..6e6d81c02c 100644 --- a/cpp/src/core/src/CMakeLists.txt +++ b/cpp/src/core/knowhere/CMakeLists.txt @@ -5,7 +5,7 @@ include_directories(${TBB_DIR}/include) include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib64) -include_directories(${CORE_SOURCE_DIR}/include) +include_directories(${CORE_SOURCE_DIR}/knowhere) include_directories(${CORE_SOURCE_DIR}/thirdparty) include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) @@ -107,7 +107,7 @@ INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so ) set(CORE_INCLUDE_DIRS - ${CORE_SOURCE_DIR}/include + ${CORE_SOURCE_DIR}/knowhere ${CORE_SOURCE_DIR}/thirdparty ${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include diff --git a/cpp/src/core/src/knowhere/adapter/arrow.cpp b/cpp/src/core/knowhere/knowhere/adapter/arrow.cpp similarity index 58% rename from cpp/src/core/src/knowhere/adapter/arrow.cpp rename to cpp/src/core/knowhere/knowhere/adapter/arrow.cpp index c70c584278..c4804d3322 100644 --- a/cpp/src/core/src/knowhere/adapter/arrow.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/arrow.cpp @@ -1,5 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. -#include "knowhere/adapter/arrow.h" + +#include "arrow.h" namespace zilliz { namespace knowhere { diff --git a/cpp/src/core/knowhere/knowhere/adapter/arrow.h b/cpp/src/core/knowhere/knowhere/adapter/arrow.h new file mode 100644 index 0000000000..318336e703 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/adapter/arrow.h @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + +#include "knowhere/common/array.h" + + +namespace zilliz { +namespace knowhere { + +ArrayPtr +CopyArray(const ArrayPtr &origin); + +SchemaPtr +CopySchema(const SchemaPtr &origin); + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/adapter/faiss_adopt.h b/cpp/src/core/knowhere/knowhere/adapter/faiss_adopt.h similarity index 100% rename from cpp/src/core/include/knowhere/adapter/faiss_adopt.h rename to cpp/src/core/knowhere/knowhere/adapter/faiss_adopt.h diff --git a/cpp/src/core/src/knowhere/adapter/sptag.cpp b/cpp/src/core/knowhere/knowhere/adapter/sptag.cpp similarity index 83% rename from cpp/src/core/src/knowhere/adapter/sptag.cpp rename to cpp/src/core/knowhere/knowhere/adapter/sptag.cpp index a724a27d91..995d068577 100644 --- a/cpp/src/core/src/knowhere/adapter/sptag.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/sptag.cpp @@ -1,7 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. #include "knowhere/index/vector_index/definitions.h" -#include "knowhere/adapter/sptag.h" -#include "knowhere/adapter/structure.h" +#include "sptag.h" +#include "structure.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/adapter/sptag.h b/cpp/src/core/knowhere/knowhere/adapter/sptag.h new file mode 100644 index 0000000000..eb7886c263 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/adapter/sptag.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + +#include + +#include "knowhere/common/dataset.h" + +namespace zilliz { +namespace knowhere { + +std::shared_ptr +ConvertToVectorSet(const DatasetPtr &dataset); + +std::shared_ptr +ConvertToMetadataSet(const DatasetPtr &dataset); + +std::vector +ConvertToQueryResult(const DatasetPtr &dataset, const Config &config); + +DatasetPtr +ConvertToDataset(std::vector query_results); + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/src/knowhere/adapter/structure.cpp b/cpp/src/core/knowhere/knowhere/adapter/structure.cpp similarity index 98% rename from cpp/src/core/src/knowhere/adapter/structure.cpp rename to cpp/src/core/knowhere/knowhere/adapter/structure.cpp index f30479dd5a..10b6af8138 100644 --- a/cpp/src/core/src/knowhere/adapter/structure.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/structure.cpp @@ -16,7 +16,7 @@ // under the License. -#include "knowhere/adapter/structure.h" +#include "structure.h" namespace zilliz { diff --git a/cpp/src/core/include/knowhere/adapter/structure.h b/cpp/src/core/knowhere/knowhere/adapter/structure.h similarity index 97% rename from cpp/src/core/include/knowhere/adapter/structure.h rename to cpp/src/core/knowhere/knowhere/adapter/structure.h index 9ffac4fdac..76a03a23e4 100644 --- a/cpp/src/core/include/knowhere/adapter/structure.h +++ b/cpp/src/core/knowhere/knowhere/adapter/structure.h @@ -19,7 +19,7 @@ #pragma once #include -#include +#include "knowhere/common/dataset.h" namespace zilliz { diff --git a/cpp/src/core/include/knowhere/common/array.h b/cpp/src/core/knowhere/knowhere/common/array.h similarity index 52% rename from cpp/src/core/include/knowhere/common/array.h rename to cpp/src/core/knowhere/knowhere/common/array.h index f3d0310596..1860524e8a 100644 --- a/cpp/src/core/include/knowhere/common/array.h +++ b/cpp/src/core/knowhere/knowhere/common/array.h @@ -1,7 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include "arrow/array.h" -#include "knowhere/common/schema.h" +#include "schema.h" namespace zilliz { diff --git a/cpp/src/core/include/knowhere/common/binary_set.h b/cpp/src/core/knowhere/knowhere/common/binary_set.h similarity index 58% rename from cpp/src/core/include/knowhere/common/binary_set.h rename to cpp/src/core/knowhere/knowhere/common/binary_set.h index b83d96dfe4..019c42a2b9 100644 --- a/cpp/src/core/include/knowhere/common/binary_set.h +++ b/cpp/src/core/knowhere/knowhere/common/binary_set.h @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include @@ -5,7 +23,7 @@ #include #include -#include "knowhere/common/id.h" +#include "id.h" namespace zilliz { diff --git a/cpp/src/core/include/knowhere/common/buffer.h b/cpp/src/core/knowhere/knowhere/common/buffer.h similarity index 56% rename from cpp/src/core/include/knowhere/common/buffer.h rename to cpp/src/core/knowhere/knowhere/common/buffer.h index 44867e760a..019c0487b7 100644 --- a/cpp/src/core/include/knowhere/common/buffer.h +++ b/cpp/src/core/knowhere/knowhere/common/buffer.h @@ -1,6 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include + #include "arrow/buffer.h" diff --git a/cpp/src/core/knowhere/knowhere/common/config.h b/cpp/src/core/knowhere/knowhere/common/config.h new file mode 100644 index 0000000000..399555c0e9 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/common/config.h @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + + +namespace zilliz { +namespace knowhere { + + +using Config = jsoncons::json; + + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/include/knowhere/common/dataset.h b/cpp/src/core/knowhere/knowhere/common/dataset.h similarity index 77% rename from cpp/src/core/include/knowhere/common/dataset.h rename to cpp/src/core/knowhere/knowhere/common/dataset.h index ee1d6e5100..068bbe4356 100644 --- a/cpp/src/core/include/knowhere/common/dataset.h +++ b/cpp/src/core/knowhere/knowhere/common/dataset.h @@ -1,12 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include #include -#include "knowhere/common/array.h" -#include "knowhere/common/buffer.h" -#include "knowhere/common/tensor.h" -#include "knowhere/common/schema.h" -#include "knowhere/common/config.h" + +#include "array.h" +#include "buffer.h" +#include "tensor.h" +#include "schema.h" +#include "config.h" #include "knowhere/adapter/arrow.h" diff --git a/cpp/src/core/src/knowhere/common/exception.cpp b/cpp/src/core/knowhere/knowhere/common/exception.cpp similarity index 97% rename from cpp/src/core/src/knowhere/common/exception.cpp rename to cpp/src/core/knowhere/knowhere/common/exception.cpp index f46cbfc35a..fcf3886d9b 100644 --- a/cpp/src/core/src/knowhere/common/exception.cpp +++ b/cpp/src/core/knowhere/knowhere/common/exception.cpp @@ -16,9 +16,10 @@ // under the License. -#include "knowhere/common/exception.h" #include +#include "exception.h" + namespace zilliz { namespace knowhere { diff --git a/cpp/src/core/include/knowhere/common/exception.h b/cpp/src/core/knowhere/knowhere/common/exception.h similarity index 100% rename from cpp/src/core/include/knowhere/common/exception.h rename to cpp/src/core/knowhere/knowhere/common/exception.h diff --git a/cpp/src/core/knowhere/knowhere/common/id.h b/cpp/src/core/knowhere/knowhere/common/id.h new file mode 100644 index 0000000000..87e1467c44 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/common/id.h @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +//#include "zcommon/id/id.h" +//using ID = zilliz::common::ID; + +#include +#include + +namespace zilliz { +namespace knowhere { + + + +class ID { + public: + constexpr static int64_t kIDSize = 20; + + public: + const int32_t * + data() const { return content_; } + + int32_t * + mutable_data() { return content_; } + + bool + IsValid() const; + + std::string + ToString() const; + + bool + operator==(const ID &that) const; + + bool + operator<(const ID &that) const; + + protected: + int32_t content_[5] = {}; +}; + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/common/schema.h b/cpp/src/core/knowhere/knowhere/common/schema.h new file mode 100644 index 0000000000..1ab3d42f8c --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/common/schema.h @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + +#include "arrow/type.h" + + +namespace zilliz { +namespace knowhere { + + +using DataType = arrow::DataType; +using Field = arrow::Field; +using FieldPtr = std::shared_ptr; +using Schema = arrow::Schema; +using SchemaPtr = std::shared_ptr; +using SchemaConstPtr = std::shared_ptr; + + + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/common/tensor.h b/cpp/src/core/knowhere/knowhere/common/tensor.h new file mode 100644 index 0000000000..8b92f1f891 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/common/tensor.h @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + +#include "arrow/tensor.h" + + +namespace zilliz { +namespace knowhere { + + +using Tensor = arrow::Tensor; +using TensorPtr = std::shared_ptr; + + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/src/knowhere/common/timer.cpp b/cpp/src/core/knowhere/knowhere/common/timer.cpp similarity index 98% rename from cpp/src/core/src/knowhere/common/timer.cpp rename to cpp/src/core/knowhere/knowhere/common/timer.cpp index 08654413ff..f57de535d4 100644 --- a/cpp/src/core/src/knowhere/common/timer.cpp +++ b/cpp/src/core/knowhere/knowhere/common/timer.cpp @@ -18,7 +18,7 @@ #include // TODO(linxj): using Log instead -#include "knowhere/common/timer.h" +#include "timer.h" namespace zilliz { namespace knowhere { diff --git a/cpp/src/core/include/knowhere/common/timer.h b/cpp/src/core/knowhere/knowhere/common/timer.h similarity index 100% rename from cpp/src/core/include/knowhere/common/timer.h rename to cpp/src/core/knowhere/knowhere/common/timer.h diff --git a/cpp/src/core/knowhere/knowhere/index/index.h b/cpp/src/core/knowhere/knowhere/index/index.h new file mode 100644 index 0000000000..275a5fd387 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/index.h @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + +#include "knowhere/common/binary_set.h" +#include "knowhere/common/dataset.h" +#include "index_type.h" +#include "index_model.h" +#include "knowhere/index/preprocessor/preprocessor.h" + + +namespace zilliz { +namespace knowhere { + + +class Index { + public: + virtual BinarySet + Serialize() = 0; + + virtual void + Load(const BinarySet &index_binary) = 0; + + // @throw + virtual DatasetPtr + Search(const DatasetPtr &dataset, const Config &config) = 0; + + public: + IndexType + idx_type() const { return idx_type_; } + + void + set_idx_type(IndexType idx_type) { idx_type_ = idx_type; } + + virtual void + set_preprocessor(PreprocessorPtr preprocessor) {} + + virtual void + set_index_model(IndexModelPtr model) {} + + private: + IndexType idx_type_; +}; + + +using IndexPtr = std::shared_ptr; + + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/index_model.h b/cpp/src/core/knowhere/knowhere/index/index_model.h new file mode 100644 index 0000000000..d00a629d7e --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/index_model.h @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include +#include "knowhere/common/binary_set.h" + +namespace zilliz { +namespace knowhere { + + +class IndexModel { + public: + virtual BinarySet + Serialize() = 0; + + virtual void + Load(const BinarySet &binary) = 0; +}; + +using IndexModelPtr = std::shared_ptr; + + + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/index_type.h b/cpp/src/core/knowhere/knowhere/index/index_type.h new file mode 100644 index 0000000000..3ece6287f5 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/index_type.h @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + + +namespace zilliz { +namespace knowhere { + + +enum class IndexType { + kUnknown = 0, + kVecIdxBegin = 100, + kVecIVFFlat = kVecIdxBegin, + kVecIdxEnd, +}; + + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/src/knowhere/index/preprocessor/normalize.cpp b/cpp/src/core/knowhere/knowhere/index/preprocessor/normalize.cpp similarity index 100% rename from cpp/src/core/src/knowhere/index/preprocessor/normalize.cpp rename to cpp/src/core/knowhere/knowhere/index/preprocessor/normalize.cpp diff --git a/cpp/src/core/include/knowhere/index/preprocessor/normalize.h b/cpp/src/core/knowhere/knowhere/index/preprocessor/normalize.h similarity index 100% rename from cpp/src/core/include/knowhere/index/preprocessor/normalize.h rename to cpp/src/core/knowhere/knowhere/index/preprocessor/normalize.h diff --git a/cpp/src/core/knowhere/knowhere/index/preprocessor/preprocessor.h b/cpp/src/core/knowhere/knowhere/index/preprocessor/preprocessor.h new file mode 100644 index 0000000000..10f0544270 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/preprocessor/preprocessor.h @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + +#include "knowhere/common/dataset.h" + + +namespace zilliz { +namespace knowhere { + + +class Preprocessor { + public: + virtual DatasetPtr + Preprocess(const DatasetPtr &input) = 0; +}; + + +using PreprocessorPtr = std::shared_ptr; + + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/src/knowhere/index/vector_index/cloner.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/cloner.cpp similarity index 91% rename from cpp/src/core/src/knowhere/index/vector_index/cloner.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/cloner.cpp index f5bcfb896d..4519def300 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/cloner.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/cloner.cpp @@ -17,10 +17,10 @@ #include "knowhere/common/exception.h" -#include "knowhere/index/vector_index/cloner.h" -#include "knowhere/index/vector_index/ivf.h" -#include "knowhere/index/vector_index/gpu_ivf.h" -#include "knowhere/index/vector_index/idmap.h" +#include "cloner.h" +#include "ivf.h" +#include "gpu_ivf.h" +#include "idmap.h" namespace zilliz { diff --git a/cpp/src/core/include/knowhere/index/vector_index/cloner.h b/cpp/src/core/knowhere/knowhere/index/vector_index/cloner.h similarity index 100% rename from cpp/src/core/include/knowhere/index/vector_index/cloner.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/cloner.h diff --git a/cpp/src/core/src/knowhere/index/vector_index/cpu_kdt_rng.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp similarity index 83% rename from cpp/src/core/src/knowhere/index/vector_index/cpu_kdt_rng.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp index 0805398144..d588efb7c6 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/cpu_kdt_rng.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include #include @@ -7,10 +24,10 @@ #undef mkdir -#include "knowhere/index/vector_index/cpu_kdt_rng.h" -#include "knowhere/index/vector_index/definitions.h" +#include "cpu_kdt_rng.h" +#include "definitions.h" //#include "knowhere/index/preprocessor/normalize.h" -#include "knowhere/index/vector_index/kdt_parameters.h" +#include "kdt_parameters.h" #include "knowhere/adapter/sptag.h" #include "knowhere/common/exception.h" diff --git a/cpp/src/core/include/knowhere/index/vector_index/cpu_kdt_rng.h b/cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.h similarity index 66% rename from cpp/src/core/include/knowhere/index/vector_index/cpu_kdt_rng.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.h index 0e1cc55e5d..35d488999e 100644 --- a/cpp/src/core/include/knowhere/index/vector_index/cpu_kdt_rng.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.h @@ -1,8 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include #include -#include "knowhere/index/vector_index/vector_index.h" +#include "vector_index.h" #include "knowhere/index/index_model.h" #include diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/definitions.h b/cpp/src/core/knowhere/knowhere/index/vector_index/definitions.h new file mode 100644 index 0000000000..3db32e00dc --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/definitions.h @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + + +namespace zilliz { +namespace knowhere { + +#define META_ROWS ("rows") +#define META_DIM ("dimension") +#define META_K ("k") + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/src/knowhere/index/vector_index/gpu_ivf.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp similarity index 99% rename from cpp/src/core/src/knowhere/index/vector_index/gpu_ivf.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp index 590ded16bf..fd04501c83 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/gpu_ivf.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp @@ -27,9 +27,9 @@ #include "knowhere/common/exception.h" -#include "knowhere/index/vector_index/cloner.h" +#include "cloner.h" #include "knowhere/adapter/faiss_adopt.h" -#include "knowhere/index/vector_index/gpu_ivf.h" +#include "gpu_ivf.h" #include diff --git a/cpp/src/core/include/knowhere/index/vector_index/gpu_ivf.h b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h similarity index 87% rename from cpp/src/core/include/knowhere/index/vector_index/gpu_ivf.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h index 8eed6f8fd1..429a94167a 100644 --- a/cpp/src/core/include/knowhere/index/vector_index/gpu_ivf.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h @@ -1,3 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include diff --git a/cpp/src/core/src/knowhere/index/vector_index/idmap.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp similarity index 99% rename from cpp/src/core/src/knowhere/index/vector_index/idmap.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp index 142d06b5d8..b4085e6df8 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/idmap.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp @@ -25,7 +25,7 @@ #include "knowhere/common/exception.h" #include "knowhere/adapter/faiss_adopt.h" -#include "knowhere/index/vector_index/idmap.h" +#include "idmap.h" namespace zilliz { diff --git a/cpp/src/core/include/knowhere/index/vector_index/idmap.h b/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.h similarity index 100% rename from cpp/src/core/include/knowhere/index/vector_index/idmap.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/idmap.h diff --git a/cpp/src/core/src/knowhere/index/vector_index/ivf.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/ivf.cpp similarity index 98% rename from cpp/src/core/src/knowhere/index/vector_index/ivf.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/ivf.cpp index acfbb17651..6d72479002 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/ivf.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/ivf.cpp @@ -25,13 +25,12 @@ #include #include #include -#include #include "knowhere/common/exception.h" -#include "knowhere/index/vector_index/ivf.h" +#include "ivf.h" #include "knowhere/adapter/faiss_adopt.h" -#include "knowhere/index/vector_index/gpu_ivf.h" +#include "gpu_ivf.h" namespace zilliz { diff --git a/cpp/src/core/include/knowhere/index/vector_index/ivf.h b/cpp/src/core/knowhere/knowhere/index/vector_index/ivf.h similarity index 98% rename from cpp/src/core/include/knowhere/index/vector_index/ivf.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/ivf.h index e8ac6cae99..c23b5abcc2 100644 --- a/cpp/src/core/include/knowhere/index/vector_index/ivf.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/ivf.h @@ -25,7 +25,7 @@ #include #include -#include "knowhere/index/vector_index/vector_index.h" +#include "vector_index.h" namespace zilliz { diff --git a/cpp/src/core/src/knowhere/index/vector_index/kdt_parameters.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.cpp similarity index 54% rename from cpp/src/core/src/knowhere/index/vector_index/kdt_parameters.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.cpp index 9f62b5d3a2..b0ed9f6d56 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/kdt_parameters.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.cpp @@ -1,6 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #include -#include "knowhere/index/vector_index/kdt_parameters.h" + +#include "kdt_parameters.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.h b/cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.h new file mode 100644 index 0000000000..03f660a0ef --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.h @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include +#include + + +namespace zilliz { +namespace knowhere { + +using KDTParameter = std::pair; + +class KDTParameterManagement { + public: + const std::vector & + GetKDTParameters(); + + public: + static KDTParameterManagement & + GetInstance() { + static KDTParameterManagement instance; + return instance; + } + + KDTParameterManagement(const KDTParameterManagement &) = delete; + KDTParameterManagement &operator=(const KDTParameterManagement &) = delete; + private: + KDTParameterManagement(); + + private: + std::vector kdt_parameters_; +}; + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/src/knowhere/index/vector_index/nsg/index_io.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/index_io.h similarity index 95% rename from cpp/src/core/src/knowhere/index/vector_index/nsg/index_io.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/index_io.h index fd8fc9e7ad..60e27bd7f6 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/nsg/index_io.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/index_io.h @@ -18,7 +18,7 @@ #pragma once -#include "knowhere/index/vector_index/nsg/nsg.h" +#include "nsg.h" namespace zilliz { namespace knowhere { diff --git a/cpp/src/core/include/knowhere/index/vector_index/nsg/neighbor.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/neighbor.h similarity index 100% rename from cpp/src/core/include/knowhere/index/vector_index/nsg/neighbor.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/neighbor.h diff --git a/cpp/src/core/src/knowhere/index/vector_index/nsg/nsg.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.cpp similarity index 99% rename from cpp/src/core/src/knowhere/index/vector_index/nsg/nsg.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.cpp index d3efc5e38a..d42f4c5662 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/nsg/nsg.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.cpp @@ -22,7 +22,7 @@ #include #include -#include "knowhere/index/vector_index/nsg/nsg.h" +#include "nsg.h" #include "knowhere/common/exception.h" #include "knowhere/common/timer.h" #include "utils.h" diff --git a/cpp/src/core/include/knowhere/index/vector_index/nsg/nsg.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.h similarity index 100% rename from cpp/src/core/include/knowhere/index/vector_index/nsg/nsg.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.h diff --git a/cpp/src/core/src/knowhere/index/vector_index/nsg/nsg_io.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.cpp similarity index 97% rename from cpp/src/core/src/knowhere/index/vector_index/nsg/nsg_io.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.cpp index 283e8cc7e2..34d629ec4b 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/nsg/nsg_io.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.cpp @@ -18,7 +18,7 @@ #include -#include "knowhere/index/vector_index/nsg/nsg_io.h" +#include "nsg_io.h" namespace zilliz { diff --git a/cpp/src/core/include/knowhere/index/vector_index/nsg/nsg_io.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.h similarity index 100% rename from cpp/src/core/include/knowhere/index/vector_index/nsg/nsg_io.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.h diff --git a/cpp/src/core/src/knowhere/index/vector_index/nsg/utils.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.cpp similarity index 100% rename from cpp/src/core/src/knowhere/index/vector_index/nsg/utils.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.cpp diff --git a/cpp/src/core/src/knowhere/index/vector_index/nsg/utils.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.h similarity index 95% rename from cpp/src/core/src/knowhere/index/vector_index/nsg/utils.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.h index 82c8a00dc8..a57a7738fd 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/nsg/utils.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.h @@ -23,7 +23,7 @@ #include -#include "knowhere/index/vector_index/nsg/nsg.h" +#include "nsg.h" #include "knowhere/common/config.h" diff --git a/cpp/src/core/src/knowhere/index/vector_index/nsg_index.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.cpp similarity index 96% rename from cpp/src/core/src/knowhere/index/vector_index/nsg_index.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.cpp index 550efef70d..724c80dfca 100644 --- a/cpp/src/core/src/knowhere/index/vector_index/nsg_index.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.cpp @@ -16,12 +16,12 @@ // under the License. -#include "knowhere/index/vector_index/nsg_index.h" +#include "nsg_index.h" #include "knowhere/index/vector_index/nsg/nsg.h" #include "knowhere/index/vector_index/nsg/nsg_io.h" -#include "knowhere/index/vector_index/idmap.h" -#include "knowhere/index/vector_index/ivf.h" -#include "knowhere/index/vector_index/gpu_ivf.h" +#include "idmap.h" +#include "ivf.h" +#include "gpu_ivf.h" #include "knowhere/adapter/faiss_adopt.h" #include "knowhere/common/exception.h" #include "knowhere/common/timer.h" diff --git a/cpp/src/core/include/knowhere/index/vector_index/nsg_index.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.h similarity index 96% rename from cpp/src/core/include/knowhere/index/vector_index/nsg_index.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.h index a0829f381c..e63dbb03b4 100644 --- a/cpp/src/core/include/knowhere/index/vector_index/nsg_index.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.h @@ -18,7 +18,7 @@ #pragma once -#include "knowhere/index/vector_index/vector_index.h" +#include "vector_index.h" namespace zilliz { diff --git a/cpp/src/core/include/knowhere/index/vector_index/vector_index.h b/cpp/src/core/knowhere/knowhere/index/vector_index/vector_index.h similarity index 52% rename from cpp/src/core/include/knowhere/index/vector_index/vector_index.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/vector_index.h index 1618bc01bb..1828e3573d 100644 --- a/cpp/src/core/include/knowhere/index/vector_index/vector_index.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/vector_index.h @@ -1,7 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #pragma once #include + #include "knowhere/common/config.h" #include "knowhere/common/dataset.h" #include "knowhere/index/index.h" diff --git a/cpp/src/core/test/CMakeLists.txt b/cpp/src/core/test/CMakeLists.txt index cba8a03b94..5b8dca2e4b 100644 --- a/cpp/src/core/test/CMakeLists.txt +++ b/cpp/src/core/test/CMakeLists.txt @@ -1,6 +1,6 @@ include_directories(${CORE_SOURCE_DIR}/thirdparty) include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) -include_directories(${CORE_SOURCE_DIR}/include) +include_directories(${CORE_SOURCE_DIR}/knowhere) include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) include_directories(/usr/local/cuda/include) link_directories(/usr/local/cuda/lib64) @@ -25,13 +25,13 @@ set(basic_libs # set(ivf_srcs - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp - ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ivf.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu_ivf.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/cloner.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/idmap.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp utils.cpp ) if(NOT TARGET test_ivf) @@ -41,13 +41,13 @@ target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs}) # set(idmap_srcs - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp - ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/idmap.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ivf.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/cloner.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu_ivf.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp utils.cpp ) if(NOT TARGET test_idmap) @@ -57,14 +57,14 @@ target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs}) # set(kdt_srcs - ${CORE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cpu_kdt_rng.cpp - ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CORE_SOURCE_DIR}/src/knowhere/adapter/sptag.cpp - ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CORE_SOURCE_DIR}/src/knowhere/adapter/arrow.cpp - ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/preprocessor/normalize.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/kdt_parameters.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/sptag.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/arrow.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp utils.cpp ) if(NOT TARGET test_kdt) From a5b3dea158906376e92aedefd99fb36c017a0114 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 18 Sep 2019 17:30:05 +0800 Subject: [PATCH 032/217] update Former-commit-id: 3d3617149b664de4d6af554af6e984163b963fe6 --- cpp/CMakeLists.txt | 6 +- cpp/src/CMakeLists.txt | 76 ++++++++++--------- cpp/src/sdk/CMakeLists.txt | 13 ---- cpp/unittest/CMakeLists.txt | 50 +++++++----- cpp/unittest/db/CMakeLists.txt | 22 +----- cpp/unittest/metrics/CMakeLists.txt | 23 +----- cpp/unittest/scheduler/CMakeLists.txt | 21 +---- cpp/unittest/server/CMakeLists.txt | 26 +------ .../{knowhere => wrapper}/CMakeLists.txt | 10 +-- cpp/unittest/{knowhere => wrapper}/utils.cpp | 0 cpp/unittest/{knowhere => wrapper}/utils.h | 0 .../wrapper_test.cpp} | 0 12 files changed, 90 insertions(+), 157 deletions(-) rename cpp/unittest/{knowhere => wrapper}/CMakeLists.txt (84%) rename cpp/unittest/{knowhere => wrapper}/utils.cpp (100%) rename cpp/unittest/{knowhere => wrapper}/utils.h (100%) rename cpp/unittest/{knowhere/knowhere_test.cpp => wrapper/wrapper_test.cpp} (100%) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 91c7b5286b..bb15ddf075 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -69,9 +69,6 @@ message(STATUS "Milvus version: " "${MILVUS_VERSION_MAJOR}.${MILVUS_VERSION_MINOR}.${MILVUS_VERSION_PATCH} " "(full: '${MILVUS_VERSION}')") -find_package(CUDA) -set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda") - set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED on) @@ -87,6 +84,9 @@ else() set(MILVUS_BUILD_ARCH unknown) endif() +find_package(CUDA) +set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda") + if(CMAKE_BUILD_TYPE STREQUAL "Release") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp") set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3") diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index b7e719666b..21a9f2bccb 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -27,55 +27,55 @@ foreach (dir ${CORE_INCLUDE_DIRS}) include_directories(${dir}) endforeach () -aux_source_directory(cache cache_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files) -aux_source_directory(config config_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) -aux_source_directory(db db_main_files) -aux_source_directory(db/engine db_engine_files) -aux_source_directory(db/insert db_insert_files) -aux_source_directory(db/meta db_meta_files) -aux_source_directory(db/scheduler scheduler_files) -aux_source_directory(db/scheduler/context scheduler_context_files) -aux_source_directory(db/scheduler/task scheduler_task_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler db_scheduler_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context db_scheduler_context_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task db_scheduler_task_files) set(db_scheduler_files - ${scheduler_files} - ${scheduler_context_files} - ${scheduler_task_files} + ${db_scheduler_files} + ${db_scheduler_context_files} + ${db_scheduler_task_files} ) set(grpc_service_files - grpc/gen-milvus/milvus.grpc.pb.cc - grpc/gen-milvus/milvus.pb.cc - grpc/gen-status/status.grpc.pb.cc - grpc/gen-status/status.pb.cc + ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.grpc.pb.cc + ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.pb.cc + ${MILVUS_ENGINE_SRC}/grpc/gen-status/status.grpc.pb.cc + ${MILVUS_ENGINE_SRC}/grpc/gen-status/status.pb.cc ) -aux_source_directory(metrics metrics_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files) -aux_source_directory(scheduler/action scheduler_action_files) -aux_source_directory(scheduler/event scheduler_event_files) -aux_source_directory(scheduler/job scheduler_job_files) -aux_source_directory(scheduler/resource scheduler_resource_files) -aux_source_directory(scheduler/task scheduler_task_files) -aux_source_directory(scheduler scheduler_root_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_main_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/job scheduler_job_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_files) set(scheduler_files + ${scheduler_main_files} ${scheduler_action_files} ${scheduler_event_files} ${scheduler_job_files} ${scheduler_resource_files} ${scheduler_task_files} - ${scheduler_root_files} ) -aux_source_directory(server server_files) -aux_source_directory(server/grpc_impl grpc_server_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_files) -aux_source_directory(utils utils_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files) -aux_source_directory(wrapper/knowhere knowhere_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere wrapper_knowhere_files) -set(db_files +set(engine_files ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp ${cache_files} ${db_main_files} @@ -85,30 +85,32 @@ set(db_files ${db_scheduler_files} ${metrics_files} ${utils_files} - ${knowhere_files} + ${wrapper_knowhere_files} ) -include_directories(/usr/include) include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") -include_directories(grpc/gen-status) -include_directories(grpc/gen-milvus) +include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status) +include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus) set(client_grpc_lib grpcpp_channelz grpc++ grpc grpc_protobuf - grpc_protoc) + grpc_protoc + ) set(prometheus_lib prometheus-cpp-push prometheus-cpp-pull - prometheus-cpp-core) + prometheus-cpp-core + ) set(boost_lib boost_system_static boost_filesystem_static - boost_serialization_static) + boost_serialization_static + ) set(third_party_libs sqlite @@ -149,7 +151,7 @@ if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") ) endif () -cuda_add_library(milvus_engine STATIC ${db_files}) +cuda_add_library(milvus_engine STATIC ${engine_files}) target_link_libraries(milvus_engine ${engine_libs} ${third_party_libs}) add_library(metrics STATIC ${metrics_files}) diff --git a/cpp/src/sdk/CMakeLists.txt b/cpp/src/sdk/CMakeLists.txt index 5d6a1ea465..e7bf989e96 100644 --- a/cpp/src/sdk/CMakeLists.txt +++ b/cpp/src/sdk/CMakeLists.txt @@ -20,23 +20,10 @@ aux_source_directory(interface interface_files) -include_directories(/usr/include) include_directories(include) -include_directories(/usr/local/include) - aux_source_directory(grpc grpc_client_files) -include_directories(${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus) -include_directories(${CMAKE_SOURCE_DIR}/src/grpc/gen-status) - -set(grpc_service_files - ${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus/milvus.grpc.pb.cc - ${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus/milvus.pb.cc - ${CMAKE_SOURCE_DIR}/src/grpc/gen-status/status.grpc.pb.cc - ${CMAKE_SOURCE_DIR}/src/grpc/gen-status/status.pb.cc - ) - add_library(milvus_sdk STATIC ${interface_files} ${grpc_client_files} diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 37dec9b0ac..14ee2c5eb1 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -28,37 +28,37 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler db_scheduler_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context db_scheduler_context_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task db_scheduler_task_files) set(db_scheduler_files - ${scheduler_files} - ${scheduler_context_files} - ${scheduler_task_files} + ${db_scheduler_files} + ${db_scheduler_context_files} + ${db_scheduler_task_files} ) set(grpc_service_files - grpc/gen-milvus/milvus.grpc.pb.cc - grpc/gen-milvus/milvus.pb.cc - grpc/gen-status/status.grpc.pb.cc - grpc/gen-status/status.pb.cc + ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.grpc.pb.cc + ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.pb.cc + ${MILVUS_ENGINE_SRC}/grpc/gen-status/status.grpc.pb.cc + ${MILVUS_ENGINE_SRC}/grpc/gen-status/status.pb.cc ) aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_main_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/job scheduler_job_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_root_files) set(scheduler_files + ${scheduler_main_files} ${scheduler_action_files} ${scheduler_event_files} ${scheduler_job_files} ${scheduler_resource_files} ${scheduler_task_files} - ${scheduler_root_files} ) aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files) @@ -66,10 +66,7 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_files) aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_files) - -aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere wrapper_knowhere_files) set(unittest_files ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp) @@ -83,7 +80,26 @@ set(helper_files ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc ) +set(common_files + ${cache_files} + ${config_files} + ${db_main_files} + ${db_engine_files} + ${db_insert_files} + ${db_meta_files} + ${db_scheduler_files} + ${metrics_files} + ${scheduler_files} + ${wrapper_knowhere_files} + ${helper_files} + ) + set(unittest_libs + sqlite + boost_system_static + boost_filesystem_static + lz4 + mysqlpp yaml-cpp gtest gmock @@ -107,7 +123,7 @@ foreach(dir ${CORE_INCLUDE_DIRS}) endforeach() add_subdirectory(db) -add_subdirectory(knowhere) +add_subdirectory(wrapper) add_subdirectory(metrics) add_subdirectory(scheduler) add_subdirectory(server) \ No newline at end of file diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 3ab33d3cc7..ca6bad8824 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -23,31 +23,13 @@ include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") set(db_test_files - ${cache_files} - ${config_files} - ${db_main_files} - ${db_engine_files} - ${db_insert_files} - ${db_meta_files} - ${db_scheduler_files} - ${metrics_files} - ${scheduler_files} - ${knowhere_files} - ${helper_files} + ${common_files} ${test_files} ) cuda_add_executable(db_test ${db_test_files}) -set(db_libs - sqlite - boost_system_static - boost_filesystem_static - lz4 - mysqlpp - ) - -target_link_libraries(db_test ${db_libs} knowhere ${unittest_libs}) +target_link_libraries(db_test knowhere ${unittest_libs}) install(TARGETS db_test DESTINATION unittest) diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 9df30c4340..04eab826bc 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -19,35 +19,16 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files) -include_directories(/usr/include) include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") set(metrics_test_files - ${cache_files} - ${config_files} - ${db_main_files} - ${db_engine_files} - ${db_insert_files} - ${db_meta_files} - ${db_scheduler_files} - ${metrics_files} - ${scheduler_files} - ${knowhere_files} - ${helper_files} + ${common_files} ${test_files} ) add_executable(metrics_test ${metrics_test_files}) -target_link_libraries(metrics_test - knowhere - sqlite - boost_system_static - boost_filesystem_static - lz4 - mysqlpp - ${unittest_libs} - ) +target_link_libraries(metrics_test knowhere ${unittest_libs}) install(TARGETS metrics_test DESTINATION unittest) \ No newline at end of file diff --git a/cpp/unittest/scheduler/CMakeLists.txt b/cpp/unittest/scheduler/CMakeLists.txt index 9386596da0..27ee11775c 100644 --- a/cpp/unittest/scheduler/CMakeLists.txt +++ b/cpp/unittest/scheduler/CMakeLists.txt @@ -23,31 +23,14 @@ include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") set(scheduler_test_files - ${cache_files} - ${config_files} - ${db_main_files} - ${db_engine_files} - ${db_insert_files} - ${db_meta_files} - ${db_scheduler_files} - ${scheduler_files} - ${knowhere_files} - ${helper_files} + ${common_files} ${unittest_files} ${test_files} ) cuda_add_executable(scheduler_test ${scheduler_test_files}) -set(scheduler_libs - sqlite - boost_system_static - boost_filesystem_static - lz4 - mysqlpp - ) - -target_link_libraries(scheduler_test ${scheduler_libs} knowhere ${unittest_libs}) +target_link_libraries(scheduler_test knowhere ${unittest_libs}) install(TARGETS scheduler_test DESTINATION unittest) diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index 395f2e40ec..d5bbd6cc47 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -19,14 +19,13 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files) -include_directories(/usr/include) include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status) include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus) -set(helper_files ${helper_files} +set(util_files ${MILVUS_ENGINE_SRC}/utils/StringHelpFunctions.cpp ${MILVUS_ENGINE_SRC}/utils/LogUtil.cpp ${MILVUS_ENGINE_SRC}/utils/SignalUtil.cpp) @@ -39,19 +38,11 @@ set(grpc_service_files ) set(server_test_files - ${cache_files} - ${config_files} - ${db_main_files} - ${db_engine_files} - ${db_insert_files} - ${db_meta_files} - ${db_scheduler_files} - ${scheduler_files} + ${common_files} ${server_files} ${grpc_server_files} ${grpc_service_files} - ${knowhere_files} - ${helper_files} + ${util_files} ${unittest_files} ${test_files} ) @@ -65,22 +56,13 @@ set(client_grpc_lib grpc_protobuf grpc_protoc) -set(require_libs +target_link_libraries(server_test knowhere stdc++ - sqlite - mysqlpp - boost_system_static - boost_filesystem_static snappy bz2 zstd - lz4 ${client_grpc_lib} - ) - -target_link_libraries(server_test - ${require_libs} ${unittest_libs} ) diff --git a/cpp/unittest/knowhere/CMakeLists.txt b/cpp/unittest/wrapper/CMakeLists.txt similarity index 84% rename from cpp/unittest/knowhere/CMakeLists.txt rename to cpp/unittest/wrapper/CMakeLists.txt index 5606b3820c..43ac0be9ce 100644 --- a/cpp/unittest/knowhere/CMakeLists.txt +++ b/cpp/unittest/wrapper/CMakeLists.txt @@ -20,12 +20,12 @@ include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") -set(knowhere_files +set(wrapper_knowhere_files ${MILVUS_ENGINE_SRC}/wrapper/knowhere/data_transfer.cpp ${MILVUS_ENGINE_SRC}/wrapper/knowhere/vec_impl.cpp ${MILVUS_ENGINE_SRC}/wrapper/knowhere/vec_index.cpp) -set(helper_files +set(util_files utils.cpp ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc ) @@ -36,7 +36,7 @@ set(knowhere_libs cublas ) -add_executable(knowhere_test knowhere_test.cpp ${knowhere_files} ${helper_files}) -target_link_libraries(knowhere_test ${knowhere_libs} ${unittest_libs}) +add_executable(wrapper_test wrapper_test.cpp ${wrapper_knowhere_files} ${util_files}) +target_link_libraries(wrapper_test ${knowhere_libs} ${unittest_libs}) -install(TARGETS knowhere_test DESTINATION unittest) \ No newline at end of file +install(TARGETS wrapper_test DESTINATION unittest) \ No newline at end of file diff --git a/cpp/unittest/knowhere/utils.cpp b/cpp/unittest/wrapper/utils.cpp similarity index 100% rename from cpp/unittest/knowhere/utils.cpp rename to cpp/unittest/wrapper/utils.cpp diff --git a/cpp/unittest/knowhere/utils.h b/cpp/unittest/wrapper/utils.h similarity index 100% rename from cpp/unittest/knowhere/utils.h rename to cpp/unittest/wrapper/utils.h diff --git a/cpp/unittest/knowhere/knowhere_test.cpp b/cpp/unittest/wrapper/wrapper_test.cpp similarity index 100% rename from cpp/unittest/knowhere/knowhere_test.cpp rename to cpp/unittest/wrapper/wrapper_test.cpp From 59ea536db600077fe4b103ce9efde4a77e5f7ee9 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Wed, 18 Sep 2019 17:30:47 +0800 Subject: [PATCH 033/217] MS-565 1. del wrapper/knowhere Former-commit-id: 3905da5c0716c6fe8317588c27a61b4a5c36316e --- cpp/src/CMakeLists.txt | 4 +- cpp/src/cache/DataObj.h | 2 +- cpp/src/db/engine/ExecutionEngineImpl.cpp | 4 +- cpp/src/db/engine/ExecutionEngineImpl.h | 2 +- cpp/src/server/Server.cpp | 2 +- .../{knowhere => }/KnowhereResource.cpp | 0 .../wrapper/{knowhere => }/KnowhereResource.h | 0 .../wrapper/{knowhere => }/data_transfer.cpp | 0 .../wrapper/{knowhere => }/data_transfer.h | 0 cpp/src/wrapper/gpu/Arithmetic.h | 79 --- cpp/src/wrapper/gpu/Topk.cu | 586 ------------------ cpp/src/wrapper/gpu/Topk.h | 73 --- cpp/src/wrapper/{knowhere => }/vec_impl.cpp | 0 cpp/src/wrapper/{knowhere => }/vec_impl.h | 0 cpp/src/wrapper/{knowhere => }/vec_index.cpp | 0 cpp/src/wrapper/{knowhere => }/vec_index.h | 0 cpp/unittest/knowhere/CMakeLists.txt | 6 +- cpp/unittest/knowhere/knowhere_test.cpp | 2 +- cpp/unittest/scheduler/scheduler_test.cpp | 2 +- cpp/unittest/server/cache_test.cpp | 2 +- 20 files changed, 13 insertions(+), 751 deletions(-) rename cpp/src/wrapper/{knowhere => }/KnowhereResource.cpp (100%) rename cpp/src/wrapper/{knowhere => }/KnowhereResource.h (100%) rename cpp/src/wrapper/{knowhere => }/data_transfer.cpp (100%) rename cpp/src/wrapper/{knowhere => }/data_transfer.h (100%) delete mode 100644 cpp/src/wrapper/gpu/Arithmetic.h delete mode 100644 cpp/src/wrapper/gpu/Topk.cu delete mode 100644 cpp/src/wrapper/gpu/Topk.h rename cpp/src/wrapper/{knowhere => }/vec_impl.cpp (100%) rename cpp/src/wrapper/{knowhere => }/vec_impl.h (100%) rename cpp/src/wrapper/{knowhere => }/vec_index.cpp (100%) rename cpp/src/wrapper/{knowhere => }/vec_index.h (100%) diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index c8cba4d9aa..60def5c4ca 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -35,7 +35,7 @@ aux_source_directory(db/engine db_engine_files) aux_source_directory(db/insert db_insert_files) aux_source_directory(db/meta db_meta_files) aux_source_directory(metrics metrics_files) -aux_source_directory(wrapper/knowhere knowhere_files) +aux_source_directory(wrapper wrapper_files) aux_source_directory(scheduler/action scheduler_action_files) aux_source_directory(scheduler/event scheduler_event_files) @@ -88,7 +88,7 @@ set(db_files ${db_meta_files} ${db_scheduler_files} ${metrics_files} - ${knowhere_files} + ${wrapper_files} ${utils_files} ) diff --git a/cpp/src/cache/DataObj.h b/cpp/src/cache/DataObj.h index 8ba59104cd..12406301cb 100644 --- a/cpp/src/cache/DataObj.h +++ b/cpp/src/cache/DataObj.h @@ -18,7 +18,7 @@ #pragma once -#include "wrapper/knowhere/vec_index.h" +#include "src/wrapper/vec_index.h" #include diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 7ad9d3ea12..2fb868f548 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -23,8 +23,8 @@ #include "utils/CommonUtil.h" #include "utils/Exception.h" -#include "wrapper/knowhere/vec_index.h" -#include "wrapper/knowhere/vec_impl.h" +#include "src/wrapper/vec_index.h" +#include "src/wrapper/vec_impl.h" #include "knowhere/common/exception.h" #include diff --git a/cpp/src/db/engine/ExecutionEngineImpl.h b/cpp/src/db/engine/ExecutionEngineImpl.h index 611a247a08..cb08c50ad4 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.h +++ b/cpp/src/db/engine/ExecutionEngineImpl.h @@ -18,7 +18,7 @@ #pragma once #include "ExecutionEngine.h" -#include "wrapper/knowhere/vec_index.h" +#include "src/wrapper/vec_index.h" #include #include diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 252f92a5f0..9920b30246 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -32,7 +32,7 @@ #include #include #include -#include "wrapper/knowhere/KnowhereResource.h" +#include "src/wrapper/KnowhereResource.h" #include "metrics/Metrics.h" #include "DBWrapper.h" diff --git a/cpp/src/wrapper/knowhere/KnowhereResource.cpp b/cpp/src/wrapper/KnowhereResource.cpp similarity index 100% rename from cpp/src/wrapper/knowhere/KnowhereResource.cpp rename to cpp/src/wrapper/KnowhereResource.cpp diff --git a/cpp/src/wrapper/knowhere/KnowhereResource.h b/cpp/src/wrapper/KnowhereResource.h similarity index 100% rename from cpp/src/wrapper/knowhere/KnowhereResource.h rename to cpp/src/wrapper/KnowhereResource.h diff --git a/cpp/src/wrapper/knowhere/data_transfer.cpp b/cpp/src/wrapper/data_transfer.cpp similarity index 100% rename from cpp/src/wrapper/knowhere/data_transfer.cpp rename to cpp/src/wrapper/data_transfer.cpp diff --git a/cpp/src/wrapper/knowhere/data_transfer.h b/cpp/src/wrapper/data_transfer.h similarity index 100% rename from cpp/src/wrapper/knowhere/data_transfer.h rename to cpp/src/wrapper/data_transfer.h diff --git a/cpp/src/wrapper/gpu/Arithmetic.h b/cpp/src/wrapper/gpu/Arithmetic.h deleted file mode 100644 index c6ab7a2bc8..0000000000 --- a/cpp/src/wrapper/gpu/Arithmetic.h +++ /dev/null @@ -1,79 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include - - -namespace zilliz { -namespace milvus { -namespace engine { - -using Bool = int8_t; -using Byte = uint8_t; -using Word = unsigned long; -using EnumType = uint64_t; - -using Float32 = float; -using Float64 = double; - -constexpr bool kBoolMax = std::numeric_limits::max(); -constexpr bool kBoolMin = std::numeric_limits::lowest(); - -constexpr int8_t kInt8Max = std::numeric_limits::max(); -constexpr int8_t kInt8Min = std::numeric_limits::lowest(); - -constexpr int16_t kInt16Max = std::numeric_limits::max(); -constexpr int16_t kInt16Min = std::numeric_limits::lowest(); - -constexpr int32_t kInt32Max = std::numeric_limits::max(); -constexpr int32_t kInt32Min = std::numeric_limits::lowest(); - -constexpr int64_t kInt64Max = std::numeric_limits::max(); -constexpr int64_t kInt64Min = std::numeric_limits::lowest(); - -constexpr float kFloatMax = std::numeric_limits::max(); -constexpr float kFloatMin = std::numeric_limits::lowest(); - -constexpr double kDoubleMax = std::numeric_limits::max(); -constexpr double kDoubleMin = std::numeric_limits::lowest(); - -constexpr uint32_t kFloat32DecimalPrecision = std::numeric_limits::digits10; -constexpr uint32_t kFloat64DecimalPrecision = std::numeric_limits::digits10; - - -constexpr uint8_t kByteWidth = 8; -constexpr uint8_t kCharWidth = kByteWidth; -constexpr uint8_t kWordWidth = sizeof(Word) * kByteWidth; -constexpr uint8_t kEnumTypeWidth = sizeof(EnumType) * kByteWidth; - -template -inline size_t -WidthOf() { return sizeof(T) << 3; } - -template -inline size_t -WidthOf(const T &) { return sizeof(T) << 3; } - - -} -} // namespace lib -} // namespace zilliz diff --git a/cpp/src/wrapper/gpu/Topk.cu b/cpp/src/wrapper/gpu/Topk.cu deleted file mode 100644 index 423c8806c4..0000000000 --- a/cpp/src/wrapper/gpu/Topk.cu +++ /dev/null @@ -1,586 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -#include "faiss/FaissAssert.h" -#include "faiss/gpu/utils/Limits.cuh" -#include "Arithmetic.h" - - -namespace faiss { -namespace gpu { - -constexpr bool kBoolMax = zilliz::milvus::engine::kBoolMax; -constexpr bool kBoolMin = zilliz::milvus::engine::kBoolMin; - -template<> -struct Limits { - static __device__ __host__ - inline bool getMin() { - return kBoolMin; - } - static __device__ __host__ - inline bool getMax() { - return kBoolMax; - } -}; - -constexpr int8_t kInt8Max = zilliz::milvus::engine::kInt8Max; -constexpr int8_t kInt8Min = zilliz::milvus::engine::kInt8Min; - -template<> -struct Limits { - static __device__ __host__ - inline int8_t getMin() { - return kInt8Min; - } - static __device__ __host__ - inline int8_t getMax() { - return kInt8Max; - } -}; - -constexpr int16_t kInt16Max = zilliz::milvus::engine::kInt16Max; -constexpr int16_t kInt16Min = zilliz::milvus::engine::kInt16Min; - -template<> -struct Limits { - static __device__ __host__ - inline int16_t getMin() { - return kInt16Min; - } - static __device__ __host__ - inline int16_t getMax() { - return kInt16Max; - } -}; - -constexpr int64_t kInt64Max = zilliz::milvus::engine::kInt64Max; -constexpr int64_t kInt64Min = zilliz::milvus::engine::kInt64Min; - -template<> -struct Limits { - static __device__ __host__ - inline int64_t getMin() { - return kInt64Min; - } - static __device__ __host__ - inline int64_t getMax() { - return kInt64Max; - } -}; - -constexpr double kDoubleMax = zilliz::milvus::engine::kDoubleMax; -constexpr double kDoubleMin = zilliz::milvus::engine::kDoubleMin; - -template<> -struct Limits { - static __device__ __host__ - inline double getMin() { - return kDoubleMin; - } - static __device__ __host__ - inline double getMax() { - return kDoubleMax; - } -}; - -} -} - -#include "faiss/gpu/utils/DeviceUtils.h" -#include "faiss/gpu/utils/MathOperators.cuh" -#include "faiss/gpu/utils/Pair.cuh" -#include "faiss/gpu/utils/Reductions.cuh" -#include "faiss/gpu/utils/Select.cuh" -#include "faiss/gpu/utils/Tensor.cuh" -#include "faiss/gpu/utils/StaticUtils.h" - -#include "Topk.h" - - -namespace zilliz { -namespace milvus { -namespace engine { -namespace gpu { - -constexpr int kWarpSize = 32; - -template -using Tensor = faiss::gpu::Tensor; - -template -using Pair = faiss::gpu::Pair; - - -// select kernel for k == 1 -template -__global__ void topkSelectMin1(Tensor productDistances, - Tensor outDistances, - Tensor outIndices) { - // Each block handles kRowsPerBlock rows of the distances (results) - Pair threadMin[kRowsPerBlock]; - __shared__ - Pair blockMin[kRowsPerBlock * (kBlockSize / kWarpSize)]; - - T distance[kRowsPerBlock]; - -#pragma unroll - for (int i = 0; i < kRowsPerBlock; ++i) { - threadMin[i].k = faiss::gpu::Limits::getMax(); - threadMin[i].v = -1; - } - - // blockIdx.x: which chunk of rows we are responsible for updating - int rowStart = blockIdx.x * kRowsPerBlock; - - // FIXME: if we have exact multiples, don't need this - bool endRow = (blockIdx.x == gridDim.x - 1); - - if (endRow) { - if (productDistances.getSize(0) % kRowsPerBlock == 0) { - endRow = false; - } - } - - if (endRow) { - for (int row = rowStart; row < productDistances.getSize(0); ++row) { - for (int col = threadIdx.x; col < productDistances.getSize(1); - col += blockDim.x) { - distance[0] = productDistances[row][col]; - - if (faiss::gpu::Math::lt(distance[0], threadMin[0].k)) { - threadMin[0].k = distance[0]; - threadMin[0].v = col; - } - } - - // Reduce within the block - threadMin[0] = - faiss::gpu::blockReduceAll, faiss::gpu::Min >, false, false>( - threadMin[0], faiss::gpu::Min >(), blockMin); - - if (threadIdx.x == 0) { - outDistances[row][0] = threadMin[0].k; - outIndices[row][0] = threadMin[0].v; - } - - // so we can use the shared memory again - __syncthreads(); - - threadMin[0].k = faiss::gpu::Limits::getMax(); - threadMin[0].v = -1; - } - } else { - for (int col = threadIdx.x; col < productDistances.getSize(1); - col += blockDim.x) { - -#pragma unroll - for (int row = 0; row < kRowsPerBlock; ++row) { - distance[row] = productDistances[rowStart + row][col]; - } - -#pragma unroll - for (int row = 0; row < kRowsPerBlock; ++row) { - if (faiss::gpu::Math::lt(distance[row], threadMin[row].k)) { - threadMin[row].k = distance[row]; - threadMin[row].v = col; - } - } - } - - // Reduce within the block - faiss::gpu::blockReduceAll, faiss::gpu::Min >, false, false>( - threadMin, faiss::gpu::Min >(), blockMin); - - if (threadIdx.x == 0) { -#pragma unroll - for (int row = 0; row < kRowsPerBlock; ++row) { - outDistances[rowStart + row][0] = threadMin[row].k; - outIndices[rowStart + row][0] = threadMin[row].v; - } - } - } -} - -// L2 + select kernel for k > 1, no re-use of ||c||^2 -template -__global__ void topkSelectMinK(Tensor productDistances, - Tensor outDistances, - Tensor outIndices, - int k, T initK) { - // Each block handles a single row of the distances (results) - constexpr int kNumWarps = ThreadsPerBlock / kWarpSize; - - __shared__ - T smemK[kNumWarps * NumWarpQ]; - __shared__ - int64_t smemV[kNumWarps * NumWarpQ]; - - faiss::gpu::BlockSelect, - NumWarpQ, NumThreadQ, ThreadsPerBlock> - heap(initK, -1, smemK, smemV, k); - - int row = blockIdx.x; - - // Whole warps must participate in the selection - int limit = faiss::gpu::utils::roundDown(productDistances.getSize(1), kWarpSize); - int i = threadIdx.x; - - for (; i < limit; i += blockDim.x) { - T v = productDistances[row][i]; - heap.add(v, i); - } - - if (i < productDistances.getSize(1)) { - T v = productDistances[row][i]; - heap.addThreadQ(v, i); - } - - heap.reduce(); - for (int i = threadIdx.x; i < k; i += blockDim.x) { - outDistances[row][i] = smemK[i]; - outIndices[row][i] = smemV[i]; - } -} - -// FIXME: no TVec specialization -template -void runTopKSelectMin(Tensor &productDistances, - Tensor &outDistances, - Tensor &outIndices, - int k, - cudaStream_t stream) { - FAISS_ASSERT(productDistances.getSize(0) == outDistances.getSize(0)); - FAISS_ASSERT(productDistances.getSize(0) == outIndices.getSize(0)); - FAISS_ASSERT(outDistances.getSize(1) == k); - FAISS_ASSERT(outIndices.getSize(1) == k); - FAISS_ASSERT(k <= 1024); - - if (k == 1) { - constexpr int kThreadsPerBlock = 256; - constexpr int kRowsPerBlock = 8; - - auto block = dim3(kThreadsPerBlock); - auto grid = dim3(faiss::gpu::utils::divUp(outDistances.getSize(0), kRowsPerBlock)); - - topkSelectMin1 - << < grid, block, 0, stream >> > (productDistances, outDistances, outIndices); - } else { - constexpr int kThreadsPerBlock = 128; - - auto block = dim3(kThreadsPerBlock); - auto grid = dim3(outDistances.getSize(0)); - -#define RUN_TOPK_SELECT_MIN(NUM_WARP_Q, NUM_THREAD_Q) \ - do { \ - topkSelectMinK \ - <<>>(productDistances, \ - outDistances, outIndices, \ - k, faiss::gpu::Limits::getMax()); \ - } while (0) - - if (k <= 32) { - RUN_TOPK_SELECT_MIN(32, 2); - } else if (k <= 64) { - RUN_TOPK_SELECT_MIN(64, 3); - } else if (k <= 128) { - RUN_TOPK_SELECT_MIN(128, 3); - } else if (k <= 256) { - RUN_TOPK_SELECT_MIN(256, 4); - } else if (k <= 512) { - RUN_TOPK_SELECT_MIN(512, 8); - } else if (k <= 1024) { - RUN_TOPK_SELECT_MIN(1024, 8); - } else { - FAISS_ASSERT(false); - } - } - - CUDA_TEST_ERROR(); -} - -//////////////////////////////////////////////////////////// -// select kernel for k == 1 -template -__global__ void topkSelectMax1(Tensor productDistances, - Tensor outDistances, - Tensor outIndices) { - // Each block handles kRowsPerBlock rows of the distances (results) - Pair threadMax[kRowsPerBlock]; - __shared__ - Pair blockMax[kRowsPerBlock * (kBlockSize / kWarpSize)]; - - T distance[kRowsPerBlock]; - -#pragma unroll - for (int i = 0; i < kRowsPerBlock; ++i) { - threadMax[i].k = faiss::gpu::Limits::getMin(); - threadMax[i].v = -1; - } - - // blockIdx.x: which chunk of rows we are responsible for updating - int rowStart = blockIdx.x * kRowsPerBlock; - - // FIXME: if we have exact multiples, don't need this - bool endRow = (blockIdx.x == gridDim.x - 1); - - if (endRow) { - if (productDistances.getSize(0) % kRowsPerBlock == 0) { - endRow = false; - } - } - - if (endRow) { - for (int row = rowStart; row < productDistances.getSize(0); ++row) { - for (int col = threadIdx.x; col < productDistances.getSize(1); - col += blockDim.x) { - distance[0] = productDistances[row][col]; - - if (faiss::gpu::Math::gt(distance[0], threadMax[0].k)) { - threadMax[0].k = distance[0]; - threadMax[0].v = col; - } - } - - // Reduce within the block - threadMax[0] = - faiss::gpu::blockReduceAll, faiss::gpu::Max >, false, false>( - threadMax[0], faiss::gpu::Max >(), blockMax); - - if (threadIdx.x == 0) { - outDistances[row][0] = threadMax[0].k; - outIndices[row][0] = threadMax[0].v; - } - - // so we can use the shared memory again - __syncthreads(); - - threadMax[0].k = faiss::gpu::Limits::getMin(); - threadMax[0].v = -1; - } - } else { - for (int col = threadIdx.x; col < productDistances.getSize(1); - col += blockDim.x) { - -#pragma unroll - for (int row = 0; row < kRowsPerBlock; ++row) { - distance[row] = productDistances[rowStart + row][col]; - } - -#pragma unroll - for (int row = 0; row < kRowsPerBlock; ++row) { - if (faiss::gpu::Math::gt(distance[row], threadMax[row].k)) { - threadMax[row].k = distance[row]; - threadMax[row].v = col; - } - } - } - - // Reduce within the block - faiss::gpu::blockReduceAll, faiss::gpu::Max >, false, false>( - threadMax, faiss::gpu::Max >(), blockMax); - - if (threadIdx.x == 0) { -#pragma unroll - for (int row = 0; row < kRowsPerBlock; ++row) { - outDistances[rowStart + row][0] = threadMax[row].k; - outIndices[rowStart + row][0] = threadMax[row].v; - } - } - } -} - -// L2 + select kernel for k > 1, no re-use of ||c||^2 -template -__global__ void topkSelectMaxK(Tensor productDistances, - Tensor outDistances, - Tensor outIndices, - int k, T initK) { - // Each block handles a single row of the distances (results) - constexpr int kNumWarps = ThreadsPerBlock / kWarpSize; - - __shared__ - T smemK[kNumWarps * NumWarpQ]; - __shared__ - int64_t smemV[kNumWarps * NumWarpQ]; - - faiss::gpu::BlockSelect, - NumWarpQ, NumThreadQ, ThreadsPerBlock> - heap(initK, -1, smemK, smemV, k); - - int row = blockIdx.x; - - // Whole warps must participate in the selection - int limit = faiss::gpu::utils::roundDown(productDistances.getSize(1), kWarpSize); - int i = threadIdx.x; - - for (; i < limit; i += blockDim.x) { - T v = productDistances[row][i]; - heap.add(v, i); - } - - if (i < productDistances.getSize(1)) { - T v = productDistances[row][i]; - heap.addThreadQ(v, i); - } - - heap.reduce(); - for (int i = threadIdx.x; i < k; i += blockDim.x) { - outDistances[row][i] = smemK[i]; - outIndices[row][i] = smemV[i]; - } -} - -// FIXME: no TVec specialization -template -void runTopKSelectMax(Tensor &productDistances, - Tensor &outDistances, - Tensor &outIndices, - int k, - cudaStream_t stream) { - FAISS_ASSERT(productDistances.getSize(0) == outDistances.getSize(0)); - FAISS_ASSERT(productDistances.getSize(0) == outIndices.getSize(0)); - FAISS_ASSERT(outDistances.getSize(1) == k); - FAISS_ASSERT(outIndices.getSize(1) == k); - FAISS_ASSERT(k <= 1024); - - if (k == 1) { - constexpr int kThreadsPerBlock = 256; - constexpr int kRowsPerBlock = 8; - - auto block = dim3(kThreadsPerBlock); - auto grid = dim3(faiss::gpu::utils::divUp(outDistances.getSize(0), kRowsPerBlock)); - - topkSelectMax1 - << < grid, block, 0, stream >> > (productDistances, outDistances, outIndices); - } else { - constexpr int kThreadsPerBlock = 128; - - auto block = dim3(kThreadsPerBlock); - auto grid = dim3(outDistances.getSize(0)); - -#define RUN_TOPK_SELECT_MAX(NUM_WARP_Q, NUM_THREAD_Q) \ - do { \ - topkSelectMaxK \ - <<>>(productDistances, \ - outDistances, outIndices, \ - k, faiss::gpu::Limits::getMin()); \ - } while (0) - - if (k <= 32) { - RUN_TOPK_SELECT_MAX(32, 2); - } else if (k <= 64) { - RUN_TOPK_SELECT_MAX(64, 3); - } else if (k <= 128) { - RUN_TOPK_SELECT_MAX(128, 3); - } else if (k <= 256) { - RUN_TOPK_SELECT_MAX(256, 4); - } else if (k <= 512) { - RUN_TOPK_SELECT_MAX(512, 8); - } else if (k <= 1024) { - RUN_TOPK_SELECT_MAX(1024, 8); - } else { - FAISS_ASSERT(false); - } - } - - CUDA_TEST_ERROR(); -} -////////////////////////////////////////////////////////////// - -template -void runTopKSelect(Tensor &productDistances, - Tensor &outDistances, - Tensor &outIndices, - bool dir, - int k, - cudaStream_t stream) { - if (dir) { - runTopKSelectMax(productDistances, - outDistances, - outIndices, - k, - stream); - } else { - runTopKSelectMin(productDistances, - outDistances, - outIndices, - k, - stream); - } -} - -template -void TopK(T *input, - int length, - int k, - T *output, - int64_t *idx, -// Ordering order_flag, - cudaStream_t stream) { - -// bool dir = (order_flag == Ordering::kAscending ? false : true); - bool dir = 0; - - Tensor t_input(input, {1, length}); - Tensor t_output(output, {1, k}); - Tensor t_idx(idx, {1, k}); - - runTopKSelect(t_input, t_output, t_idx, dir, k, stream); -} - -//INSTANTIATION_TOPK_2(bool); -//INSTANTIATION_TOPK_2(int8_t); -//INSTANTIATION_TOPK_2(int16_t); -INSTANTIATION_TOPK_2(int32_t); -//INSTANTIATION_TOPK_2(int64_t); -INSTANTIATION_TOPK_2(float); -//INSTANTIATION_TOPK_2(double); -//INSTANTIATION_TOPK(TimeInterval); -//INSTANTIATION_TOPK(Float128); -//INSTANTIATION_TOPK(char); - -} - -void TopK(float *host_input, - int length, - int k, - float *output, - int64_t *indices) { - float *device_input, *device_output; - int64_t *ids; - - cudaMalloc((void **) &device_input, sizeof(float) * length); - cudaMalloc((void **) &device_output, sizeof(float) * k); - cudaMalloc((void **) &ids, sizeof(int64_t) * k); - - cudaMemcpy(device_input, host_input, sizeof(float) * length, cudaMemcpyHostToDevice); - - gpu::TopK(device_input, length, k, device_output, ids, nullptr); - - cudaMemcpy(output, device_output, sizeof(float) * k, cudaMemcpyDeviceToHost); - cudaMemcpy(indices, ids, sizeof(int64_t) * k, cudaMemcpyDeviceToHost); - - cudaFree(device_input); - cudaFree(device_output); - cudaFree(ids); -} - -} -} -} diff --git a/cpp/src/wrapper/gpu/Topk.h b/cpp/src/wrapper/gpu/Topk.h deleted file mode 100644 index d8dc69fd09..0000000000 --- a/cpp/src/wrapper/gpu/Topk.h +++ /dev/null @@ -1,73 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -#pragma once - -#include -#include - - -namespace zilliz { -namespace milvus { -namespace engine { -namespace gpu { - -template -void -TopK(T *input, - int length, - int k, - T *output, - int64_t *indices, -// Ordering order_flag, - cudaStream_t stream = nullptr); - - -#define INSTANTIATION_TOPK_2(T) \ - template void \ - TopK(T *input, \ - int length, \ - int k, \ - T *output, \ - int64_t *indices, \ - cudaStream_t stream) -// Ordering order_flag, \ -// cudaStream_t stream) - -//extern INSTANTIATION_TOPK_2(int8_t); -//extern INSTANTIATION_TOPK_2(int16_t); -extern INSTANTIATION_TOPK_2(int32_t); -//extern INSTANTIATION_TOPK_2(int64_t); -extern INSTANTIATION_TOPK_2(float); -//extern INSTANTIATION_TOPK_2(double); -//extern INSTANTIATION_TOPK(TimeInterval); -//extern INSTANTIATION_TOPK(Float128); - -} - -// User Interface. -void TopK(float *input, - int length, - int k, - float *output, - int64_t *indices); - - -} -} -} diff --git a/cpp/src/wrapper/knowhere/vec_impl.cpp b/cpp/src/wrapper/vec_impl.cpp similarity index 100% rename from cpp/src/wrapper/knowhere/vec_impl.cpp rename to cpp/src/wrapper/vec_impl.cpp diff --git a/cpp/src/wrapper/knowhere/vec_impl.h b/cpp/src/wrapper/vec_impl.h similarity index 100% rename from cpp/src/wrapper/knowhere/vec_impl.h rename to cpp/src/wrapper/vec_impl.h diff --git a/cpp/src/wrapper/knowhere/vec_index.cpp b/cpp/src/wrapper/vec_index.cpp similarity index 100% rename from cpp/src/wrapper/knowhere/vec_index.cpp rename to cpp/src/wrapper/vec_index.cpp diff --git a/cpp/src/wrapper/knowhere/vec_index.h b/cpp/src/wrapper/vec_index.h similarity index 100% rename from cpp/src/wrapper/knowhere/vec_index.h rename to cpp/src/wrapper/vec_index.h diff --git a/cpp/unittest/knowhere/CMakeLists.txt b/cpp/unittest/knowhere/CMakeLists.txt index 27e2612ae7..21faf3568c 100644 --- a/cpp/unittest/knowhere/CMakeLists.txt +++ b/cpp/unittest/knowhere/CMakeLists.txt @@ -21,9 +21,9 @@ include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") set(knowhere_src - ${MILVUS_ENGINE_SRC}/wrapper/knowhere/data_transfer.cpp - ${MILVUS_ENGINE_SRC}/wrapper/knowhere/vec_impl.cpp - ${MILVUS_ENGINE_SRC}/wrapper/knowhere/vec_index.cpp) + ${MILVUS_ENGINE_SRC}/wrapper/data_transfer.cpp + ${MILVUS_ENGINE_SRC}/wrapper/vec_impl.cpp + ${MILVUS_ENGINE_SRC}/wrapper/vec_index.cpp) set(helper utils.cpp diff --git a/cpp/unittest/knowhere/knowhere_test.cpp b/cpp/unittest/knowhere/knowhere_test.cpp index 6d8f57f322..ac185efbb3 100644 --- a/cpp/unittest/knowhere/knowhere_test.cpp +++ b/cpp/unittest/knowhere/knowhere_test.cpp @@ -19,7 +19,7 @@ #include #include "utils/easylogging++.h" -#include +#include "src/wrapper/vec_index.h" #include "knowhere/index/vector_index/gpu_ivf.h" #include "utils.h" diff --git a/cpp/unittest/scheduler/scheduler_test.cpp b/cpp/unittest/scheduler/scheduler_test.cpp index 6f680f4a60..13bdf53fb0 100644 --- a/cpp/unittest/scheduler/scheduler_test.cpp +++ b/cpp/unittest/scheduler/scheduler_test.cpp @@ -25,7 +25,7 @@ #include "scheduler/ResourceFactory.h" #include "scheduler/resource/Resource.h" #include "utils/Error.h" -#include "wrapper/knowhere/vec_index.h" +#include "src/wrapper/vec_index.h" #include "scheduler/tasklabel/SpecResLabel.h" diff --git a/cpp/unittest/server/cache_test.cpp b/cpp/unittest/server/cache_test.cpp index 5bb1fb8612..bf9b6edf7d 100644 --- a/cpp/unittest/server/cache_test.cpp +++ b/cpp/unittest/server/cache_test.cpp @@ -21,7 +21,7 @@ #include "server/ServerConfig.h" #include "utils/Error.h" -#include "wrapper/knowhere/vec_index.h" +#include "src/wrapper/vec_index.h" using namespace zilliz::milvus; From 4305c8305f97a0a59dd105f4fc8acf7c9a708424 Mon Sep 17 00:00:00 2001 From: jinhai Date: Wed, 18 Sep 2019 17:47:04 +0800 Subject: [PATCH 034/217] MS-567 - Add NOTICE.md Former-commit-id: dba7a33adab252b6bfe853dfc1ddedf357073716 --- cpp/CHANGELOG.md | 1 + cpp/NOTICE.md | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 cpp/NOTICE.md diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 4408778a04..886b383cd8 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -19,6 +19,7 @@ Please mark all change in change log and use the ticket from JIRA. ## Task - MS-554 - Change license to Apache 2.0 - MS-561 - Add contributing guidelines, code of conduct and README docs +- MS-567 - Add NOTICE.md # Milvus 0.4.0 (2019-09-12) diff --git a/cpp/NOTICE.md b/cpp/NOTICE.md new file mode 100644 index 0000000000..970dd054c0 --- /dev/null +++ b/cpp/NOTICE.md @@ -0,0 +1,29 @@ + + + + + + +| Name | License | +| ------------- | ------------------------------------------------------------ | +| Apache Arrow | Apache License 2.0 | +| Boost | Boost Software License | +| BZIP2 | BSD-like license | +| FAISS | MIT | +| Gtest | BSD 3-Clause | +| LAPACK | BSD 3-Clause | +| LZ4 | [BSD 2-Clause](https://github.com/Blosc/c-blosc/blob/master/LICENSES/LZ4.txt) | +| MySQLPP | LGPL 2.1 | +| OpenBLAS | BSD 3-Clause | +| Prometheus | Apache License 2.0 | +| Snappy | [BSD](https://github.com/Blosc/c-blosc/blob/master/LICENSES/SNAPPY.txt) | +| SQLite | [Public Domain](https://www.sqlite.org/copyright.html) | +| SQLite-ORM | BSD 3-Clause | +| yaml-cpp | MIT | +| ZLIB | [zlib](http://zlib.net/zlib_license.html) | +| ZSTD | [BSD](https://github.com/facebook/zstd/blob/dev/LICENSE) | +| libunwind | MIT | +| gperftools | BSD 3-Clause | +| gRPC | Apache 2.0 | +| EASYLOGGINGPP | MIT | + From e72b0348f76d6525a764d14f25ec54629f449ecf Mon Sep 17 00:00:00 2001 From: wxyu Date: Wed, 18 Sep 2019 19:44:38 +0800 Subject: [PATCH 035/217] MS-555 Remove old scheduler Former-commit-id: 8eb7f773a15df78f46bf61a59ec0ea9d8923cc28 --- cpp/CHANGELOG.md | 1 + cpp/src/db/DBImpl.cpp | 73 +++++---- cpp/src/db/DBImpl.h | 1 - cpp/src/db/scheduler/TaskDispatchQueue.cpp | 102 ------------- cpp/src/db/scheduler/TaskDispatchQueue.h | 68 --------- cpp/src/db/scheduler/TaskDispatchStrategy.cpp | 143 ------------------ cpp/src/db/scheduler/TaskDispatchStrategy.h | 36 ----- cpp/src/db/scheduler/TaskScheduler.cpp | 135 ----------------- cpp/src/db/scheduler/TaskScheduler.h | 63 -------- .../db/scheduler/context/DeleteContext.cpp | 52 ------- cpp/src/db/scheduler/context/DeleteContext.h | 52 ------- .../db/scheduler/context/IScheduleContext.h | 50 ------ .../db/scheduler/context/SearchContext.cpp | 70 --------- cpp/src/db/scheduler/context/SearchContext.h | 94 ------------ cpp/src/db/scheduler/task/DeleteTask.cpp | 37 ----- cpp/src/db/scheduler/task/DeleteTask.h | 41 ----- cpp/src/db/scheduler/task/IScheduleTask.h | 53 ------- cpp/src/db/scheduler/task/IndexLoadTask.cpp | 41 ----- cpp/src/db/scheduler/task/IndexLoadTask.h | 42 ----- cpp/src/db/scheduler/task/SearchTask.cpp | 39 ----- cpp/src/db/scheduler/task/SearchTask.h | 46 ------ cpp/src/scheduler/JobMgr.h | 2 + cpp/src/scheduler/SchedInst.cpp | 7 +- cpp/src/scheduler/SchedInst.h | 19 +++ cpp/src/scheduler/TaskCreator.cpp | 10 +- cpp/src/scheduler/job/Job.h | 1 + cpp/src/scheduler/job/SearchJob.cpp | 12 +- cpp/src/scheduler/job/SearchJob.h | 4 + cpp/src/scheduler/task/DeleteTask.cpp | 6 +- cpp/src/scheduler/task/DeleteTask.h | 6 +- cpp/src/scheduler/task/SearchTask.cpp | 69 +++++---- cpp/src/scheduler/task/SearchTask.h | 16 +- cpp/src/scheduler/task/Task.h | 7 +- cpp/src/scheduler/task/TaskConvert.cpp | 53 ------- cpp/src/scheduler/task/TaskConvert.h | 35 ----- cpp/src/scheduler/task/TestTask.cpp | 2 +- cpp/src/scheduler/task/TestTask.h | 3 +- cpp/unittest/db/CMakeLists.txt | 10 +- cpp/unittest/db/scheduler_test.cpp | 137 ----------------- cpp/unittest/db/search_test.cpp | 39 +++-- cpp/unittest/db/utils.cpp | 7 +- cpp/unittest/scheduler/CMakeLists.txt | 2 + cpp/unittest/scheduler/normal_test.cpp | 2 +- cpp/unittest/scheduler/resource_mgr_test.cpp | 2 +- cpp/unittest/scheduler/resource_test.cpp | 8 +- cpp/unittest/scheduler/scheduler_test.cpp | 6 +- cpp/unittest/scheduler/tasktable_test.cpp | 4 +- 47 files changed, 184 insertions(+), 1524 deletions(-) delete mode 100644 cpp/src/db/scheduler/TaskDispatchQueue.cpp delete mode 100644 cpp/src/db/scheduler/TaskDispatchQueue.h delete mode 100644 cpp/src/db/scheduler/TaskDispatchStrategy.cpp delete mode 100644 cpp/src/db/scheduler/TaskDispatchStrategy.h delete mode 100644 cpp/src/db/scheduler/TaskScheduler.cpp delete mode 100644 cpp/src/db/scheduler/TaskScheduler.h delete mode 100644 cpp/src/db/scheduler/context/DeleteContext.cpp delete mode 100644 cpp/src/db/scheduler/context/DeleteContext.h delete mode 100644 cpp/src/db/scheduler/context/IScheduleContext.h delete mode 100644 cpp/src/db/scheduler/context/SearchContext.cpp delete mode 100644 cpp/src/db/scheduler/context/SearchContext.h delete mode 100644 cpp/src/db/scheduler/task/DeleteTask.cpp delete mode 100644 cpp/src/db/scheduler/task/DeleteTask.h delete mode 100644 cpp/src/db/scheduler/task/IScheduleTask.h delete mode 100644 cpp/src/db/scheduler/task/IndexLoadTask.cpp delete mode 100644 cpp/src/db/scheduler/task/IndexLoadTask.h delete mode 100644 cpp/src/db/scheduler/task/SearchTask.cpp delete mode 100644 cpp/src/db/scheduler/task/SearchTask.h delete mode 100644 cpp/src/scheduler/task/TaskConvert.cpp delete mode 100644 cpp/src/scheduler/task/TaskConvert.h delete mode 100644 cpp/unittest/db/scheduler_test.cpp diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index c5d27e3461..76ef4a98a7 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -13,6 +13,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-556 - Add Job Definition in Scheduler - MS-558 - Refine status code - MS-562 - Add JobMgr and TaskCreator in Scheduler +- MS-555 - Remove old scheduler ## New Feature diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 648038de25..51bad594b2 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -24,8 +24,8 @@ #include "meta/MetaFactory.h" #include "meta/MetaConsts.h" #include "metrics/Metrics.h" -#include "scheduler/TaskScheduler.h" -#include "scheduler/context/DeleteContext.h" +#include "scheduler/job/SearchJob.h" +#include "scheduler/job/DeleteJob.h" #include "scheduler/SchedInst.h" #include "utils/TimeRecorder.h" #include "utils/Log.h" @@ -133,12 +133,10 @@ Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& date meta_ptr_->DeleteTable(table_id); //soft delete table //scheduler will determine when to delete table files - TaskScheduler& scheduler = TaskScheduler::GetInstance(); - DeleteContextPtr context = std::make_shared(table_id, - meta_ptr_, - ResMgrInst::GetInstance()->GetNumOfComputeResource()); - scheduler.Schedule(context); - context->WaitAndDelete(); + auto nres = ResMgrInst::GetInstance()->GetNumOfComputeResource(); + scheduler::DeleteJobPtr job = std::make_shared(0, table_id, meta_ptr_, nres); + JobMgrInst::GetInstance()->Put(job); + job->WaitAndDelete(); } else { meta_ptr_->DropPartitionsByDates(table_id, dates); } @@ -418,51 +416,50 @@ Status DBImpl::Size(uint64_t& result) { Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) { + using namespace scheduler; server::CollectQueryMetrics metrics(nq); TimeRecorder rc(""); //step 1: get files to search ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size() << " date range count: " << dates.size(); - SearchContextPtr context = std::make_shared(k, nq, nprobe, vectors); - for (auto &file : files) { + SearchJobPtr job = std::make_shared(0, k, nq, nprobe, vectors); + for (auto &file : files) { TableFileSchemaPtr file_ptr = std::make_shared(file); - context->AddIndexFile(file_ptr); + job->AddIndexFile(file_ptr); } //step 2: put search task to scheduler - TaskScheduler& scheduler = TaskScheduler::GetInstance(); - scheduler.Schedule(context); - - context->WaitResult(); - if (!context->GetStatus().ok()) { - return context->GetStatus(); + JobMgrInst::GetInstance()->Put(job); + job->WaitResult(); + if (!job->GetStatus().ok()) { + return job->GetStatus(); } //step 3: print time cost information - double load_cost = context->LoadCost(); - double search_cost = context->SearchCost(); - double reduce_cost = context->ReduceCost(); - std::string load_info = TimeRecorder::GetTimeSpanStr(load_cost); - std::string search_info = TimeRecorder::GetTimeSpanStr(search_cost); - std::string reduce_info = TimeRecorder::GetTimeSpanStr(reduce_cost); - if(search_cost > 0.0 || reduce_cost > 0.0) { - double total_cost = load_cost + search_cost + reduce_cost; - double load_percent = load_cost/total_cost; - double search_percent = search_cost/total_cost; - double reduce_percent = reduce_cost/total_cost; - - ENGINE_LOG_DEBUG << "Engine load index totally cost: " << load_info << " percent: " << load_percent*100 << "%"; - ENGINE_LOG_DEBUG << "Engine search index totally cost: " << search_info << " percent: " << search_percent*100 << "%"; - ENGINE_LOG_DEBUG << "Engine reduce topk totally cost: " << reduce_info << " percent: " << reduce_percent*100 << "%"; - } else { - ENGINE_LOG_DEBUG << "Engine load cost: " << load_info - << " search cost: " << search_info - << " reduce cost: " << reduce_info; - } +// double load_cost = context->LoadCost(); +// double search_cost = context->SearchCost(); +// double reduce_cost = context->ReduceCost(); +// std::string load_info = TimeRecorder::GetTimeSpanStr(load_cost); +// std::string search_info = TimeRecorder::GetTimeSpanStr(search_cost); +// std::string reduce_info = TimeRecorder::GetTimeSpanStr(reduce_cost); +// if(search_cost > 0.0 || reduce_cost > 0.0) { +// double total_cost = load_cost + search_cost + reduce_cost; +// double load_percent = load_cost/total_cost; +// double search_percent = search_cost/total_cost; +// double reduce_percent = reduce_cost/total_cost; +// +// ENGINE_LOG_DEBUG << "Engine load index totally cost: " << load_info << " percent: " << load_percent*100 << "%"; +// ENGINE_LOG_DEBUG << "Engine search index totally cost: " << search_info << " percent: " << search_percent*100 << "%"; +// ENGINE_LOG_DEBUG << "Engine reduce topk totally cost: " << reduce_info << " percent: " << reduce_percent*100 << "%"; +// } else { +// ENGINE_LOG_DEBUG << "Engine load cost: " << load_info +// << " search cost: " << search_info +// << " reduce cost: " << reduce_info; +// } //step 4: construct results - results = context->GetResult(); + results = job->GetResult(); rc.ElapseFromBegin("Engine query totally cost"); return Status::OK(); diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 85803b763f..6062c5e2db 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -29,7 +29,6 @@ #include #include #include -#include "scheduler/context/SearchContext.h" namespace zilliz { diff --git a/cpp/src/db/scheduler/TaskDispatchQueue.cpp b/cpp/src/db/scheduler/TaskDispatchQueue.cpp deleted file mode 100644 index 41a95060a8..0000000000 --- a/cpp/src/db/scheduler/TaskDispatchQueue.cpp +++ /dev/null @@ -1,102 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -#include "TaskDispatchQueue.h" -#include "TaskDispatchStrategy.h" -#include "utils/Error.h" -#include "utils/Log.h" - -namespace zilliz { -namespace milvus { -namespace engine { - -void -TaskDispatchQueue::Put(const ScheduleContextPtr &context) { - std::unique_lock lock(mtx); - full_.wait(lock, [this] { return (queue_.size() < capacity_); }); - - if(context == nullptr) { - queue_.push_front(nullptr); - empty_.notify_all(); - return; - } - - TaskDispatchStrategy::Schedule(context, queue_); - - empty_.notify_all(); -} - -ScheduleTaskPtr -TaskDispatchQueue::Take() { - std::unique_lock lock(mtx); - empty_.wait(lock, [this] { return !queue_.empty(); }); - - ScheduleTaskPtr front(queue_.front()); - queue_.pop_front(); - full_.notify_all(); - return front; -} - -size_t -TaskDispatchQueue::Size() { - std::lock_guard lock(mtx); - return queue_.size(); -} - -ScheduleTaskPtr -TaskDispatchQueue::Front() { - std::unique_lock lock(mtx); - empty_.wait(lock, [this] { return !queue_.empty(); }); - if (queue_.empty()) { - std::string error_msg = "blocking queue empty"; - SERVER_LOG_ERROR << error_msg; - throw server::ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg); - } - ScheduleTaskPtr front(queue_.front()); - return front; -} - -ScheduleTaskPtr -TaskDispatchQueue::Back() { - std::unique_lock lock(mtx); - empty_.wait(lock, [this] { return !queue_.empty(); }); - - if (queue_.empty()) { - std::string error_msg = "blocking queue empty"; - SERVER_LOG_ERROR << error_msg; - throw server::ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg); - } - - ScheduleTaskPtr back(queue_.back()); - return back; -} - -bool -TaskDispatchQueue::Empty() { - std::unique_lock lock(mtx); - return queue_.empty(); -} - -void -TaskDispatchQueue::SetCapacity(const size_t capacity) { - capacity_ = (capacity > 0 ? capacity : capacity_); -} - -} -} -} \ No newline at end of file diff --git a/cpp/src/db/scheduler/TaskDispatchQueue.h b/cpp/src/db/scheduler/TaskDispatchQueue.h deleted file mode 100644 index b8a0dd611e..0000000000 --- a/cpp/src/db/scheduler/TaskDispatchQueue.h +++ /dev/null @@ -1,68 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "context/IScheduleContext.h" -#include "task/IScheduleTask.h" - -#include -#include -#include -#include - - -namespace zilliz { -namespace milvus { -namespace engine { - -class TaskDispatchQueue { -public: - TaskDispatchQueue() : mtx(), full_(), empty_() {} - - TaskDispatchQueue(const TaskDispatchQueue &rhs) = delete; - - TaskDispatchQueue &operator=(const TaskDispatchQueue &rhs) = delete; - - using TaskList = std::list; - - void Put(const ScheduleContextPtr &context); - - ScheduleTaskPtr Take(); - - ScheduleTaskPtr Front(); - - ScheduleTaskPtr Back(); - - size_t Size(); - - bool Empty(); - - void SetCapacity(const size_t capacity); - -private: - mutable std::mutex mtx; - std::condition_variable full_; - std::condition_variable empty_; - - TaskList queue_; - size_t capacity_ = 1000000; -}; - -} -} -} diff --git a/cpp/src/db/scheduler/TaskDispatchStrategy.cpp b/cpp/src/db/scheduler/TaskDispatchStrategy.cpp deleted file mode 100644 index 6ce9e582de..0000000000 --- a/cpp/src/db/scheduler/TaskDispatchStrategy.cpp +++ /dev/null @@ -1,143 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "TaskDispatchStrategy.h" -#include "context/SearchContext.h" -#include "context/DeleteContext.h" -#include "task/IndexLoadTask.h" -#include "task/DeleteTask.h" -#include "cache/CpuCacheMgr.h" -#include "utils/Error.h" -#include "utils/Log.h" - -namespace zilliz { -namespace milvus { -namespace engine { - -class ReuseCacheIndexStrategy { -public: - bool Schedule(const SearchContextPtr &context, std::list& task_list) { - if(context == nullptr) { - ENGINE_LOG_ERROR << "Task Dispatch context doesn't exist"; - return false; - } - - SearchContext::Id2IndexMap index_files = context->GetIndexMap(); - //some index loader alread exists - for(auto& task : task_list) { - if(task->type() != ScheduleTaskType::kIndexLoad) { - continue; - } - - IndexLoadTaskPtr loader = std::static_pointer_cast(task); - if(index_files.find(loader->file_->id_) != index_files.end()){ - ENGINE_LOG_DEBUG << "Append SearchContext to exist IndexLoaderContext"; - index_files.erase(loader->file_->id_); - loader->search_contexts_.push_back(context); - } - } - - //index_files still contains some index files, create new loader - for(auto& pair : index_files) { - ENGINE_LOG_DEBUG << "Create new IndexLoaderContext for: " << pair.second->location_; - IndexLoadTaskPtr new_loader = std::make_shared(); - new_loader->search_contexts_.push_back(context); - new_loader->file_ = pair.second; - - auto index = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(pair.second->location_); - if(index != nullptr) { - //if the index file has been in memory, increase its priority - task_list.push_front(new_loader); - } else { - //index file not in memory, put it to tail - task_list.push_back(new_loader); - } - } - - return true; - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class DeleteTableStrategy { -public: - bool Schedule(const DeleteContextPtr &context, std::list &task_list) { - if (context == nullptr) { - ENGINE_LOG_ERROR << "Task Dispatch context doesn't exist"; - return false; - } - - DeleteTaskPtr delete_task = std::make_shared(context); - if(task_list.empty()) { - task_list.push_back(delete_task); - return true; - } - - std::string table_id = context->table_id(); - - //put delete task to proper position - //for example: task_list has 10 IndexLoadTask, only the No.5 IndexLoadTask is for table1 - //if user want to delete table1, the DeleteTask will be insert into No.6 position - for(std::list::reverse_iterator iter = task_list.rbegin(); iter != task_list.rend(); ++iter) { - if((*iter)->type() != ScheduleTaskType::kIndexLoad) { - continue; - } - - IndexLoadTaskPtr loader = std::static_pointer_cast(*iter); - if(loader->file_->table_id_ != table_id) { - continue; - } - - task_list.insert(iter.base(), delete_task); - return true; - } - - //no task is searching this table, put DeleteTask to front of list so that the table will be delete asap - task_list.push_front(delete_task); - return true; - } -}; - - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -bool TaskDispatchStrategy::Schedule(const ScheduleContextPtr &context_ptr, - std::list &task_list) { - if(context_ptr == nullptr) { - ENGINE_LOG_ERROR << "Task Dispatch context doesn't exist"; - return false; - } - - switch(context_ptr->type()) { - case ScheduleContextType::kSearch: { - SearchContextPtr search_context = std::static_pointer_cast(context_ptr); - ReuseCacheIndexStrategy strategy; - return strategy.Schedule(search_context, task_list); - } - case ScheduleContextType::kDelete: { - DeleteContextPtr delete_context = std::static_pointer_cast(context_ptr); - DeleteTableStrategy strategy; - return strategy.Schedule(delete_context, task_list); - } - default: - ENGINE_LOG_ERROR << "Invalid schedule task type"; - return false; - } -} - -} -} -} \ No newline at end of file diff --git a/cpp/src/db/scheduler/TaskDispatchStrategy.h b/cpp/src/db/scheduler/TaskDispatchStrategy.h deleted file mode 100644 index 8d1b51605e..0000000000 --- a/cpp/src/db/scheduler/TaskDispatchStrategy.h +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "context/IScheduleContext.h" -#include "task/IScheduleTask.h" - -#include - -namespace zilliz { -namespace milvus { -namespace engine { - -class TaskDispatchStrategy { -public: - static bool Schedule(const ScheduleContextPtr &context_ptr, std::list& task_list); -}; - -} -} -} diff --git a/cpp/src/db/scheduler/TaskScheduler.cpp b/cpp/src/db/scheduler/TaskScheduler.cpp deleted file mode 100644 index a546dbabf1..0000000000 --- a/cpp/src/db/scheduler/TaskScheduler.cpp +++ /dev/null @@ -1,135 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -#include "server/ServerConfig.h" -#include "TaskScheduler.h" -#include "TaskDispatchQueue.h" -#include "utils/Log.h" -#include "utils/TimeRecorder.h" -#include "db/engine/EngineFactory.h" -#include "scheduler/task/TaskConvert.h" -#include "scheduler/SchedInst.h" -#include "scheduler/ResourceFactory.h" - -namespace zilliz { -namespace milvus { -namespace engine { - -TaskScheduler::TaskScheduler() - : stopped_(true) { - Start(); -} - -TaskScheduler::~TaskScheduler() { - Stop(); -} - -TaskScheduler& TaskScheduler::GetInstance() { - static TaskScheduler s_instance; - return s_instance; -} - -bool -TaskScheduler::Start() { - if(!stopped_) { - SERVER_LOG_INFO << "Task Scheduler isn't started"; - return true; - } - - stopped_ = false; - - task_queue_.SetCapacity(2); - - task_dispatch_thread_ = std::make_shared(&TaskScheduler::TaskDispatchWorker, this); - task_thread_ = std::make_shared(&TaskScheduler::TaskWorker, this); - - return true; -} - -bool -TaskScheduler::Stop() { - if(stopped_) { - SERVER_LOG_INFO << "Task Scheduler already stopped"; - return true; - } - - if(task_dispatch_thread_) { - task_dispatch_queue_.Put(nullptr); - task_dispatch_thread_->join(); - task_dispatch_thread_ = nullptr; - } - - if(task_thread_) { - task_queue_.Put(nullptr); - task_thread_->join(); - task_thread_ = nullptr; - } - - stopped_ = true; - - return true; -} - -bool -TaskScheduler::Schedule(ScheduleContextPtr context) { - task_dispatch_queue_.Put(context); - - return true; -} - -bool -TaskScheduler::TaskDispatchWorker() { - while(true) { - ScheduleTaskPtr task_ptr = task_dispatch_queue_.Take(); - if(task_ptr == nullptr) { - SERVER_LOG_INFO << "Stop db task dispatch thread"; - return true; - } - - // TODO: Put task into Disk-TaskTable - auto task = TaskConvert(task_ptr); - auto disk_list = ResMgrInst::GetInstance()->GetDiskResources(); - if (!disk_list.empty()) { - if (auto disk = disk_list[0].lock()) { - disk->task_table().Put(task); - } - } - } -} - -bool -TaskScheduler::TaskWorker() { - while(true) { - // TODO: expected blocking forever - ScheduleTaskPtr task_ptr = task_queue_.Take(); - if(task_ptr == nullptr) { - SERVER_LOG_INFO << "Stop db task worker thread"; - return true; - } - - //execute task - ScheduleTaskPtr next_task = task_ptr->Execute(); - if(next_task != nullptr) { - task_queue_.Put(next_task); - } - } -} - -} -} -} diff --git a/cpp/src/db/scheduler/TaskScheduler.h b/cpp/src/db/scheduler/TaskScheduler.h deleted file mode 100644 index 86feb37f8f..0000000000 --- a/cpp/src/db/scheduler/TaskScheduler.h +++ /dev/null @@ -1,63 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "context/IScheduleContext.h" -#include "task/IScheduleTask.h" -#include "TaskDispatchQueue.h" -#include "utils/BlockingQueue.h" - -#include - -namespace zilliz { -namespace milvus { -namespace engine { - -class TaskScheduler { -private: - TaskScheduler(); - virtual ~TaskScheduler(); - -public: - static TaskScheduler& GetInstance(); - - bool Schedule(ScheduleContextPtr context); - -private: - bool Start(); - bool Stop(); - - bool TaskDispatchWorker(); - bool TaskWorker(); - -private: - std::shared_ptr task_dispatch_thread_; - std::shared_ptr task_thread_; - - TaskDispatchQueue task_dispatch_queue_; - - using TaskQueue = server::BlockingQueue; - TaskQueue task_queue_; - - bool stopped_ = true; -}; - - -} -} -} diff --git a/cpp/src/db/scheduler/context/DeleteContext.cpp b/cpp/src/db/scheduler/context/DeleteContext.cpp deleted file mode 100644 index 32ec2f8f09..0000000000 --- a/cpp/src/db/scheduler/context/DeleteContext.cpp +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -#include "DeleteContext.h" - - -namespace zilliz { -namespace milvus { -namespace engine { - -DeleteContext::DeleteContext(const std::string &table_id, meta::MetaPtr &meta_ptr, uint64_t num_resource) - : IScheduleContext(ScheduleContextType::kDelete), - table_id_(table_id), - meta_ptr_(meta_ptr), - num_resource_(num_resource) { - -} - -void DeleteContext::WaitAndDelete() { -#ifdef NEW_SCHEDULER - std::unique_lock lock(mutex_); - cv_.wait(lock, [&] { return done_resource == num_resource_; }); - meta_ptr_->DeleteTableFiles(table_id_); -#endif -} - -void DeleteContext::ResourceDone() { - { - std::lock_guard lock(mutex_); - ++done_resource; - } - cv_.notify_one(); -} - -} -} -} \ No newline at end of file diff --git a/cpp/src/db/scheduler/context/DeleteContext.h b/cpp/src/db/scheduler/context/DeleteContext.h deleted file mode 100644 index 7132acc827..0000000000 --- a/cpp/src/db/scheduler/context/DeleteContext.h +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "IScheduleContext.h" -#include "db/meta/Meta.h" -#include -#include - -namespace zilliz { -namespace milvus { -namespace engine { - -class DeleteContext : public IScheduleContext { -public: - DeleteContext(const std::string& table_id, meta::MetaPtr& meta_ptr, uint64_t num_resource); - - std::string table_id() const { return table_id_; } - meta::MetaPtr meta() const { return meta_ptr_; } - void WaitAndDelete(); - void ResourceDone(); - -private: - std::string table_id_; - meta::MetaPtr meta_ptr_; - - uint64_t num_resource_; - uint64_t done_resource = 0; - std::mutex mutex_; - std::condition_variable cv_; -}; - -using DeleteContextPtr = std::shared_ptr; - -} -} -} diff --git a/cpp/src/db/scheduler/context/IScheduleContext.h b/cpp/src/db/scheduler/context/IScheduleContext.h deleted file mode 100644 index 1ac347526c..0000000000 --- a/cpp/src/db/scheduler/context/IScheduleContext.h +++ /dev/null @@ -1,50 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -namespace zilliz { -namespace milvus { -namespace engine { - -enum class ScheduleContextType { - kUnknown = 0, - kSearch, - kDelete, -}; - -class IScheduleContext { -public: - IScheduleContext(ScheduleContextType type) - : type_(type) { - } - - virtual ~IScheduleContext() = default; - - ScheduleContextType type() const { return type_; } - -protected: - ScheduleContextType type_; -}; - -using ScheduleContextPtr = std::shared_ptr; - -} -} -} diff --git a/cpp/src/db/scheduler/context/SearchContext.cpp b/cpp/src/db/scheduler/context/SearchContext.cpp deleted file mode 100644 index ae9ced5938..0000000000 --- a/cpp/src/db/scheduler/context/SearchContext.cpp +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -#include "SearchContext.h" -#include "utils/Log.h" - -#include - -namespace zilliz { -namespace milvus { -namespace engine { - -SearchContext::SearchContext(uint64_t topk, uint64_t nq, uint64_t nprobe, const float* vectors) - : IScheduleContext(ScheduleContextType::kSearch), - topk_(topk), - nq_(nq), - nprobe_(nprobe), - vectors_(vectors) { - //use current time to identify this context - std::chrono::system_clock::time_point tp = std::chrono::system_clock::now(); - long id = tp.time_since_epoch().count(); - identity_ = std::to_string(id); -} - -bool -SearchContext::AddIndexFile(TableFileSchemaPtr& index_file) { - std::unique_lock lock(mtx_); - if(index_file == nullptr || map_index_files_.find(index_file->id_) != map_index_files_.end()) { - return false; - } - - SERVER_LOG_DEBUG << "SearchContext " << identity_ << " add index file: " << index_file->id_; - - map_index_files_[index_file->id_] = index_file; - return true; -} - -void -SearchContext::IndexSearchDone(size_t index_id) { - std::unique_lock lock(mtx_); - map_index_files_.erase(index_id); - done_cond_.notify_all(); - SERVER_LOG_DEBUG << "SearchContext " << identity_ << " finish index file: " << index_id; -} - -void -SearchContext::WaitResult() { - std::unique_lock lock(mtx_); - done_cond_.wait(lock, [this] { return map_index_files_.empty(); }); - SERVER_LOG_DEBUG << "SearchContext " << identity_ << " all done"; -} - -} -} -} \ No newline at end of file diff --git a/cpp/src/db/scheduler/context/SearchContext.h b/cpp/src/db/scheduler/context/SearchContext.h deleted file mode 100644 index 3aee2c436e..0000000000 --- a/cpp/src/db/scheduler/context/SearchContext.h +++ /dev/null @@ -1,94 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "IScheduleContext.h" -#include "db/meta/MetaTypes.h" - -#include -#include -#include -#include - -namespace zilliz { -namespace milvus { -namespace engine { - -using TableFileSchemaPtr = std::shared_ptr; - -class SearchContext : public IScheduleContext { -public: - SearchContext(uint64_t topk, uint64_t nq, uint64_t nprobe, const float* vectors); - - bool AddIndexFile(TableFileSchemaPtr& index_file); - - uint64_t topk() const { return topk_; } - uint64_t nq() const { return nq_; } - uint64_t nprobe() const { return nprobe_; } - const float* vectors() const { return vectors_; } - - using Id2IndexMap = std::unordered_map; - const Id2IndexMap& GetIndexMap() const { return map_index_files_; } - - using Id2DistanceMap = std::vector>; - using ResultSet = std::vector; - const ResultSet& GetResult() const { return result_; } - ResultSet& GetResult() { return result_; } - - const std::string& Identity() const { return identity_; } - const Status& GetStatus() const { return status_; } - Status& GetStatus() { return status_; } - - void IndexSearchDone(size_t index_id); - void WaitResult(); - - void AccumLoadCost(double span) { time_cost_load_ += span; } - void AccumSearchCost(double span) { time_cost_search_ += span; } - void AccumReduceCost(double span) { time_cost_reduce_ += span; } - - double LoadCost() const { return time_cost_load_; } - double SearchCost() const { return time_cost_search_; } - double ReduceCost() const { return time_cost_reduce_; } - -private: - uint64_t topk_ = 0; - uint64_t nq_ = 0; - uint64_t nprobe_ = 10; - const float* vectors_ = nullptr; - - Id2IndexMap map_index_files_; - ResultSet result_; - - std::mutex mtx_; - std::condition_variable done_cond_; - - std::string identity_; //for debug - Status status_; - - double time_cost_load_ = 0.0; //time cost for load all index files, unit: us - double time_cost_search_ = 0.0; //time cost for entire search, unit: us - double time_cost_reduce_ = 0.0; //time cost for entire reduce, unit: us -}; - -using SearchContextPtr = std::shared_ptr; - - - -} -} -} diff --git a/cpp/src/db/scheduler/task/DeleteTask.cpp b/cpp/src/db/scheduler/task/DeleteTask.cpp deleted file mode 100644 index 310b844ff9..0000000000 --- a/cpp/src/db/scheduler/task/DeleteTask.cpp +++ /dev/null @@ -1,37 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -#include "DeleteTask.h" - -namespace zilliz { -namespace milvus { -namespace engine { - -DeleteTask::DeleteTask(const DeleteContextPtr& context) - : IScheduleTask(ScheduleTaskType::kDelete), - context_(context) { - -} - -std::shared_ptr DeleteTask::Execute() { - return nullptr; -} - -} -} -} \ No newline at end of file diff --git a/cpp/src/db/scheduler/task/DeleteTask.h b/cpp/src/db/scheduler/task/DeleteTask.h deleted file mode 100644 index 362b125c92..0000000000 --- a/cpp/src/db/scheduler/task/DeleteTask.h +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "IScheduleTask.h" -#include "db/scheduler/context/DeleteContext.h" - -namespace zilliz { -namespace milvus { -namespace engine { - -class DeleteTask : public IScheduleTask { -public: - DeleteTask(const DeleteContextPtr& context); - - virtual std::shared_ptr Execute() override; - -public: - DeleteContextPtr context_; -}; - -using DeleteTaskPtr = std::shared_ptr; - -} -} -} diff --git a/cpp/src/db/scheduler/task/IScheduleTask.h b/cpp/src/db/scheduler/task/IScheduleTask.h deleted file mode 100644 index 07025a7d72..0000000000 --- a/cpp/src/db/scheduler/task/IScheduleTask.h +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -namespace zilliz { -namespace milvus { -namespace engine { - -enum class ScheduleTaskType { - kUnknown = 0, - kIndexLoad, - kSearch, - kDelete, -}; - -class IScheduleTask { -public: - IScheduleTask(ScheduleTaskType type) - : type_(type) { - } - - virtual ~IScheduleTask() = default; - - ScheduleTaskType type() const { return type_; } - - virtual std::shared_ptr Execute() = 0; - -protected: - ScheduleTaskType type_; -}; - -using ScheduleTaskPtr = std::shared_ptr; - -} -} -} diff --git a/cpp/src/db/scheduler/task/IndexLoadTask.cpp b/cpp/src/db/scheduler/task/IndexLoadTask.cpp deleted file mode 100644 index be83b86683..0000000000 --- a/cpp/src/db/scheduler/task/IndexLoadTask.cpp +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -#include "IndexLoadTask.h" -#include "SearchTask.h" -#include "db/engine/EngineFactory.h" -#include "utils/Log.h" -#include "utils/TimeRecorder.h" -#include "metrics/Metrics.h" - -namespace zilliz { -namespace milvus { -namespace engine { - -IndexLoadTask::IndexLoadTask() - : IScheduleTask(ScheduleTaskType::kIndexLoad) { - -} - -std::shared_ptr IndexLoadTask::Execute() { - return nullptr; -} - -} -} -} \ No newline at end of file diff --git a/cpp/src/db/scheduler/task/IndexLoadTask.h b/cpp/src/db/scheduler/task/IndexLoadTask.h deleted file mode 100644 index 29356e5f58..0000000000 --- a/cpp/src/db/scheduler/task/IndexLoadTask.h +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "IScheduleTask.h" -#include "db/scheduler/context/SearchContext.h" - -namespace zilliz { -namespace milvus { -namespace engine { - -class IndexLoadTask : public IScheduleTask { -public: - IndexLoadTask(); - - virtual std::shared_ptr Execute() override; - -public: - TableFileSchemaPtr file_; - std::vector search_contexts_; -}; - -using IndexLoadTaskPtr = std::shared_ptr; - -} -} -} diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp deleted file mode 100644 index d0962ec101..0000000000 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "SearchTask.h" -#include "metrics/Metrics.h" -#include "utils/Log.h" -#include "utils/TimeRecorder.h" - -#include - -namespace zilliz { -namespace milvus { -namespace engine { - -SearchTask::SearchTask() -: IScheduleTask(ScheduleTaskType::kSearch) { -} - -std::shared_ptr SearchTask::Execute() { - return nullptr; -} - -} -} -} diff --git a/cpp/src/db/scheduler/task/SearchTask.h b/cpp/src/db/scheduler/task/SearchTask.h deleted file mode 100644 index 47c4813fd2..0000000000 --- a/cpp/src/db/scheduler/task/SearchTask.h +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "IScheduleTask.h" -#include "db/scheduler/context/SearchContext.h" -#include "db/engine/ExecutionEngine.h" - -namespace zilliz { -namespace milvus { -namespace engine { - -class SearchTask : public IScheduleTask { -public: - SearchTask(); - - virtual std::shared_ptr Execute() override; - -public: - size_t index_id_ = 0; - int file_type_ = 0; //for metrics - ExecutionEnginePtr index_engine_; - std::vector search_contexts_; -}; - -using SearchTaskPtr = std::shared_ptr; - - -} -} -} \ No newline at end of file diff --git a/cpp/src/scheduler/JobMgr.h b/cpp/src/scheduler/JobMgr.h index f096ab6121..1d8720c4b4 100644 --- a/cpp/src/scheduler/JobMgr.h +++ b/cpp/src/scheduler/JobMgr.h @@ -73,6 +73,8 @@ private: ResourceMgrPtr res_mgr_ = nullptr; }; +using JobMgrPtr = std::shared_ptr; + } } } diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index 5432227523..424bc9f778 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -33,6 +33,9 @@ std::mutex ResMgrInst::mutex_; SchedulerPtr SchedInst::instance = nullptr; std::mutex SchedInst::mutex_; +scheduler::JobMgrPtr JobMgrInst::instance = nullptr; +std::mutex JobMgrInst::mutex_; + void load_simple_config() { server::ConfigNode &config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_RESOURCE); @@ -151,12 +154,14 @@ StartSchedulerService() { // load_advance_config(); ResMgrInst::GetInstance()->Start(); SchedInst::GetInstance()->Start(); + JobMgrInst::GetInstance()->Start(); } void StopSchedulerService() { - ResMgrInst::GetInstance()->Stop(); + JobMgrInst::GetInstance()->Stop(); SchedInst::GetInstance()->Stop(); + ResMgrInst::GetInstance()->Stop(); } } } diff --git a/cpp/src/scheduler/SchedInst.h b/cpp/src/scheduler/SchedInst.h index fe73aae362..010a7e84f5 100644 --- a/cpp/src/scheduler/SchedInst.h +++ b/cpp/src/scheduler/SchedInst.h @@ -19,6 +19,7 @@ #include "ResourceMgr.h" #include "Scheduler.h" +#include "JobMgr.h" #include #include @@ -64,6 +65,24 @@ private: static std::mutex mutex_; }; +class JobMgrInst { +public: + static scheduler::JobMgrPtr + GetInstance() { + if (instance == nullptr) { + std::lock_guard lock(mutex_); + if (instance == nullptr) { + instance = std::make_shared(ResMgrInst::GetInstance()); + } + } + return instance; + } + +private: + static scheduler::JobMgrPtr instance; + static std::mutex mutex_; +}; + void StartSchedulerService(); diff --git a/cpp/src/scheduler/TaskCreator.cpp b/cpp/src/scheduler/TaskCreator.cpp index afe0d9d868..1d471135d6 100644 --- a/cpp/src/scheduler/TaskCreator.cpp +++ b/cpp/src/scheduler/TaskCreator.cpp @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. +#include #include "TaskCreator.h" +#include "tasklabel/DefaultLabel.h" namespace zilliz { @@ -43,6 +45,8 @@ TaskCreator::Create(const SearchJobPtr &job) { std::vector tasks; for (auto &index_file : job->index_files()) { auto task = std::make_shared(index_file.second); + task->label() = std::make_shared(); + task->job_ = job; tasks.emplace_back(task); } @@ -52,8 +56,10 @@ TaskCreator::Create(const SearchJobPtr &job) { std::vector TaskCreator::Create(const DeleteJobPtr &job) { std::vector tasks; -// auto task = std::make_shared(job); -// tasks.emplace_back(task); + auto task = std::make_shared(job); + task->label() = std::make_shared(); + task->job_ = job; + tasks.emplace_back(task); return tasks; } diff --git a/cpp/src/scheduler/job/Job.h b/cpp/src/scheduler/job/Job.h index 845a5dc165..242c33a4e5 100644 --- a/cpp/src/scheduler/job/Job.h +++ b/cpp/src/scheduler/job/Job.h @@ -62,6 +62,7 @@ private: }; using JobPtr = std::shared_ptr; +using JobWPtr = std::weak_ptr; } } diff --git a/cpp/src/scheduler/job/SearchJob.cpp b/cpp/src/scheduler/job/SearchJob.cpp index 786c1fb7b5..65b6701b5f 100644 --- a/cpp/src/scheduler/job/SearchJob.cpp +++ b/cpp/src/scheduler/job/SearchJob.cpp @@ -28,7 +28,12 @@ SearchJob::SearchJob(zilliz::milvus::scheduler::JobId id, uint64_t topk, uint64_t nq, uint64_t nprobe, - const float *vectors) : Job(id, JobType::SEARCH) {} + const float *vectors) + : Job(id, JobType::SEARCH), + topk_(topk), + nq_(nq), + nprobe_(nprobe), + vectors_(vectors) {} bool SearchJob::AddIndexFile(const TableFileSchemaPtr &index_file) { @@ -64,6 +69,11 @@ SearchJob::GetResult() { return result_; } +Status& +SearchJob::GetStatus() { + return status_; +} + } } diff --git a/cpp/src/scheduler/job/SearchJob.h b/cpp/src/scheduler/job/SearchJob.h index ed6531767c..12edeab199 100644 --- a/cpp/src/scheduler/job/SearchJob.h +++ b/cpp/src/scheduler/job/SearchJob.h @@ -59,6 +59,9 @@ public: ResultSet & GetResult(); + Status & + GetStatus(); + public: uint64_t topk() const { @@ -94,6 +97,7 @@ private: Id2IndexMap index_files_; // TODO: column-base better ? ResultSet result_; + Status status_; std::mutex mutex_; std::condition_variable cv_; diff --git a/cpp/src/scheduler/task/DeleteTask.cpp b/cpp/src/scheduler/task/DeleteTask.cpp index ded1ad0462..b1b1697f0e 100644 --- a/cpp/src/scheduler/task/DeleteTask.cpp +++ b/cpp/src/scheduler/task/DeleteTask.cpp @@ -23,8 +23,8 @@ namespace zilliz { namespace milvus { namespace engine { -XDeleteTask::XDeleteTask(DeleteContextPtr &delete_context) - : Task(TaskType::DeleteTask), delete_context_ptr_(delete_context) {} +XDeleteTask::XDeleteTask(const scheduler::DeleteJobPtr &delete_job) + : Task(TaskType::DeleteTask), delete_job_(delete_job) {} void XDeleteTask::Load(LoadType type, uint8_t device_id) { @@ -33,7 +33,7 @@ XDeleteTask::Load(LoadType type, uint8_t device_id) { void XDeleteTask::Execute() { - delete_context_ptr_->ResourceDone(); + delete_job_->ResourceDone(); } } diff --git a/cpp/src/scheduler/task/DeleteTask.h b/cpp/src/scheduler/task/DeleteTask.h index b49107cc2a..f21a1a96e4 100644 --- a/cpp/src/scheduler/task/DeleteTask.h +++ b/cpp/src/scheduler/task/DeleteTask.h @@ -17,7 +17,7 @@ #pragma once -#include +#include "scheduler/job/DeleteJob.h" #include "Task.h" @@ -28,7 +28,7 @@ namespace engine { class XDeleteTask : public Task { public: explicit - XDeleteTask(DeleteContextPtr &delete_context); + XDeleteTask(const scheduler::DeleteJobPtr &job); void Load(LoadType type, uint8_t device_id) override; @@ -37,7 +37,7 @@ public: Execute() override; public: - DeleteContextPtr delete_context_ptr_; + scheduler::DeleteJobPtr delete_job_; }; } diff --git a/cpp/src/scheduler/task/SearchTask.cpp b/cpp/src/scheduler/task/SearchTask.cpp index d8222da39f..e39cd00d2e 100644 --- a/cpp/src/scheduler/task/SearchTask.cpp +++ b/cpp/src/scheduler/task/SearchTask.cpp @@ -22,6 +22,7 @@ #include "utils/Log.h" #include +#include "scheduler/job/SearchJob.h" namespace zilliz { @@ -94,7 +95,7 @@ CollectFileMetrics(int file_type, size_t file_size) { } } -XSearchTask::XSearchTask(TableFileSchemaPtr file) +XSearchTask::XSearchTask(meta::TableFileSchemaPtr file) : Task(TaskType::SearchTask), file_(file) { if (file_) { index_engine_ = EngineFactory::Build(file_->dimension_, @@ -143,9 +144,10 @@ XSearchTask::Load(LoadType type, uint8_t device_id) { s = Status(SERVER_UNEXPECTED_ERROR, error_msg); } - for (auto &context : search_contexts_) { - context->IndexSearchDone(file_->id_);//mark as done avoid dead lock, even failed - context->GetStatus() = s; + if (auto job = job_.lock()){ + auto search_job = std::static_pointer_cast(job); + search_job->SearchDone(file_->id_); + search_job->GetStatus() = s; } return; @@ -156,16 +158,16 @@ XSearchTask::Load(LoadType type, uint8_t device_id) { std::string info = "Load file id:" + std::to_string(file_->id_) + " file type:" + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) + " bytes from location: " + file_->location_ + " totally cost"; double span = rc.ElapseFromBegin(info); - for (auto &context : search_contexts_) { - context->AccumLoadCost(span); - } +// for (auto &context : search_contexts_) { +// context->AccumLoadCost(span); +// } CollectFileMetrics(file_->file_type_, file_size); //step 2: return search task for later execution index_id_ = file_->id_; index_type_ = file_->file_type_; - search_contexts_.swap(search_contexts_); +// search_contexts_.swap(search_contexts_); } void @@ -174,8 +176,8 @@ XSearchTask::Execute() { return; } - ENGINE_LOG_DEBUG << "Searching in file id:" << index_id_ << " with " - << search_contexts_.size() << " tasks"; +// ENGINE_LOG_DEBUG << "Searching in file id:" << index_id_ << " with " +// << search_contexts_.size() << " tasks"; TimeRecorder rc("DoSearch file id:" + std::to_string(index_id_)); @@ -183,16 +185,18 @@ XSearchTask::Execute() { std::vector output_ids; std::vector output_distance; - for (auto &context : search_contexts_) { + + if (auto job = job_.lock()) { + auto search_job = std::static_pointer_cast(job); //step 1: allocate memory - uint64_t nq = context->nq(); - uint64_t topk = context->topk(); - uint64_t nprobe = context->nprobe(); - const float* vectors = context->vectors(); + uint64_t nq = search_job->nq(); + uint64_t topk = search_job->topk(); + uint64_t nprobe = search_job->nprobe(); + const float* vectors = search_job->vectors(); output_ids.resize(topk * nq); output_distance.resize(topk * nq); - std::string hdr = "context " + context->Identity() + + std::string hdr = "job " + std::to_string(search_job->id()) + " nq " + std::to_string(nq) + " topk " + std::to_string(topk); @@ -201,30 +205,29 @@ XSearchTask::Execute() { index_engine_->Search(nq, vectors, topk, nprobe, output_distance.data(), output_ids.data()); double span = rc.RecordSection(hdr + ", do search"); - context->AccumSearchCost(span); +// search_job->AccumSearchCost(span); //step 3: cluster result - SearchContext::ResultSet result_set; + scheduler::ResultSet result_set; auto spec_k = index_engine_->Count() < topk ? index_engine_->Count() : topk; XSearchTask::ClusterResult(output_ids, output_distance, nq, spec_k, result_set); span = rc.RecordSection(hdr + ", cluster result"); - context->AccumReduceCost(span); +// search_job->AccumReduceCost(span); // step 4: pick up topk result - XSearchTask::TopkResult(result_set, topk, metric_l2, context->GetResult()); + XSearchTask::TopkResult(result_set, topk, metric_l2, search_job->GetResult()); span = rc.RecordSection(hdr + ", reduce topk"); - context->AccumReduceCost(span); +// search_job->AccumReduceCost(span); } catch (std::exception &ex) { ENGINE_LOG_ERROR << "SearchTask encounter exception: " << ex.what(); - context->IndexSearchDone(index_id_);//mark as done avoid dead lock, even search failed - continue; +// search_job->IndexSearchDone(index_id_);//mark as done avoid dead lock, even search failed } //step 5: notify to send result to client - context->IndexSearchDone(index_id_); + search_job->SearchDone(index_id_); } rc.ElapseFromBegin("totally cost"); @@ -237,7 +240,7 @@ Status XSearchTask::ClusterResult(const std::vector &output_ids, const std::vector &output_distance, uint64_t nq, uint64_t topk, - SearchContext::ResultSet &result_set) { + scheduler::ResultSet &result_set) { if (output_ids.size() < nq * topk || output_distance.size() < nq * topk) { std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) + " distance array size: " + std::to_string(output_distance.size()); @@ -250,7 +253,7 @@ Status XSearchTask::ClusterResult(const std::vector &output_ids, std::function reduce_worker = [&](size_t from_index, size_t to_index) { for (auto i = from_index; i < to_index; i++) { - SearchContext::Id2DistanceMap id_distance; + scheduler::Id2DistanceMap id_distance; id_distance.reserve(topk); for (auto k = 0; k < topk; k++) { uint64_t index = i * topk + k; @@ -272,8 +275,8 @@ Status XSearchTask::ClusterResult(const std::vector &output_ids, return Status::OK(); } -Status XSearchTask::MergeResult(SearchContext::Id2DistanceMap &distance_src, - SearchContext::Id2DistanceMap &distance_target, +Status XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, + scheduler::Id2DistanceMap &distance_target, uint64_t topk, bool ascending) { //Note: the score_src and score_target are already arranged by score in ascending order @@ -290,7 +293,7 @@ Status XSearchTask::MergeResult(SearchContext::Id2DistanceMap &distance_src, size_t src_count = distance_src.size(); size_t target_count = distance_target.size(); - SearchContext::Id2DistanceMap distance_merged; + scheduler::Id2DistanceMap distance_merged; distance_merged.reserve(topk); size_t src_index = 0, target_index = 0; while (true) { @@ -346,10 +349,10 @@ Status XSearchTask::MergeResult(SearchContext::Id2DistanceMap &distance_src, return Status::OK(); } -Status XSearchTask::TopkResult(SearchContext::ResultSet &result_src, +Status XSearchTask::TopkResult(scheduler::ResultSet &result_src, uint64_t topk, bool ascending, - SearchContext::ResultSet &result_target) { + scheduler::ResultSet &result_target) { if (result_target.empty()) { result_target.swap(result_src); return Status::OK(); @@ -363,8 +366,8 @@ Status XSearchTask::TopkResult(SearchContext::ResultSet &result_src, std::function ReduceWorker = [&](size_t from_index, size_t to_index) { for (size_t i = from_index; i < to_index; i++) { - SearchContext::Id2DistanceMap &score_src = result_src[i]; - SearchContext::Id2DistanceMap &score_target = result_target[i]; + scheduler::Id2DistanceMap &score_src = result_src[i]; + scheduler::Id2DistanceMap &score_target = result_target[i]; XSearchTask::MergeResult(score_src, score_target, topk, ascending); } }; diff --git a/cpp/src/scheduler/task/SearchTask.h b/cpp/src/scheduler/task/SearchTask.h index b92e5bd851..e54e742e24 100644 --- a/cpp/src/scheduler/task/SearchTask.h +++ b/cpp/src/scheduler/task/SearchTask.h @@ -18,6 +18,8 @@ #pragma once #include "Task.h" +#include "db/meta/MetaTypes.h" +#include "scheduler/job/SearchJob.h" namespace zilliz { @@ -28,7 +30,7 @@ namespace engine { class XSearchTask : public Task { public: explicit - XSearchTask(TableFileSchemaPtr file); + XSearchTask(meta::TableFileSchemaPtr file); void Load(LoadType type, uint8_t device_id) override; @@ -41,20 +43,20 @@ public: const std::vector &output_distence, uint64_t nq, uint64_t topk, - SearchContext::ResultSet &result_set); + scheduler::ResultSet &result_set); - static Status MergeResult(SearchContext::Id2DistanceMap &distance_src, - SearchContext::Id2DistanceMap &distance_target, + static Status MergeResult(scheduler::Id2DistanceMap &distance_src, + scheduler::Id2DistanceMap &distance_target, uint64_t topk, bool ascending); - static Status TopkResult(SearchContext::ResultSet &result_src, + static Status TopkResult(scheduler::ResultSet &result_src, uint64_t topk, bool ascending, - SearchContext::ResultSet &result_target); + scheduler::ResultSet &result_target); public: - TableFileSchemaPtr file_; + meta::TableFileSchemaPtr file_; size_t index_id_ = 0; int index_type_ = 0; diff --git a/cpp/src/scheduler/task/Task.h b/cpp/src/scheduler/task/Task.h index bfaa1fe68d..c4a2c1cb4c 100644 --- a/cpp/src/scheduler/task/Task.h +++ b/cpp/src/scheduler/task/Task.h @@ -17,9 +17,9 @@ #pragma once -#include "db/scheduler/context/SearchContext.h" -#include "db/scheduler/task/IScheduleTask.h" #include "scheduler/tasklabel/TaskLabel.h" +#include "scheduler/job/Job.h" +#include "utils/Status.h" #include "Path.h" #include @@ -84,7 +84,8 @@ public: public: Path task_path_; - std::vector search_contexts_; +// std::vector search_contexts_; + scheduler::JobWPtr job_; TaskType type_; TaskLabelPtr label_ = nullptr; }; diff --git a/cpp/src/scheduler/task/TaskConvert.cpp b/cpp/src/scheduler/task/TaskConvert.cpp deleted file mode 100644 index 6be6309093..0000000000 --- a/cpp/src/scheduler/task/TaskConvert.cpp +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -#include "TaskConvert.h" -#include "scheduler/tasklabel/DefaultLabel.h" -#include "scheduler/tasklabel/BroadcastLabel.h" - - -namespace zilliz { -namespace milvus { -namespace engine { - -TaskPtr -TaskConvert(const ScheduleTaskPtr &schedule_task) { - switch (schedule_task->type()) { - case ScheduleTaskType::kIndexLoad: { - auto load_task = std::static_pointer_cast(schedule_task); - auto task = std::make_shared(load_task->file_); - task->label() = std::make_shared(); - task->search_contexts_ = load_task->search_contexts_; - return task; - } - case ScheduleTaskType::kDelete: { - auto delete_task = std::static_pointer_cast(schedule_task); - auto task = std::make_shared(delete_task->context_); - task->label() = std::make_shared(); - return task; - } - default: { - // TODO: unexpected !!! - return nullptr; - } - } -} - -} -} -} diff --git a/cpp/src/scheduler/task/TaskConvert.h b/cpp/src/scheduler/task/TaskConvert.h deleted file mode 100644 index 2420c48b23..0000000000 --- a/cpp/src/scheduler/task/TaskConvert.h +++ /dev/null @@ -1,35 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -#include "db/scheduler/task/DeleteTask.h" -#include "db/scheduler/task/IndexLoadTask.h" -#include "Task.h" -#include "SearchTask.h" -#include "DeleteTask.h" - -namespace zilliz { -namespace milvus { -namespace engine { - - -TaskPtr -TaskConvert(const ScheduleTaskPtr &schedule_task); - -} -} -} diff --git a/cpp/src/scheduler/task/TestTask.cpp b/cpp/src/scheduler/task/TestTask.cpp index f1f9c29477..5d5ac976b5 100644 --- a/cpp/src/scheduler/task/TestTask.cpp +++ b/cpp/src/scheduler/task/TestTask.cpp @@ -25,7 +25,7 @@ namespace milvus { namespace engine { -TestTask::TestTask(TableFileSchemaPtr &file) : XSearchTask(file) {} +TestTask::TestTask(meta::TableFileSchemaPtr &file) : XSearchTask(file) {} void TestTask::Load(LoadType type, uint8_t device_id) { diff --git a/cpp/src/scheduler/task/TestTask.h b/cpp/src/scheduler/task/TestTask.h index 66fd864e1d..e489fc537f 100644 --- a/cpp/src/scheduler/task/TestTask.h +++ b/cpp/src/scheduler/task/TestTask.h @@ -26,7 +26,8 @@ namespace engine { class TestTask : public XSearchTask { public: - TestTask(TableFileSchemaPtr& file); + explicit + TestTask(meta::TableFileSchemaPtr& file); public: void diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 29440dcf1a..b64696fe50 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -26,6 +26,7 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/job scheduler_job_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs) @@ -37,14 +38,6 @@ set(util_files ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files) -set(db_scheduler_srcs - ${scheduler_files} - ${scheduler_context_files} - ${scheduler_task_files} - ) include_directories(/usr/local/cuda/include) link_directories("/usr/local/cuda/lib64") @@ -62,6 +55,7 @@ set(db_test_src ${wrapper_src} ${scheduler_action_srcs} ${scheduler_event_srcs} + ${scheduler_job_srcs} ${scheduler_resource_srcs} ${scheduler_task_srcs} ${scheduler_srcs} diff --git a/cpp/unittest/db/scheduler_test.cpp b/cpp/unittest/db/scheduler_test.cpp deleted file mode 100644 index e61741a8fb..0000000000 --- a/cpp/unittest/db/scheduler_test.cpp +++ /dev/null @@ -1,137 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include -#include "utils/easylogging++.h" -#include - -#include "db/scheduler/TaskScheduler.h" -#include "db/scheduler/TaskDispatchStrategy.h" -#include "db/scheduler/TaskDispatchQueue.h" -#include "db/scheduler/task/SearchTask.h" -#include "db/scheduler/task/DeleteTask.h" -#include "db/scheduler/task/IndexLoadTask.h" - -using namespace zilliz::milvus; - -namespace { - -engine::TableFileSchemaPtr CreateTabileFileStruct(size_t id, const std::string& table_id) { - auto file = std::make_shared(); - file->id_ = id; - file->table_id_ = table_id; - return file; -} - -} - -TEST(DBSchedulerTest, TASK_QUEUE_TEST) { - engine::TaskDispatchQueue queue; - queue.SetCapacity(1000); - queue.Put(nullptr); - ASSERT_EQ(queue.Size(), 1UL); - - auto ptr = queue.Take(); - ASSERT_EQ(ptr, nullptr); - ASSERT_TRUE(queue.Empty()); - - engine::SearchContextPtr context_ptr = std::make_shared(1, 1, 10, nullptr); - for(size_t i = 0; i < 10; i++) { - auto file = CreateTabileFileStruct(i, "tbl"); - context_ptr->AddIndexFile(file); - } - - queue.Put(context_ptr); - ASSERT_EQ(queue.Size(), 10); - - auto index_files = context_ptr->GetIndexMap(); - - ptr = queue.Front(); - ASSERT_EQ(ptr->type(), engine::ScheduleTaskType::kIndexLoad); - engine::IndexLoadTaskPtr load_task = std::static_pointer_cast(ptr); - ASSERT_EQ(load_task->file_->id_, index_files.begin()->first); - - ptr = queue.Back(); - ASSERT_EQ(ptr->type(), engine::ScheduleTaskType::kIndexLoad); - load_task->Execute(); -} - -TEST(DBSchedulerTest, SEARCH_SCHEDULER_TEST) { - std::list task_list; - bool ret = engine::TaskDispatchStrategy::Schedule(nullptr, task_list); - ASSERT_FALSE(ret); - - for(size_t i = 10; i < 30; i++) { - engine::IndexLoadTaskPtr task_ptr = std::make_shared(); - task_ptr->file_ = CreateTabileFileStruct(i, "tbl"); - task_list.push_back(task_ptr); - } - - engine::SearchContextPtr context_ptr = std::make_shared(1, 1, 10, nullptr); - for(size_t i = 0; i < 20; i++) { - auto file = CreateTabileFileStruct(i, "tbl"); - context_ptr->AddIndexFile(file); - } - - ret = engine::TaskDispatchStrategy::Schedule(context_ptr, task_list); - ASSERT_TRUE(ret); - ASSERT_EQ(task_list.size(), 30); -} - -TEST(DBSchedulerTest, DELETE_SCHEDULER_TEST) { - std::list task_list; - bool ret = engine::TaskDispatchStrategy::Schedule(nullptr, task_list); - ASSERT_FALSE(ret); - - const std::string table_id = "to_delete_table"; - for(size_t i = 0; i < 10; i++) { - engine::IndexLoadTaskPtr task_ptr = std::make_shared(); - task_ptr->file_ = CreateTabileFileStruct(i, table_id); - task_list.push_back(task_ptr); - } - - for(size_t i = 0; i < 10; i++) { - engine::IndexLoadTaskPtr task_ptr = std::make_shared(); - task_ptr->file_ = CreateTabileFileStruct(i, "other_table"); - task_list.push_back(task_ptr); - } - - engine::meta::MetaPtr meta_ptr; - engine::DeleteContextPtr context_ptr = std::make_shared(table_id, meta_ptr, 0); - ret = engine::TaskDispatchStrategy::Schedule(context_ptr, task_list); - ASSERT_TRUE(ret); - ASSERT_EQ(task_list.size(), 21); - - auto temp_list = task_list; - for(size_t i = 0; ; i++) { - engine::ScheduleTaskPtr task_ptr = temp_list.front(); - temp_list.pop_front(); - if(task_ptr->type() == engine::ScheduleTaskType::kDelete) { - ASSERT_EQ(i, 10); - break; - } - } - - context_ptr = std::make_shared("no_task_table", meta_ptr, 0); - ret = engine::TaskDispatchStrategy::Schedule(context_ptr, task_list); - ASSERT_TRUE(ret); - ASSERT_EQ(task_list.size(), 22); - - engine::ScheduleTaskPtr task_ptr = task_list.front(); - ASSERT_EQ(task_ptr->type(), engine::ScheduleTaskType::kDelete); -} diff --git a/cpp/unittest/db/search_test.cpp b/cpp/unittest/db/search_test.cpp index 93b71ab406..295650f2d2 100644 --- a/cpp/unittest/db/search_test.cpp +++ b/cpp/unittest/db/search_test.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -#include "db/scheduler/task/SearchTask.h" #include "server/ServerConfig.h" #include "utils/TimeRecorder.h" @@ -50,9 +49,9 @@ void BuildResult(uint64_t nq, } } -void CheckResult(const engine::SearchContext::Id2DistanceMap& src_1, - const engine::SearchContext::Id2DistanceMap& src_2, - const engine::SearchContext::Id2DistanceMap& target, +void CheckResult(const scheduler::Id2DistanceMap& src_1, + const scheduler::Id2DistanceMap& src_2, + const scheduler::Id2DistanceMap& target, bool ascending) { for(uint64_t i = 0; i < target.size() - 1; i++) { if(ascending) { @@ -81,7 +80,7 @@ void CheckResult(const engine::SearchContext::Id2DistanceMap& src_1, void CheckCluster(const std::vector& target_ids, const std::vector& target_distence, - const engine::SearchContext::ResultSet& src_result, + const scheduler::ResultSet& src_result, int64_t nq, int64_t topk) { ASSERT_EQ(src_result.size(), nq); @@ -98,7 +97,7 @@ void CheckCluster(const std::vector& target_ids, } } -void CheckTopkResult(const engine::SearchContext::ResultSet& src_result, +void CheckTopkResult(const scheduler::ResultSet& src_result, bool ascending, int64_t nq, int64_t topk) { @@ -127,7 +126,7 @@ TEST(DBSearchTest, TOPK_TEST) { bool ascending = true; std::vector target_ids; std::vector target_distence; - engine::SearchContext::ResultSet src_result; + scheduler::ResultSet src_result; auto status = engine::XSearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result); ASSERT_FALSE(status.ok()); ASSERT_TRUE(src_result.empty()); @@ -137,7 +136,7 @@ TEST(DBSearchTest, TOPK_TEST) { ASSERT_TRUE(status.ok()); ASSERT_EQ(src_result.size(), NQ); - engine::SearchContext::ResultSet target_result; + scheduler::ResultSet target_result; status = engine::XSearchTask::TopkResult(target_result, TOP_K, ascending, target_result); ASSERT_TRUE(status.ok()); @@ -179,7 +178,7 @@ TEST(DBSearchTest, MERGE_TEST) { std::vector target_distence; std::vector src_ids; std::vector src_distence; - engine::SearchContext::ResultSet src_result, target_result; + scheduler::ResultSet src_result, target_result; uint64_t src_count = 5, target_count = 8; BuildResult(1, src_count, ascending, src_ids, src_distence); @@ -190,8 +189,8 @@ TEST(DBSearchTest, MERGE_TEST) { ASSERT_TRUE(status.ok()); { - engine::SearchContext::Id2DistanceMap src = src_result[0]; - engine::SearchContext::Id2DistanceMap target = target_result[0]; + scheduler::Id2DistanceMap src = src_result[0]; + scheduler::Id2DistanceMap target = target_result[0]; status = engine::XSearchTask::MergeResult(src, target, 10, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), 10); @@ -199,8 +198,8 @@ TEST(DBSearchTest, MERGE_TEST) { } { - engine::SearchContext::Id2DistanceMap src = src_result[0]; - engine::SearchContext::Id2DistanceMap target; + scheduler::Id2DistanceMap src = src_result[0]; + scheduler::Id2DistanceMap target; status = engine::XSearchTask::MergeResult(src, target, 10, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), src_count); @@ -209,8 +208,8 @@ TEST(DBSearchTest, MERGE_TEST) { } { - engine::SearchContext::Id2DistanceMap src = src_result[0]; - engine::SearchContext::Id2DistanceMap target = target_result[0]; + scheduler::Id2DistanceMap src = src_result[0]; + scheduler::Id2DistanceMap target = target_result[0]; status = engine::XSearchTask::MergeResult(src, target, 30, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), src_count + target_count); @@ -218,8 +217,8 @@ TEST(DBSearchTest, MERGE_TEST) { } { - engine::SearchContext::Id2DistanceMap target = src_result[0]; - engine::SearchContext::Id2DistanceMap src = target_result[0]; + scheduler::Id2DistanceMap target = src_result[0]; + scheduler::Id2DistanceMap src = target_result[0]; status = engine::XSearchTask::MergeResult(src, target, 30, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), src_count + target_count); @@ -235,7 +234,7 @@ TEST(DBSearchTest, PARALLEL_CLUSTER_TEST) { bool ascending = true; std::vector target_ids; std::vector target_distence; - engine::SearchContext::ResultSet src_result; + scheduler::ResultSet src_result; auto DoCluster = [&](int64_t nq, int64_t topk) { TimeRecorder rc("DoCluster"); @@ -270,11 +269,11 @@ TEST(DBSearchTest, PARALLEL_TOPK_TEST) { std::vector target_ids; std::vector target_distence; - engine::SearchContext::ResultSet src_result; + scheduler::ResultSet src_result; std::vector insufficient_ids; std::vector insufficient_distence; - engine::SearchContext::ResultSet insufficient_result; + scheduler::ResultSet insufficient_result; auto DoTopk = [&](int64_t nq, int64_t topk,int64_t insufficient_topk, bool ascending) { src_result.clear(); diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 4bc9699e05..a956215e49 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -80,7 +80,7 @@ void DBTest::SetUp() { auto res_mgr = engine::ResMgrInst::GetInstance(); res_mgr->Clear(); res_mgr->Add(engine::ResourceFactory::Create("disk", "DISK", 0, true, false)); - res_mgr->Add(engine::ResourceFactory::Create("cpu", "CPU", 0, true, true)); + res_mgr->Add(engine::ResourceFactory::Create("cpu", "CPU", 0, true, false)); res_mgr->Add(engine::ResourceFactory::Create("gtx1660", "GPU", 0, true, true)); auto default_conn = engine::Connection("IO", 500.0); @@ -90,6 +90,8 @@ void DBTest::SetUp() { res_mgr->Start(); engine::SchedInst::GetInstance()->Start(); + engine::JobMgrInst::GetInstance()->Start(); + auto options = GetOptions(); db_ = engine::DBFactory::Build(options); } @@ -100,8 +102,9 @@ void DBTest::TearDown() { BaseTest::TearDown(); - engine::ResMgrInst::GetInstance()->Stop(); + engine::JobMgrInst::GetInstance()->Stop(); engine::SchedInst::GetInstance()->Stop(); + engine::ResMgrInst::GetInstance()->Stop(); auto options = GetOptions(); boost::filesystem::remove_all(options.meta.path); diff --git a/cpp/unittest/scheduler/CMakeLists.txt b/cpp/unittest/scheduler/CMakeLists.txt index 7359420e16..106b1a8b44 100644 --- a/cpp/unittest/scheduler/CMakeLists.txt +++ b/cpp/unittest/scheduler/CMakeLists.txt @@ -30,6 +30,7 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs) +aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/job scheduler_job_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs) @@ -55,6 +56,7 @@ set(scheduler_test_src ${test_srcs} ${scheduler_action_srcs} ${scheduler_event_srcs} + ${scheduler_job_srcs} ${scheduler_resource_srcs} ${scheduler_task_srcs} ${scheduler_srcs} diff --git a/cpp/unittest/scheduler/normal_test.cpp b/cpp/unittest/scheduler/normal_test.cpp index 5ddfd49b55..02ed3dae69 100644 --- a/cpp/unittest/scheduler/normal_test.cpp +++ b/cpp/unittest/scheduler/normal_test.cpp @@ -46,7 +46,7 @@ TEST(NormalTest, INST_TEST) { const uint64_t NUM_TASK = 1000; std::vector> tasks; - TableFileSchemaPtr dummy = nullptr; + meta::TableFileSchemaPtr dummy = nullptr; auto disks = res_mgr->GetDiskResources(); ASSERT_FALSE(disks.empty()); diff --git a/cpp/unittest/scheduler/resource_mgr_test.cpp b/cpp/unittest/scheduler/resource_mgr_test.cpp index 56db020058..8aaef5bdf5 100644 --- a/cpp/unittest/scheduler/resource_mgr_test.cpp +++ b/cpp/unittest/scheduler/resource_mgr_test.cpp @@ -187,7 +187,7 @@ TEST_F(ResourceMgrAdvanceTest, REGISTER_SUBSCRIBER) { flag = true; }; mgr1_->RegisterSubscriber(callback); - TableFileSchemaPtr dummy = nullptr; + meta::TableFileSchemaPtr dummy = nullptr; disk_res->task_table().Put(std::make_shared(dummy)); sleep(1); ASSERT_TRUE(flag); diff --git a/cpp/unittest/scheduler/resource_test.cpp b/cpp/unittest/scheduler/resource_test.cpp index b5a83ac776..1fdeb13062 100644 --- a/cpp/unittest/scheduler/resource_test.cpp +++ b/cpp/unittest/scheduler/resource_test.cpp @@ -180,7 +180,7 @@ protected: TEST_F(ResourceAdvanceTest, DISK_RESOURCE_TEST) { const uint64_t NUM = 100; std::vector> tasks; - TableFileSchemaPtr dummy = nullptr; + meta::TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { auto task = std::make_shared(dummy); tasks.push_back(task); @@ -205,7 +205,7 @@ TEST_F(ResourceAdvanceTest, DISK_RESOURCE_TEST) { TEST_F(ResourceAdvanceTest, CPU_RESOURCE_TEST) { const uint64_t NUM = 100; std::vector> tasks; - TableFileSchemaPtr dummy = nullptr; + meta::TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { auto task = std::make_shared(dummy); tasks.push_back(task); @@ -230,7 +230,7 @@ TEST_F(ResourceAdvanceTest, CPU_RESOURCE_TEST) { TEST_F(ResourceAdvanceTest, GPU_RESOURCE_TEST) { const uint64_t NUM = 100; std::vector> tasks; - TableFileSchemaPtr dummy = nullptr; + meta::TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { auto task = std::make_shared(dummy); tasks.push_back(task); @@ -255,7 +255,7 @@ TEST_F(ResourceAdvanceTest, GPU_RESOURCE_TEST) { TEST_F(ResourceAdvanceTest, TEST_RESOURCE_TEST) { const uint64_t NUM = 100; std::vector> tasks; - TableFileSchemaPtr dummy = nullptr; + meta::TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { auto task = std::make_shared(dummy); tasks.push_back(task); diff --git a/cpp/unittest/scheduler/scheduler_test.cpp b/cpp/unittest/scheduler/scheduler_test.cpp index 6f680f4a60..440a66bcbd 100644 --- a/cpp/unittest/scheduler/scheduler_test.cpp +++ b/cpp/unittest/scheduler/scheduler_test.cpp @@ -157,7 +157,7 @@ insert_dummy_index_into_gpu_cache(uint64_t device_id) { TEST_F(SchedulerTest, ON_LOAD_COMPLETED) { const uint64_t NUM = 10; std::vector> tasks; - TableFileSchemaPtr dummy = std::make_shared(); + meta::TableFileSchemaPtr dummy = std::make_shared(); dummy->location_ = "location"; insert_dummy_index_into_gpu_cache(1); @@ -177,7 +177,7 @@ TEST_F(SchedulerTest, ON_LOAD_COMPLETED) { TEST_F(SchedulerTest, PUSH_TASK_TO_NEIGHBOUR_RANDOMLY_TEST) { const uint64_t NUM = 10; std::vector> tasks; - TableFileSchemaPtr dummy1 = std::make_shared(); + meta::TableFileSchemaPtr dummy1 = std::make_shared(); dummy1->location_ = "location"; tasks.clear(); @@ -248,7 +248,7 @@ protected: TEST_F(SchedulerTest2, SPECIFIED_RESOURCE_TEST) { const uint64_t NUM = 10; std::vector> tasks; - TableFileSchemaPtr dummy = std::make_shared(); + meta::TableFileSchemaPtr dummy = std::make_shared(); dummy->location_ = "location"; for (uint64_t i = 0; i < NUM; ++i) { diff --git a/cpp/unittest/scheduler/tasktable_test.cpp b/cpp/unittest/scheduler/tasktable_test.cpp index 9e1f9b2c3a..0c3f65e146 100644 --- a/cpp/unittest/scheduler/tasktable_test.cpp +++ b/cpp/unittest/scheduler/tasktable_test.cpp @@ -169,7 +169,7 @@ class TaskTableBaseTest : public ::testing::Test { protected: void SetUp() override { - TableFileSchemaPtr dummy = nullptr; + meta::TableFileSchemaPtr dummy = nullptr; invalid_task_ = nullptr; task1_ = std::make_shared(dummy); task2_ = std::make_shared(dummy); @@ -339,7 +339,7 @@ class TaskTableAdvanceTest : public ::testing::Test { protected: void SetUp() override { - TableFileSchemaPtr dummy = nullptr; + meta::TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < 8; ++i) { auto task = std::make_shared(dummy); table1_.Put(task); From e8224fc61dcd20f2595ed899fb2acdf019ced005 Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Wed, 18 Sep 2019 19:53:08 +0800 Subject: [PATCH 036/217] MS-569 Complete the NOTICE.md Former-commit-id: dd04582d0ac67e31fdb503a70647cdd3eadebd4a --- cpp/CHANGELOG.md | 1 + cpp/NOTICE.md | 34 +++++++++++++++++----------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 886b383cd8..1327fee901 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -20,6 +20,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-554 - Change license to Apache 2.0 - MS-561 - Add contributing guidelines, code of conduct and README docs - MS-567 - Add NOTICE.md +- MS-569 - Complete the NOTICE.md # Milvus 0.4.0 (2019-09-12) diff --git a/cpp/NOTICE.md b/cpp/NOTICE.md index 970dd054c0..57766a77db 100644 --- a/cpp/NOTICE.md +++ b/cpp/NOTICE.md @@ -6,24 +6,24 @@ | Name | License | | ------------- | ------------------------------------------------------------ | -| Apache Arrow | Apache License 2.0 | -| Boost | Boost Software License | -| BZIP2 | BSD-like license | -| FAISS | MIT | -| Gtest | BSD 3-Clause | -| LAPACK | BSD 3-Clause | +| Apache Arrow | [Apache License 2.0](https://github.com/apache/arrow/blob/master/LICENSE.txt) | +| Boost | [Boost Software License](https://github.com/boostorg/boost/blob/master/LICENSE_1_0.txt) | +| BZip2 | [BZip2](http://www.bzip.org/) | +| FAISS | [MIT](https://github.com/facebookresearch/faiss/blob/master/LICENSE) | +| Gtest | [BSD 3-Clause](https://github.com/google/googletest/blob/master/LICENSE) | +| LAPACK | [LAPACK](https://github.com/Reference-LAPACK/lapack/blob/master/LICENSE) | | LZ4 | [BSD 2-Clause](https://github.com/Blosc/c-blosc/blob/master/LICENSES/LZ4.txt) | -| MySQLPP | LGPL 2.1 | -| OpenBLAS | BSD 3-Clause | -| Prometheus | Apache License 2.0 | -| Snappy | [BSD](https://github.com/Blosc/c-blosc/blob/master/LICENSES/SNAPPY.txt) | +| MySQLPP | [LGPL 2.1](https://tangentsoft.com/mysqlpp/artifact/b128a66dab867923) | +| OpenBLAS | [BSD 3-Clause](https://github.com/xianyi/OpenBLAS/blob/develop/LICENSE) | +| Prometheus | [Apache License 2.0](https://github.com/prometheus/prometheus/blob/master/LICENSE) | +| Snappy | [BSD](https://github.com/google/snappy/blob/master/COPYING) | | SQLite | [Public Domain](https://www.sqlite.org/copyright.html) | -| SQLite-ORM | BSD 3-Clause | -| yaml-cpp | MIT | -| ZLIB | [zlib](http://zlib.net/zlib_license.html) | +| SQLite-ORM | [BSD 3-Clause](https://github.com/fnc12/sqlite_orm/blob/master/LICENSE) | +| yaml-cpp | [MIT](https://github.com/jbeder/yaml-cpp/blob/master/LICENSE) | +| ZLIB | [ZLIB](http://zlib.net/zlib_license.html) | | ZSTD | [BSD](https://github.com/facebook/zstd/blob/dev/LICENSE) | -| libunwind | MIT | -| gperftools | BSD 3-Clause | -| gRPC | Apache 2.0 | -| EASYLOGGINGPP | MIT | +| libunwind | [MIT](https://github.com/libunwind/libunwind/blob/master/LICENSE) | +| gperftools | [BSD 3-Clause](https://github.com/gperftools/gperftools/blob/master/COPYING) | +| grpc | [Apache 2.0](https://github.com/grpc/grpc/blob/master/LICENSE) | +| EASYLOGGINGPP | [MIT](https://github.com/zuhd-org/easyloggingpp/blob/master/LICENSEhttps://github.com/zuhd-org/easyloggingpp/blob/master/LICENSE) | From 8382fa623c5fc4507e56d11e104859325c5aa16b Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Wed, 18 Sep 2019 20:03:01 +0800 Subject: [PATCH 037/217] MS-570 - Add prometheus docker-compose file Former-commit-id: 1b60196354afab6e8fd3355d0752d66cf9722702 --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index c5d27e3461..a1b06bba14 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -136,6 +136,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-523 - Config file validation - MS-539 - Remove old task code - MS-546 - Add simple mode resource_config +- MS-570 - Add prometheus docker-compose file ## New Feature - MS-343 - Implement ResourceMgr From 1a3231e6f6e4f0343180397e395cb277f74483a8 Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Wed, 18 Sep 2019 20:03:19 +0800 Subject: [PATCH 038/217] MS-570 - Add prometheus docker-compose file Former-commit-id: a81bcfd940892284d9f0c26049e4c42c0cf80f34 --- docker/alertmanager.yml | 19 +++++++++++ docker/docker-compose-monitor.yml | 56 +++++++++++++++++++++++++++++++ docker/prometheus.yml | 46 +++++++++++++++++++++++++ docker/server_down.yml | 8 +++++ 4 files changed, 129 insertions(+) create mode 100644 docker/alertmanager.yml create mode 100644 docker/docker-compose-monitor.yml create mode 100644 docker/prometheus.yml create mode 100644 docker/server_down.yml diff --git a/docker/alertmanager.yml b/docker/alertmanager.yml new file mode 100644 index 0000000000..e1f079dce0 --- /dev/null +++ b/docker/alertmanager.yml @@ -0,0 +1,19 @@ +global: + resolve_timeout: 5m + +route: + group_by: ['alertname'] + group_wait: 10s + group_interval: 10s + repeat_interval: 1h + receiver: 'web.hook' +receivers: +- name: 'web.hook' + webhook_configs: + - url: 'http://127.0.0.1:5001/' +inhibit_rules: + - source_match: + severity: 'critical' + target_match: + severity: 'warning' + equal: ['alertname', 'dev', 'instance'] diff --git a/docker/docker-compose-monitor.yml b/docker/docker-compose-monitor.yml new file mode 100644 index 0000000000..9cb1fb1902 --- /dev/null +++ b/docker/docker-compose-monitor.yml @@ -0,0 +1,56 @@ +version: '2.3' + +networks: + monitor: + driver: bridge + +services: + prometheus: + image: prom/prometheus:v2.11.1 + container_name: prometheus + hostname: prometheus + restart: always + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + - ./server_down.yml:/etc/prometheus/node_down.yml + ports: + - "9090:9090" + networks: + - monitor + + alertmanager: + image: prom/alertmanager + container_name: alertmanager + hostname: alertmanager + restart: always + volumes: + - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml + ports: + - "9093:9093" + networks: + - monitor + + grafana: + image: grafana/grafana + container_name: grafana + hostname: grafana + restart: always + ports: + - "3000:3000" + networks: + - monitor + + milvus: + runtime: nvidia + image: registry.zilliz.com/milvus/engine:branch-0.4.0-release + container_name: milvus + hostname: milvus + restart: always + volumes: + - ../cpp/conf/server_config.yaml:/opt/milvus/cpp/conf/server_config.yaml + - ../cpp/conf/log_config.conf:/opt/milvus/cpp/conf/log_config.conf + ports: + - "8080:8080" + - "19530:19530" + networks: + - monitor diff --git a/docker/prometheus.yml b/docker/prometheus.yml new file mode 100644 index 0000000000..7a2a07b390 --- /dev/null +++ b/docker/prometheus.yml @@ -0,0 +1,46 @@ +# my global config +global: + scrape_interval: 15s # Set the scrape interval to every 1 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: ['localhost:9093'] + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + - "serverdown.yml" # add alerting rules + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: 'prometheus' + + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + + static_configs: + - targets: ['localhost:9090'] + + # scrape metrics of server + - job_name: 'milvus_server' + scrape_interval: 1s + static_configs: + - targets: ['localhost:8080'] + + # scrape metrics of server + - job_name: 'milvus_server_1' + scrape_interval: 1s + static_configs: + - targets: ['localhost:8080'] + + + + # under development + - job_name: 'pushgateway' + static_configs: + - targets: ['localhost:9091'] diff --git a/docker/server_down.yml b/docker/server_down.yml new file mode 100644 index 0000000000..8a7077c7f8 --- /dev/null +++ b/docker/server_down.yml @@ -0,0 +1,8 @@ +groups: +- name: milvus + rules: + - alert: MilvusServerDown + expr: up{job="milvus_server"} + for: 1s + labels: + serverity: page From 3978c2a71f73397a15ccc465ed1f339c6e3fa08a Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Thu, 19 Sep 2019 11:04:56 +0800 Subject: [PATCH 039/217] add Log.h in knowhere Former-commit-id: e866a09e487f1ef60816bd0ee6c7308e4bf1ff7e --- cpp/src/core/knowhere/knowhere/common/Log.h | 35 +++++++++++++++++++ .../knowhere/knowhere/common/exception.cpp | 1 + .../knowhere/index/vector_index/gpu_ivf.cpp | 2 ++ 3 files changed, 38 insertions(+) create mode 100644 cpp/src/core/knowhere/knowhere/common/Log.h diff --git a/cpp/src/core/knowhere/knowhere/common/Log.h b/cpp/src/core/knowhere/knowhere/common/Log.h new file mode 100644 index 0000000000..0d99c624ab --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/common/Log.h @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include "utils/easylogging++.h" + +namespace zilliz { +namespace knowhere { +#define KNOWHERE_DOMAIN_NAME "[KNOWHERE] " +#define KNOWHERE_ERROR_TEXT "KNOWHERE Error:" + +#define KNOWHERE_LOG_TRACE LOG(TRACE) << KNOWHERE_DOMAIN_NAME +#define KNOWHERE_LOG_DEBUG LOG(DEBUG) << KNOWHERE_DOMAIN_NAME +#define KNOWHERE_LOG_INFO LOG(INFO) << KNOWHERE_DOMAIN_NAME +#define KNOWHERE_LOG_WARNING LOG(WARNING) << KNOWHERE_DOMAIN_NAME +#define KNOWHERE_LOG_ERROR LOG(ERROR) << KNOWHERE_DOMAIN_NAME +#define KNOWHERE_LOG_FATAL LOG(FATAL) << KNOWHERE_DOMAIN_NAME +} // namespace knowhere +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/core/knowhere/knowhere/common/exception.cpp b/cpp/src/core/knowhere/knowhere/common/exception.cpp index fcf3886d9b..eb19e21bb0 100644 --- a/cpp/src/core/knowhere/knowhere/common/exception.cpp +++ b/cpp/src/core/knowhere/knowhere/common/exception.cpp @@ -19,6 +19,7 @@ #include #include "exception.h" +#include "Log.h" namespace zilliz { namespace knowhere { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp index fd04501c83..51fe237658 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp @@ -30,6 +30,7 @@ #include "cloner.h" #include "knowhere/adapter/faiss_adopt.h" #include "gpu_ivf.h" +#include "knowhere/common/Log.h" #include @@ -223,6 +224,7 @@ VectorIndexPtr GPUIVFPQ::CopyGpuToCpu(const Config &config) { } IndexModelPtr GPUIVFSQ::Train(const DatasetPtr &dataset, const Config &config) { + KNOWHERE_LOG_WARNING << "lalalalalalala"; auto nlist = config["nlist"].as(); auto nbits = config["nbits"].as(); // TODO(linxj): gpu only support SQ4 SQ8 SQ16 gpu_id_ = config.get_with_default("gpu_id", gpu_id_); From 5c16730469489982b30af86ca83fa189cc730f73 Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 19 Sep 2019 12:52:46 +0800 Subject: [PATCH 040/217] fix cmake Former-commit-id: dd8288d065c48eccc6dff699d153e48fbe5205f4 --- cpp/src/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 3bf20ac2fd..6ee3966c4f 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -152,7 +152,7 @@ if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") endif () cuda_add_library(milvus_engine STATIC ${engine_files}) -target_link_libraries(milvus_engine ${engine_libs} ${third_party_libs}) +target_link_libraries(milvus_engine ${engine_libs} knowhere ${third_party_libs}) add_library(metrics STATIC ${metrics_files}) @@ -180,7 +180,7 @@ add_executable(milvus_server ${utils_files} ) -target_link_libraries(milvus_server ${server_libs} knowhere ${third_party_libs}) +target_link_libraries(milvus_server ${server_libs}) install(TARGETS milvus_server DESTINATION bin) From 4b6be7b0c09fc4555267e6e5a0e6858ab81779e3 Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Thu, 19 Sep 2019 14:10:22 +0800 Subject: [PATCH 041/217] Update README.md Former-commit-id: aeac98a8cdc3cf186ee9c3e8916ca646345e6218 --- cpp/README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index 18c1131f10..d32f04f316 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -4,9 +4,9 @@ Firstly, welcome, and thanks for your interest in [Milvus](https://milvus.io)! N ## What is Milvus -Milvus is an open source vector search engine that makes incredibly fast querying speed enhancement over current solutions of massive vector processing. Built on optimized indexing algorithm, it is compatible with major AI/ML models. +Milvus is an open source vector search engine that supports similarity search of large-scale vectors. Built on optimized indexing algorithm, it is compatible with major AI/ML models. -Milvus was developed by researchers and engineers in ZILLIZ, a tech startup that intends to reinvent data science, with the purpose of providing enterprises with efficient and scalable similarity search and analysis of feature vectors and unstructured data. +Milvus was developed by ZILLIZ, a tech startup that intends to reinvent data science, with the purpose of providing enterprises with efficient and scalable similarity search and analysis of feature vectors and unstructured data. Milvus provides stable Python and C++ APIs, as well as RESTful API. @@ -182,11 +182,11 @@ $ python3 example.py Contributions are welcomed and greatly appreciated. If you want to contribute to Milvus, please read the [contribution guidelines](CONTRIBUTING.md). This project adheres to the [code of conduct](CODE OF CONDUCT.md) of Milvus. By participating, you are expected to uphold this code. -We use [GitHub issues] to track issues and bugs. For general questions and discussions, please go to [Milvus Forum]. +We use [GitHub issues](https://github.com/milvus-io/milvus/issues) to track issues and bugs. For general questions and discussions, please go to [Milvus Forum]. ## Join the Milvus community -For public discussion of Milvus, please join our [discussion group]. +For public discussion of Milvus, please join our [discussion group](milvusio.slack.com). ## Milvus Roadmap @@ -194,7 +194,7 @@ Please read our [roadmap](milvus-io/milvus/docs/master/roadmap.md) to learn abou ## Resources -[Milvus official website](https://milvus.io) +[Milvus official website](https://www.milvus.io) [Milvus docs](https://www.milvus.io/docs/en/QuickStart/) @@ -204,7 +204,6 @@ Please read our [roadmap](milvus-io/milvus/docs/master/roadmap.md) to learn abou [Milvus roadmap](https://www.milvus-io/docs/master/roadmap.md) -[Milvus white paper] ## License From c89a99b8002866822c64c58daa427bd872bed2bd Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Thu, 19 Sep 2019 09:29:21 +0800 Subject: [PATCH 042/217] MS-572 fix Milvus crash when get SIGINT Former-commit-id: 32b92068024731fcc3451271c7f66f8a37b68cbd --- cpp/src/server/Server.cpp | 15 +++++++--- cpp/src/server/Server.h | 15 +++++++--- cpp/src/server/grpc_impl/GrpcMilvusServer.cpp | 28 ++++++++++++------- cpp/src/server/grpc_impl/GrpcMilvusServer.h | 25 +++++++++++++---- cpp/src/utils/SignalUtil.cpp | 4 +-- 5 files changed, 62 insertions(+), 25 deletions(-) diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 9920b30246..e4cb7b8d4a 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -42,7 +42,7 @@ namespace zilliz { namespace milvus { namespace server { -Server& +Server & Server::Instance() { static Server server; return server; @@ -203,6 +203,10 @@ Server::Start() { std::cout << "Milvus server start successfully." << std::endl; StartService(); + while (running_) { + sleep(10); + } + } catch (std::exception &ex) { std::cerr << "Milvus server encounter exception: " << std::string(ex.what()) << "Is another server instance running?"; @@ -210,7 +214,6 @@ Server::Start() { } } while (false); - Stop(); return 0; } @@ -242,6 +245,10 @@ Server::Stop() { } StopService(); + { + std::lock_guard lock(mutex_); + running_ = false; + } std::cerr << "Milvus server is closed!" << std::endl; } @@ -264,12 +271,12 @@ Server::StartService() { engine::KnowhereResource::Initialize(); engine::StartSchedulerService(); DBWrapper::GetInstance().StartService(); - grpc::GrpcMilvusServer::StartService(); + grpc::GrpcMilvusServer::GetInstance().Start(); } void Server::StopService() { - grpc::GrpcMilvusServer::StopService(); + grpc::GrpcMilvusServer::GetInstance().Stop(); DBWrapper::GetInstance().StopService(); engine::StopSchedulerService(); engine::KnowhereResource::Finalize(); diff --git a/cpp/src/server/Server.h b/cpp/src/server/Server.h index eaf19d6b15..a4757cd0c6 100644 --- a/cpp/src/server/Server.h +++ b/cpp/src/server/Server.h @@ -21,23 +21,27 @@ #include #include +#include + namespace zilliz { namespace milvus { namespace server { class Server { -public: + public: static Server &Instance(); - void Init(int64_t daemonized, const std::string &pid_filename, const std::string &config_filename, + void Init(int64_t daemonized, + const std::string &pid_filename, + const std::string &config_filename, const std::string &log_config_file); int Start(); void Stop(); -private: + private: Server(); ~Server(); @@ -50,12 +54,15 @@ private: void StopService(); -private: + private: int64_t daemonized_ = 0; int pid_fd = -1; std::string pid_filename_; std::string config_filename_; std::string log_config_file_; + + std::mutex mutex_; + bool running_ = true; }; // Server } // server diff --git a/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp b/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp index 6770e3111a..1af3557488 100644 --- a/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp @@ -42,7 +42,6 @@ namespace milvus { namespace server { namespace grpc { -static std::unique_ptr<::grpc::Server> server; constexpr long MESSAGE_SIZE = -1; @@ -58,13 +57,22 @@ class NoReusePortOption : public ::grpc::ServerBuilderOption { }; +void +GrpcMilvusServer::Start() { + thread_ptr_ = std::make_shared(&GrpcMilvusServer::StartService, this); +} + +void +GrpcMilvusServer::Stop() { + StopService(); + if (thread_ptr_) { + thread_ptr_->join(); + thread_ptr_ = nullptr; + } +} + Status GrpcMilvusServer::StartService() { - if (server != nullptr) { - std::cout << "stop service!\n"; - StopService(); - } - ServerConfig &config = ServerConfig::GetInstance(); ConfigNode server_config = config.GetConfig(CONFIG_SERVER); ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); @@ -87,16 +95,16 @@ GrpcMilvusServer::StartService() { builder.AddListeningPort(server_address, ::grpc::InsecureServerCredentials()); builder.RegisterService(&service); - server = builder.BuildAndStart(); - server->Wait(); + server_ptr_ = builder.BuildAndStart(); + server_ptr_->Wait(); return Status::OK(); } Status GrpcMilvusServer::StopService() { - if (server != nullptr) { - server->Shutdown(); + if (server_ptr_ != nullptr) { + server_ptr_->Shutdown(); } return Status::OK(); diff --git a/cpp/src/server/grpc_impl/GrpcMilvusServer.h b/cpp/src/server/grpc_impl/GrpcMilvusServer.h index 751a2cd2a3..40df193a2f 100644 --- a/cpp/src/server/grpc_impl/GrpcMilvusServer.h +++ b/cpp/src/server/grpc_impl/GrpcMilvusServer.h @@ -21,6 +21,8 @@ #include #include +#include +#include namespace zilliz { namespace milvus { @@ -28,12 +30,25 @@ namespace server { namespace grpc { class GrpcMilvusServer { -public: - static Status - StartService(); + public: + static GrpcMilvusServer& GetInstance() { + static GrpcMilvusServer grpc_server; + return grpc_server; + } - static Status - StopService(); + void Start(); + void Stop(); + + private: + GrpcMilvusServer() = default; + ~GrpcMilvusServer() = default; + + Status StartService(); + Status StopService(); + + private: + std::unique_ptr<::grpc::Server> server_ptr_; + std::shared_ptr thread_ptr_; }; } diff --git a/cpp/src/utils/SignalUtil.cpp b/cpp/src/utils/SignalUtil.cpp index e380dc6c85..4da8d31eb0 100644 --- a/cpp/src/utils/SignalUtil.cpp +++ b/cpp/src/utils/SignalUtil.cpp @@ -31,7 +31,7 @@ void SignalUtil::HandleSignal(int signum){ switch(signum){ case SIGINT: case SIGUSR2:{ - SERVER_LOG_INFO << "Server received signal:" << std::to_string(signum); + SERVER_LOG_INFO << "Server received signal: " << signum; server::Server& server_ptr = server::Server::Instance(); server_ptr.Stop(); @@ -39,7 +39,7 @@ void SignalUtil::HandleSignal(int signum){ exit(0); } default:{ - SERVER_LOG_INFO << "Server received critical signal:" << std::to_string(signum); + SERVER_LOG_INFO << "Server received critical signal: " << signum; SignalUtil::PrintStacktrace(); server::Server& server_ptr = server::Server::Instance(); From 4d23ecf082b06a6b19e5e053af22adc792e8c3e4 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Thu, 19 Sep 2019 14:15:56 +0800 Subject: [PATCH 043/217] MS-572 re-org signal handle Former-commit-id: a357dd78d214c3f2c39538ea32f7fc492d72b6f2 --- cpp/CHANGELOG.md | 1 + cpp/src/main.cpp | 54 ++++----- cpp/src/server/Server.cpp | 109 ++++++++---------- cpp/src/server/Server.h | 8 +- cpp/src/server/grpc_impl/GrpcRequestTask.cpp | 2 +- .../{GrpcMilvusServer.cpp => GrpcServer.cpp} | 18 +-- .../{GrpcMilvusServer.h => GrpcServer.h} | 11 +- cpp/src/utils/SignalUtil.cpp | 18 +-- 8 files changed, 101 insertions(+), 120 deletions(-) rename cpp/src/server/grpc_impl/{GrpcMilvusServer.cpp => GrpcServer.cpp} (90%) rename cpp/src/server/grpc_impl/{GrpcMilvusServer.h => GrpcServer.h} (87%) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index d4d609d4aa..300d40f0af 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -5,6 +5,7 @@ Please mark all change in change log and use the ticket from JIRA. # Milvus 0.5.0 (TODO) ## Bug +- MS-572 - Milvus crash when get SIGINT ## Improvement - MS-552 - Add and change the easylogging library diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index 3065e7d684..5819463d60 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -15,19 +15,20 @@ // specific language governing permissions and limitations // under the License. -#include "server/Server.h" -#include "version.h" - #include #include #include #include #include -#include "utils/easylogging++.h" -#include "metrics/Metrics.h" +#include +#include "utils/easylogging++.h" #include "utils/SignalUtil.h" #include "utils/CommonUtil.h" +#include "metrics/Metrics.h" +#include "server/Server.h" +#include "version.h" + INITIALIZE_EASYLOGGINGPP @@ -40,12 +41,6 @@ main(int argc, char *argv[]) { std::cout << std::endl << "Welcome to use Milvus by Zilliz!" << std::endl; std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << " built at " << BUILD_TIME << std::endl; - signal(SIGINT, server::SignalUtil::HandleSignal); - signal(SIGSEGV, server::SignalUtil::HandleSignal); - signal(SIGUSR1, server::SignalUtil::HandleSignal); - signal(SIGUSR2, server::SignalUtil::HandleSignal); - - std::string app_name = basename(argv[0]); static struct option long_options[] = {{"conf_file", required_argument, 0, 'c'}, {"log_conf_file", required_argument, 0, 'l'}, {"help", no_argument, 0, 'h'}, @@ -55,14 +50,12 @@ main(int argc, char *argv[]) { int option_index = 0; int64_t start_daemonized = 0; -// int pid_fd; std::string config_filename, log_config_file; std::string pid_filename; + std::string app_name = argv[0]; - app_name = argv[0]; - - if(argc < 2) { + if (argc < 2) { print_help(app_name); std::cout << "Milvus server exit..." << std::endl; return EXIT_FAILURE; @@ -94,29 +87,38 @@ main(int argc, char *argv[]) { break; } - case 'd': - start_daemonized = 1; + case 'd':start_daemonized = 1; break; - case 'h': - print_help(app_name); + case 'h':print_help(app_name); return EXIT_SUCCESS; - case '?': - print_help(app_name); + case '?':print_help(app_name); return EXIT_FAILURE; - default: - print_help(app_name); + default:print_help(app_name); break; } } - server::Server& server = server::Server::Instance(); + server::Server &server = server::Server::Instance(); server.Init(start_daemonized, pid_filename, config_filename, log_config_file); - return server.Start(); + server.Start(); + + /* Handle Signal */ + signal(SIGHUP, server::SignalUtil::HandleSignal); + signal(SIGINT, server::SignalUtil::HandleSignal); + signal(SIGUSR1, server::SignalUtil::HandleSignal); + signal(SIGSEGV, server::SignalUtil::HandleSignal); + signal(SIGUSR2, server::SignalUtil::HandleSignal); + signal(SIGTERM, server::SignalUtil::HandleSignal); + + /* wait signal */ + pause(); + + return 0; } void print_help(const std::string &app_name) { - std::cout << std::endl<< "Usage: " << app_name << " [OPTIONS]" << std::endl << std::endl; + std::cout << std::endl << "Usage: " << app_name << " [OPTIONS]" << std::endl << std::endl; std::cout << " Options:" << std::endl; std::cout << " -h --help Print this help" << std::endl; std::cout << " -c --conf_file filename Read configuration from the file" << std::endl; diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index e4cb7b8d4a..38b41a5331 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -17,7 +17,7 @@ #include #include "Server.h" -#include "server/grpc_impl/GrpcMilvusServer.h" +#include "server/grpc_impl/GrpcServer.h" #include "utils/Log.h" #include "utils/LogUtil.h" #include "utils/SignalUtil.h" @@ -154,74 +154,61 @@ Server::Daemonize() { } } -int +void Server::Start() { - if (daemonized_) { Daemonize(); } - do { - try { - // Read config file - if (LoadConfig() != SERVER_SUCCESS) { - return 1; - } - - //log path is defined by LoadConfig, so InitLog must be called after LoadConfig - ServerConfig &config = ServerConfig::GetInstance(); - ConfigNode server_config = config.GetConfig(CONFIG_SERVER); - - std::string time_zone = server_config.GetValue(CONFIG_TIME_ZONE, "UTC+8"); - if (time_zone.length() == 3) { - time_zone = "CUT"; - } else { - int time_bias = std::stoi(time_zone.substr(3, std::string::npos)); - if (time_bias == 0) - time_zone = "CUT"; - else if (time_bias > 0) { - time_zone = "CUT" + std::to_string(-time_bias); - } else { - time_zone = "CUT+" + std::to_string(-time_bias); - } - } - - if (setenv("TZ", time_zone.c_str(), 1) != 0) { - return -1; - } - tzset(); - - InitLog(log_config_file_); - - // Handle Signal - signal(SIGINT, SignalUtil::HandleSignal); - signal(SIGHUP, SignalUtil::HandleSignal); - signal(SIGTERM, SignalUtil::HandleSignal); - server::Metrics::GetInstance().Init(); - server::SystemInfo::GetInstance().Init(); - - std::cout << "Milvus server start successfully." << std::endl; - StartService(); - - while (running_) { - sleep(10); - } - - } catch (std::exception &ex) { - std::cerr << "Milvus server encounter exception: " << std::string(ex.what()) - << "Is another server instance running?"; - break; + try { + /* Read config file */ + if (LoadConfig() != SERVER_SUCCESS) { + std::cerr << "Milvus server fail to load config file" << std::endl; + return; } - } while (false); - return 0; + /* log path is defined in Config file, so InitLog must be called after LoadConfig */ + ServerConfig &config = ServerConfig::GetInstance(); + ConfigNode server_config = config.GetConfig(CONFIG_SERVER); + + std::string time_zone = server_config.GetValue(CONFIG_TIME_ZONE, "UTC+8"); + if (time_zone.length() == 3) { + time_zone = "CUT"; + } else { + int time_bias = std::stoi(time_zone.substr(3, std::string::npos)); + if (time_bias == 0) + time_zone = "CUT"; + else if (time_bias > 0) { + time_zone = "CUT" + std::to_string(-time_bias); + } else { + time_zone = "CUT+" + std::to_string(-time_bias); + } + } + + if (setenv("TZ", time_zone.c_str(), 1) != 0) { + std::cerr << "Fail to setenv" << std::endl; + return; + } + tzset(); + + InitLog(log_config_file_); + + server::Metrics::GetInstance().Init(); + server::SystemInfo::GetInstance().Init(); + + std::cout << "Milvus server start successfully." << std::endl; + StartService(); + + } catch (std::exception &ex) { + std::cerr << "Milvus server encounter exception: " << ex.what(); + } } void Server::Stop() { std::cerr << "Milvus server is going to shutdown ..." << std::endl; - // Unlock and close lockfile + /* Unlock and close lockfile */ if (pid_fd != -1) { int ret = lockf(pid_fd, F_ULOCK, 0); if (ret != 0) { @@ -235,7 +222,7 @@ Server::Stop() { } } - // Try to delete lockfile + /* delete lockfile */ if (!pid_filename_.empty()) { int ret = unlink(pid_filename_.c_str()); if (ret != 0) { @@ -245,10 +232,6 @@ Server::Stop() { } StopService(); - { - std::lock_guard lock(mutex_); - running_ = false; - } std::cerr << "Milvus server is closed!" << std::endl; } @@ -271,12 +254,12 @@ Server::StartService() { engine::KnowhereResource::Initialize(); engine::StartSchedulerService(); DBWrapper::GetInstance().StartService(); - grpc::GrpcMilvusServer::GetInstance().Start(); + grpc::GrpcServer::GetInstance().Start(); } void Server::StopService() { - grpc::GrpcMilvusServer::GetInstance().Stop(); + grpc::GrpcServer::GetInstance().Stop(); DBWrapper::GetInstance().StopService(); engine::StopSchedulerService(); engine::KnowhereResource::Finalize(); diff --git a/cpp/src/server/Server.h b/cpp/src/server/Server.h index a4757cd0c6..2c912c3e91 100644 --- a/cpp/src/server/Server.h +++ b/cpp/src/server/Server.h @@ -37,13 +37,11 @@ class Server { const std::string &config_filename, const std::string &log_config_file); - int Start(); - + void Start(); void Stop(); private: Server(); - ~Server(); void Daemonize(); @@ -51,7 +49,6 @@ class Server { ErrorCode LoadConfig(); void StartService(); - void StopService(); private: @@ -60,9 +57,6 @@ class Server { std::string pid_filename_; std::string config_filename_; std::string log_config_file_; - - std::mutex mutex_; - bool running_ = true; }; // Server } // server diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index e49ea48236..ad0abd713d 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -23,7 +23,7 @@ #include "utils/ValidationUtil.h" #include "../DBWrapper.h" #include "version.h" -#include "GrpcMilvusServer.h" +#include "GrpcServer.h" #include "db/Utils.h" #include "scheduler/SchedInst.h" //#include diff --git a/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp b/cpp/src/server/grpc_impl/GrpcServer.cpp similarity index 90% rename from cpp/src/server/grpc_impl/GrpcMilvusServer.cpp rename to cpp/src/server/grpc_impl/GrpcServer.cpp index 1af3557488..4ff7a8200f 100644 --- a/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcServer.cpp @@ -16,7 +16,7 @@ // under the License. #include "milvus.grpc.pb.h" -#include "GrpcMilvusServer.h" +#include "GrpcServer.h" #include "server/ServerConfig.h" #include "server/DBWrapper.h" #include "utils/Log.h" @@ -34,7 +34,6 @@ #include #include #include -#include namespace zilliz { @@ -52,18 +51,19 @@ class NoReusePortOption : public ::grpc::ServerBuilderOption { args->SetInt(GRPC_ARG_ALLOW_REUSEPORT, 0); } - void UpdatePlugins(std::vector> * - plugins) override {} + void UpdatePlugins(std::vector> *plugins) override { + + } }; void -GrpcMilvusServer::Start() { - thread_ptr_ = std::make_shared(&GrpcMilvusServer::StartService, this); +GrpcServer::Start() { + thread_ptr_ = std::make_shared(&GrpcServer::StartService, this); } void -GrpcMilvusServer::Stop() { +GrpcServer::Stop() { StopService(); if (thread_ptr_) { thread_ptr_->join(); @@ -72,7 +72,7 @@ GrpcMilvusServer::Stop() { } Status -GrpcMilvusServer::StartService() { +GrpcServer::StartService() { ServerConfig &config = ServerConfig::GetInstance(); ConfigNode server_config = config.GetConfig(CONFIG_SERVER); ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); @@ -102,7 +102,7 @@ GrpcMilvusServer::StartService() { } Status -GrpcMilvusServer::StopService() { +GrpcServer::StopService() { if (server_ptr_ != nullptr) { server_ptr_->Shutdown(); } diff --git a/cpp/src/server/grpc_impl/GrpcMilvusServer.h b/cpp/src/server/grpc_impl/GrpcServer.h similarity index 87% rename from cpp/src/server/grpc_impl/GrpcMilvusServer.h rename to cpp/src/server/grpc_impl/GrpcServer.h index 40df193a2f..a861692fac 100644 --- a/cpp/src/server/grpc_impl/GrpcMilvusServer.h +++ b/cpp/src/server/grpc_impl/GrpcServer.h @@ -24,15 +24,16 @@ #include #include + namespace zilliz { namespace milvus { namespace server { namespace grpc { -class GrpcMilvusServer { +class GrpcServer { public: - static GrpcMilvusServer& GetInstance() { - static GrpcMilvusServer grpc_server; + static GrpcServer &GetInstance() { + static GrpcServer grpc_server; return grpc_server; } @@ -40,8 +41,8 @@ class GrpcMilvusServer { void Stop(); private: - GrpcMilvusServer() = default; - ~GrpcMilvusServer() = default; + GrpcServer() = default; + ~GrpcServer() = default; Status StartService(); Status StopService(); diff --git a/cpp/src/utils/SignalUtil.cpp b/cpp/src/utils/SignalUtil.cpp index 4da8d31eb0..8677016cf3 100644 --- a/cpp/src/utils/SignalUtil.cpp +++ b/cpp/src/utils/SignalUtil.cpp @@ -22,27 +22,28 @@ #include #include + namespace zilliz { namespace milvus { namespace server { -void SignalUtil::HandleSignal(int signum){ +void SignalUtil::HandleSignal(int signum) { - switch(signum){ + switch (signum) { case SIGINT: - case SIGUSR2:{ + case SIGUSR2: { SERVER_LOG_INFO << "Server received signal: " << signum; - server::Server& server_ptr = server::Server::Instance(); + server::Server &server_ptr = server::Server::Instance(); server_ptr.Stop(); exit(0); } - default:{ + default: { SERVER_LOG_INFO << "Server received critical signal: " << signum; SignalUtil::PrintStacktrace(); - server::Server& server_ptr = server::Server::Instance(); + server::Server &server_ptr = server::Server::Instance(); server_ptr.Stop(); exit(1); @@ -54,9 +55,9 @@ void SignalUtil::PrintStacktrace() { SERVER_LOG_INFO << "Call stack:"; const int size = 32; - void* array[size]; + void *array[size]; int stack_num = backtrace(array, size); - char ** stacktrace = backtrace_symbols(array, stack_num); + char **stacktrace = backtrace_symbols(array, stack_num); for (int i = 0; i < stack_num; ++i) { std::string info = stacktrace[i]; SERVER_LOG_INFO << info; @@ -64,7 +65,6 @@ void SignalUtil::PrintStacktrace() { free(stacktrace); } - } } } From 19d7c065acf7cf408b7f700a3885f21abd6c52cc Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Thu, 19 Sep 2019 14:46:09 +0800 Subject: [PATCH 044/217] MS-572 correct format Former-commit-id: 0bdbc27634eed7ba241b40d5957f5689d50d6a6a --- cpp/src/main.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index 5819463d60..54f9541eff 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -87,13 +87,17 @@ main(int argc, char *argv[]) { break; } - case 'd':start_daemonized = 1; + case 'd': + start_daemonized = 1; break; - case 'h':print_help(app_name); + case 'h': + print_help(app_name); return EXIT_SUCCESS; - case '?':print_help(app_name); + case '?': + print_help(app_name); return EXIT_FAILURE; - default:print_help(app_name); + default: + print_help(app_name); break; } } From 02270a6298a91856926a64b274e47ccdf7dd7eb4 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Thu, 19 Sep 2019 14:52:14 +0800 Subject: [PATCH 045/217] MS-568 Fix GpuResource free error Former-commit-id: 52acd35e93a5293c70c45bb681fc54046b43a2cb --- cpp/CHANGELOG.md | 1 + cpp/src/core/knowhere/CMakeLists.txt | 1 + .../vector_index/FaissGpuResourceMgr.cpp | 128 +++++++++++++++++ .../index/vector_index/FaissGpuResourceMgr.h | 130 ++++++++++++++++++ .../knowhere/index/vector_index/gpu_ivf.cpp | 119 ++-------------- .../knowhere/index/vector_index/gpu_ivf.h | 106 +------------- .../knowhere/index/vector_index/idmap.cpp | 2 +- cpp/src/core/test/CMakeLists.txt | 2 + cpp/src/wrapper/KnowhereResource.cpp | 1 + cpp/src/wrapper/KnowhereResource.h | 1 - 10 files changed, 276 insertions(+), 215 deletions(-) create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.cpp create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.h diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 9af19afe26..63a1e887b6 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -5,6 +5,7 @@ Please mark all change in change log and use the ticket from JIRA. # Milvus 0.5.0 (TODO) ## Bug +- MS-568 - Fix gpuresource free error ## Improvement - MS-552 - Add and change the easylogging library diff --git a/cpp/src/core/knowhere/CMakeLists.txt b/cpp/src/core/knowhere/CMakeLists.txt index 24cae1d6f9..55f23e1095 100644 --- a/cpp/src/core/knowhere/CMakeLists.txt +++ b/cpp/src/core/knowhere/CMakeLists.txt @@ -48,6 +48,7 @@ set(index_srcs knowhere/index/vector_index/nsg/nsg_io.cpp knowhere/index/vector_index/nsg/utils.cpp knowhere/index/vector_index/cloner.cpp + knowhere/index/vector_index/FaissGpuResourceMgr.cpp ) set(depend_libs diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.cpp new file mode 100644 index 0000000000..c50594b4b0 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.cpp @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include "FaissGpuResourceMgr.h" + + +namespace zilliz { +namespace knowhere { + +FaissGpuResourceMgr &FaissGpuResourceMgr::GetInstance() { + static FaissGpuResourceMgr instance; + return instance; +} + +void FaissGpuResourceMgr::AllocateTempMem(ResPtr &resource, + const int64_t &device_id, + const int64_t &size) { + if (size) { + resource->faiss_res->setTempMemory(size); + } + else { + auto search = devices_params_.find(device_id); + if (search != devices_params_.end()) { + resource->faiss_res->setTempMemory(search->second.temp_mem_size); + } + // else do nothing. allocate when use. + } +} + +void FaissGpuResourceMgr::InitDevice(int64_t device_id, + int64_t pin_mem_size, + int64_t temp_mem_size, + int64_t res_num) { + DeviceParams params; + params.pinned_mem_size = pin_mem_size; + params.temp_mem_size = temp_mem_size; + params.resource_num = res_num; + + devices_params_.emplace(device_id, params); +} + +void FaissGpuResourceMgr::InitResource() { + if(is_init) return ; + + is_init = true; + + //std::cout << "InitResource" << std::endl; + for(auto& device : devices_params_) { + auto& device_id = device.first; + + mutex_cache_.emplace(device_id, std::make_unique()); + + //std::cout << "Device Id: " << device_id << std::endl; + auto& device_param = device.second; + auto& bq = idle_map_[device_id]; + + for (int64_t i = 0; i < device_param.resource_num; ++i) { + //std::cout << "Resource Id: " << i << std::endl; + auto raw_resource = std::make_shared(); + + // TODO(linxj): enable set pinned memory + auto res_wrapper = std::make_shared(raw_resource); + AllocateTempMem(res_wrapper, device_id, 0); + + bq.Put(res_wrapper); + } + } + //std::cout << "End initResource" << std::endl; +} + +ResPtr FaissGpuResourceMgr::GetRes(const int64_t &device_id, + const int64_t &alloc_size) { + InitResource(); + + auto finder = idle_map_.find(device_id); + if (finder != idle_map_.end()) { + auto& bq = finder->second; + auto&& resource = bq.Take(); + AllocateTempMem(resource, device_id, alloc_size); + return resource; + } + return nullptr; +} + +void FaissGpuResourceMgr::MoveToIdle(const int64_t &device_id, const ResPtr &res) { + auto finder = idle_map_.find(device_id); + if (finder != idle_map_.end()) { + auto& bq = finder->second; + bq.Put(res); + } +} + +void FaissGpuResourceMgr::Free() { + for (auto &item : idle_map_) { + auto& bq = item.second; + while (!bq.Empty()) { + bq.Take(); + } + } + is_init = false; +} + +void +FaissGpuResourceMgr::Dump() { + for (auto &item : idle_map_) { + auto& bq = item.second; + std::cout << "device_id: " << item.first + << ", resource count:" << bq.Size(); + } +} + +} // knowhere +} // zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.h b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.h new file mode 100644 index 0000000000..3b4786a2a5 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.h @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include +#include +#include + +#include + +#include "src/utils/BlockingQueue.h" + +namespace zilliz { +namespace knowhere { + +struct Resource { + explicit Resource(std::shared_ptr &r) : faiss_res(r) { + static int64_t global_id = 0; + id = global_id++; + } + + std::shared_ptr faiss_res; + int64_t id; + std::mutex mutex; +}; +using ResPtr = std::shared_ptr; +using ResWPtr = std::weak_ptr; + +class FaissGpuResourceMgr { +public: + friend class ResScope; + using ResBQ = zilliz::milvus::server::BlockingQueue; + +public: + struct DeviceParams { + int64_t temp_mem_size = 0; + int64_t pinned_mem_size = 0; + int64_t resource_num = 2; + }; + +public: + static FaissGpuResourceMgr & + GetInstance(); + + // Free gpu resource, avoid cudaGetDevice error when deallocate. + // this func should be invoke before main return + void + Free(); + + void + AllocateTempMem(ResPtr &resource, const int64_t& device_id, const int64_t& size); + + void + InitDevice(int64_t device_id, + int64_t pin_mem_size = 0, + int64_t temp_mem_size = 0, + int64_t res_num = 2); + + void + InitResource(); + + // allocate gpu memory invoke by build or copy_to_gpu + ResPtr + GetRes(const int64_t &device_id, const int64_t& alloc_size = 0); + + void + MoveToIdle(const int64_t &device_id, const ResPtr& res); + + void + Dump(); + +protected: + bool is_init = false; + + std::map> mutex_cache_; + std::map devices_params_; + std::map idle_map_; +}; + +class ResScope { +public: + ResScope(ResPtr &res, const int64_t& device_id, const bool& isown) + : resource(res), device_id(device_id), move(true), own(isown) { + Lock(); + } + + // specif for search + // get the ownership of gpuresource and gpu + ResScope(ResWPtr &res, const int64_t &device_id) + :device_id(device_id),move(false),own(true) { + resource = res.lock(); + Lock(); + } + + void Lock() { + if (own) FaissGpuResourceMgr::GetInstance().mutex_cache_[device_id]->lock(); + resource->mutex.lock(); + } + + ~ResScope() { + if (own) FaissGpuResourceMgr::GetInstance().mutex_cache_[device_id]->unlock(); + if (move) FaissGpuResourceMgr::GetInstance().MoveToIdle(device_id, resource); + resource->mutex.unlock(); + } + +private: + ResPtr resource; // hold resource until deconstruct + int64_t device_id; + bool move = true; + bool own = false; +}; + +} // knowhere +} // zilliz \ No newline at end of file diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp index fd04501c83..4efd5f24ae 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp @@ -67,9 +67,9 @@ void GPUIVF::set_index_model(IndexModelPtr model) { auto host_index = std::static_pointer_cast(model); if (auto gpures = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_)) { ResScope rs(gpures, gpu_id_, false); - res_ = gpures; - auto device_index = faiss::gpu::index_cpu_to_gpu(res_->faiss_res.get(), gpu_id_, host_index->index_.get()); + auto device_index = faiss::gpu::index_cpu_to_gpu(gpures->faiss_res.get(), gpu_id_, host_index->index_.get()); index_.reset(device_index); + res_ = gpures; } else { KNOWHERE_THROW_MSG("load index model error, can't get gpu_resource"); } @@ -114,9 +114,9 @@ void GPUIVF::LoadImpl(const BinarySet &index_binary) { if (auto temp_res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_)) { ResScope rs(temp_res, gpu_id_, false); - res_ = temp_res; - auto device_index = faiss::gpu::index_cpu_to_gpu(res_->faiss_res.get(), gpu_id_, index); + auto device_index = faiss::gpu::index_cpu_to_gpu(temp_res->faiss_res.get(), gpu_id_, index); index_.reset(device_index); + res_ = temp_res; } else { KNOWHERE_THROW_MSG("Load error, can't get gpu resource"); } @@ -176,12 +176,13 @@ VectorIndexPtr GPUIVF::CopyGpuToGpu(const int64_t &device_id, const Config &conf auto host_index = CopyGpuToCpu(config); return std::static_pointer_cast(host_index)->CopyCpuToGpu(device_id, config); } + void GPUIVF::Add(const DatasetPtr &dataset, const Config &config) { - auto temp_resource = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_); - if (temp_resource != nullptr) { - ResScope rs(temp_resource, gpu_id_, true); + if (auto spt = res_.lock()) { + ResScope rs(res_, gpu_id_); IVF::Add(dataset, config); - } else { + } + else { KNOWHERE_THROW_MSG("Add IVF can't get gpu resource"); } } @@ -264,108 +265,6 @@ VectorIndexPtr GPUIVFSQ::CopyGpuToCpu(const Config &config) { return std::make_shared(new_index); } -FaissGpuResourceMgr &FaissGpuResourceMgr::GetInstance() { - static FaissGpuResourceMgr instance; - return instance; -} - -void FaissGpuResourceMgr::AllocateTempMem(ResPtr &resource, - const int64_t &device_id, - const int64_t &size) { - if (size) { - resource->faiss_res->setTempMemory(size); - } - else { - auto search = devices_params_.find(device_id); - if (search != devices_params_.end()) { - resource->faiss_res->setTempMemory(search->second.temp_mem_size); - } - // else do nothing. allocate when use. - } -} - -void FaissGpuResourceMgr::InitDevice(int64_t device_id, - int64_t pin_mem_size, - int64_t temp_mem_size, - int64_t res_num) { - DeviceParams params; - params.pinned_mem_size = pin_mem_size; - params.temp_mem_size = temp_mem_size; - params.resource_num = res_num; - - devices_params_.emplace(device_id, params); -} - -void FaissGpuResourceMgr::InitResource() { - if(is_init) return ; - - is_init = true; - - //std::cout << "InitResource" << std::endl; - for(auto& device : devices_params_) { - auto& device_id = device.first; - - mutex_cache_.emplace(device_id, std::make_unique()); - - //std::cout << "Device Id: " << device_id << std::endl; - auto& device_param = device.second; - auto& bq = idle_map_[device_id]; - - for (int64_t i = 0; i < device_param.resource_num; ++i) { - //std::cout << "Resource Id: " << i << std::endl; - auto raw_resource = std::make_shared(); - - // TODO(linxj): enable set pinned memory - auto res_wrapper = std::make_shared(raw_resource); - AllocateTempMem(res_wrapper, device_id, 0); - - bq.Put(res_wrapper); - } - } - //std::cout << "End initResource" << std::endl; -} - -ResPtr FaissGpuResourceMgr::GetRes(const int64_t &device_id, - const int64_t &alloc_size) { - InitResource(); - - auto finder = idle_map_.find(device_id); - if (finder != idle_map_.end()) { - auto& bq = finder->second; - auto&& resource = bq.Take(); - AllocateTempMem(resource, device_id, alloc_size); - return resource; - } - return nullptr; -} - -void FaissGpuResourceMgr::MoveToIdle(const int64_t &device_id, const ResPtr &res) { - auto finder = idle_map_.find(device_id); - if (finder != idle_map_.end()) { - auto& bq = finder->second; - bq.Put(res); - } -} - -void FaissGpuResourceMgr::Free() { - for (auto &item : idle_map_) { - auto& bq = item.second; - while (!bq.Empty()) { - bq.Take(); - } - } - is_init = false; -} - -void -FaissGpuResourceMgr::Dump() { - for (auto &item : idle_map_) { - auto& bq = item.second; - std::cout << "device_id: " << item.first - << ", resource count:" << bq.Size(); - } -} - void GPUIndex::SetGpuDevice(const int &gpu_id) { gpu_id_ = gpu_id; } diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h index 429a94167a..4948d16ac4 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h @@ -18,118 +18,18 @@ #pragma once -#include #include "ivf.h" -#include "src/utils/BlockingQueue.h" +#include "FaissGpuResourceMgr.h" namespace zilliz { namespace knowhere { -struct Resource { - explicit Resource(std::shared_ptr &r): faiss_res(r) { - static int64_t global_id = 0; - id = global_id++; - } - - std::shared_ptr faiss_res; - int64_t id; - std::mutex mutex; -}; -using ResPtr = std::shared_ptr; -using ResWPtr = std::weak_ptr; - -class FaissGpuResourceMgr { - public: - friend class ResScope; - - public: - using ResBQ = zilliz::milvus::server::BlockingQueue; - - struct DeviceParams { - int64_t temp_mem_size = 0; - int64_t pinned_mem_size = 0; - int64_t resource_num = 2; - }; - - public: - static FaissGpuResourceMgr & - GetInstance(); - - // Free gpu resource, avoid cudaGetDevice error when deallocate. - // this func should be invoke before main return - void - Free(); - - void - AllocateTempMem(ResPtr &resource, const int64_t& device_id, const int64_t& size); - - void - InitDevice(int64_t device_id, - int64_t pin_mem_size = 0, - int64_t temp_mem_size = 0, - int64_t res_num = 2); - - void - InitResource(); - - // allocate gpu memory invoke by build or copy_to_gpu - ResPtr - GetRes(const int64_t &device_id, const int64_t& alloc_size = 0); - - // allocate gpu memory before search - // this func will return True if the device is idle and exists an idle resource. - //bool - //GetRes(const int64_t& device_id, ResPtr &res, const int64_t& alloc_size = 0); - - void - MoveToIdle(const int64_t &device_id, const ResPtr& res); - - void - Dump(); - - protected: - bool is_init = false; - - std::map> mutex_cache_; - std::map devices_params_; - std::map idle_map_; -}; - -class ResScope { - public: - ResScope(ResPtr &res, const int64_t& device_id, const bool& isown) - : resource(res), device_id(device_id), move(true), own(isown) { - if (isown) FaissGpuResourceMgr::GetInstance().mutex_cache_[device_id]->lock(); - res->mutex.lock(); - } - - // specif for search - // get the ownership of gpuresource and gpu - ResScope(ResPtr &res, const int64_t &device_id) - : resource(res), device_id(device_id), move(false), own(true) { - FaissGpuResourceMgr::GetInstance().mutex_cache_[device_id]->lock(); - res->mutex.lock(); - } - - ~ResScope() { - if (own) FaissGpuResourceMgr::GetInstance().mutex_cache_[device_id]->unlock(); - if (move) FaissGpuResourceMgr::GetInstance().MoveToIdle(device_id, resource); - resource->mutex.unlock(); - } - - private: - ResPtr resource; - int64_t device_id; - bool move = true; - bool own = false; -}; - class GPUIndex { public: explicit GPUIndex(const int &device_id) : gpu_id_(device_id) {} - GPUIndex(const int& device_id, ResPtr resource): gpu_id_(device_id), res_(std::move(resource)){} + GPUIndex(const int& device_id, const ResPtr& resource): gpu_id_(device_id), res_(resource){} virtual VectorIndexPtr CopyGpuToCpu(const Config &config) = 0; virtual VectorIndexPtr CopyGpuToGpu(const int64_t &device_id, const Config &config) = 0; @@ -139,7 +39,7 @@ class GPUIndex { protected: int64_t gpu_id_; - ResPtr res_ = nullptr; + ResWPtr res_; }; class GPUIVF : public IVF, public GPUIndex { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp index b4085e6df8..494da4438a 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp @@ -224,9 +224,9 @@ void GPUIDMAP::LoadImpl(const BinarySet &index_binary) { if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_) ){ ResScope rs(res, gpu_id_, false); - res_ = res; auto device_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), gpu_id_, index); index_.reset(device_index); + res_ = res; } else { KNOWHERE_THROW_MSG("Load error, can't get gpu resource"); } diff --git a/cpp/src/core/test/CMakeLists.txt b/cpp/src/core/test/CMakeLists.txt index 5b8dca2e4b..b62fd6cda3 100644 --- a/cpp/src/core/test/CMakeLists.txt +++ b/cpp/src/core/test/CMakeLists.txt @@ -32,6 +32,7 @@ set(ivf_srcs ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp ${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp ${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.cpp utils.cpp ) if(NOT TARGET test_ivf) @@ -48,6 +49,7 @@ set(idmap_srcs ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp ${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp ${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.cpp utils.cpp ) if(NOT TARGET test_idmap) diff --git a/cpp/src/wrapper/KnowhereResource.cpp b/cpp/src/wrapper/KnowhereResource.cpp index afbeb81816..c8c03423a8 100644 --- a/cpp/src/wrapper/KnowhereResource.cpp +++ b/cpp/src/wrapper/KnowhereResource.cpp @@ -17,6 +17,7 @@ #include "KnowhereResource.h" +#include "knowhere/index/vector_index/FaissGpuResourceMgr.h" #include "server/ServerConfig.h" #include diff --git a/cpp/src/wrapper/KnowhereResource.h b/cpp/src/wrapper/KnowhereResource.h index c14234eaed..5a7b9aa6c5 100644 --- a/cpp/src/wrapper/KnowhereResource.h +++ b/cpp/src/wrapper/KnowhereResource.h @@ -19,7 +19,6 @@ #pragma once #include "utils/Error.h" -#include "knowhere/index/vector_index/gpu_ivf.h" namespace zilliz { namespace milvus { From dc71feababde36c539e9a55646d919978fffa54e Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Thu, 19 Sep 2019 14:52:25 +0800 Subject: [PATCH 046/217] use better name Former-commit-id: ccf466b6906067d03cb62f0a813a3d7b4c346e2c --- cpp/src/server/Server.h | 1 - cpp/src/utils/SignalUtil.cpp | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cpp/src/server/Server.h b/cpp/src/server/Server.h index 2c912c3e91..30ea95a7bb 100644 --- a/cpp/src/server/Server.h +++ b/cpp/src/server/Server.h @@ -21,7 +21,6 @@ #include #include -#include namespace zilliz { diff --git a/cpp/src/utils/SignalUtil.cpp b/cpp/src/utils/SignalUtil.cpp index 8677016cf3..6c68f702d2 100644 --- a/cpp/src/utils/SignalUtil.cpp +++ b/cpp/src/utils/SignalUtil.cpp @@ -34,8 +34,8 @@ void SignalUtil::HandleSignal(int signum) { case SIGUSR2: { SERVER_LOG_INFO << "Server received signal: " << signum; - server::Server &server_ptr = server::Server::Instance(); - server_ptr.Stop(); + server::Server &server = server::Server::Instance(); + server.Stop(); exit(0); } @@ -43,8 +43,8 @@ void SignalUtil::HandleSignal(int signum) { SERVER_LOG_INFO << "Server received critical signal: " << signum; SignalUtil::PrintStacktrace(); - server::Server &server_ptr = server::Server::Instance(); - server_ptr.Stop(); + server::Server &server = server::Server::Instance(); + server.Stop(); exit(1); } From 029d4a97b3967e2ac6f150562c7f2a15bca7fec6 Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Thu, 19 Sep 2019 15:12:46 +0800 Subject: [PATCH 047/217] MS-573 Enable log in knowhere Former-commit-id: 7b60e5fc6edcaabea9e967ba4f3312a17533c986 --- cpp/src/core/test/CMakeLists.txt | 9 ++++++--- cpp/src/core/test/test_nsg/CMakeLists.txt | 1 + cpp/src/core/test/utils.cpp | 9 +++++++++ cpp/src/core/test/utils.h | 3 +++ 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/cpp/src/core/test/CMakeLists.txt b/cpp/src/core/test/CMakeLists.txt index 5b8dca2e4b..d70007229b 100644 --- a/cpp/src/core/test/CMakeLists.txt +++ b/cpp/src/core/test/CMakeLists.txt @@ -22,6 +22,9 @@ set(basic_libs gomp gfortran pthread ) +set(util_srcs + ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc + ) # set(ivf_srcs @@ -35,7 +38,7 @@ set(ivf_srcs utils.cpp ) if(NOT TARGET test_ivf) - add_executable(test_ivf test_ivf.cpp ${ivf_srcs}) + add_executable(test_ivf test_ivf.cpp ${ivf_srcs} ${util_srcs}) endif() target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs}) @@ -51,7 +54,7 @@ set(idmap_srcs utils.cpp ) if(NOT TARGET test_idmap) - add_executable(test_idmap test_idmap.cpp ${idmap_srcs}) + add_executable(test_idmap test_idmap.cpp ${idmap_srcs} ${util_srcs}) endif() target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs}) @@ -68,7 +71,7 @@ set(kdt_srcs utils.cpp ) if(NOT TARGET test_kdt) - add_executable(test_kdt test_kdt.cpp ${kdt_srcs}) + add_executable(test_kdt test_kdt.cpp ${kdt_srcs} ${util_srcs}) endif() target_link_libraries(test_kdt SPTAGLibStatic diff --git a/cpp/src/core/test/test_nsg/CMakeLists.txt b/cpp/src/core/test/test_nsg/CMakeLists.txt index 9ee8e76525..17b62cd40d 100644 --- a/cpp/src/core/test/test_nsg/CMakeLists.txt +++ b/cpp/src/core/test/test_nsg/CMakeLists.txt @@ -34,6 +34,7 @@ if(NOT TARGET test_nsg) test_nsg.cpp ${interface_src} ${nsg_src} + ${util_srcs} ) endif() diff --git a/cpp/src/core/test/utils.cpp b/cpp/src/core/test/utils.cpp index 97f4600ca7..4e240f5f74 100644 --- a/cpp/src/core/test/utils.cpp +++ b/cpp/src/core/test/utils.cpp @@ -18,6 +18,15 @@ #include "utils.h" +INITIALIZE_EASYLOGGINGPP + +void InitLog() { + el::Configurations defaultConf; + defaultConf.setToDefault(); + defaultConf.set(el::Level::Debug, + el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)"); + el::Loggers::reconfigureLogger("default", defaultConf); +} void DataGen::Init_with_default() { Generate(dim, nb, nq); diff --git a/cpp/src/core/test/utils.h b/cpp/src/core/test/utils.h index b9db01de51..df7f11d689 100644 --- a/cpp/src/core/test/utils.h +++ b/cpp/src/core/test/utils.h @@ -24,6 +24,7 @@ #include #include "knowhere/adapter/structure.h" +#include "knowhere/common/Log.h" class DataGen { protected: @@ -65,6 +66,8 @@ extern void GenBase(const int64_t &dim, float *xb, int64_t *ids); +extern void InitLog(); + zilliz::knowhere::DatasetPtr generate_dataset(int64_t nb, int64_t dim, float *xb, long *ids); From 6ed8251c3b3da41782a67989d9dafd4b570937ef Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 19 Sep 2019 15:15:35 +0800 Subject: [PATCH 048/217] fix unitest failed issue Former-commit-id: b4f12f185be2a302c25fa04695966f49c2a446ed --- cpp/unittest/db/mem_test.cpp | 59 +++++++++++++++++++----------------- cpp/unittest/db/utils.cpp | 8 +++-- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 4e958795ad..b85718a865 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -63,8 +63,8 @@ void BuildVectors(int64_t n, std::vector &vectors) { vectors.resize(n * TABLE_DIM); float *data = vectors.data(); for (int i = 0; i < n; i++) { - for (int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); - data[TABLE_DIM * i] += i / 2000.; + for (int j = 0; j < TABLE_DIM; j++) + data[TABLE_DIM * i + j] = drand48(); } } } @@ -224,39 +224,42 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) { ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - std::map> search_vectors; - { - engine::IDNumbers vector_ids; - int64_t nb = 100000; - std::vector xb; - BuildVectors(nb, xb); - stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); - ASSERT_TRUE(stat.ok()); + int64_t nb = 100000; + std::vector xb; + BuildVectors(nb, xb); - std::this_thread::sleep_for(std::chrono::seconds(3)); - - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_int_distribution dis(0, nb - 1); - - int64_t num_query = 10; - for (int64_t i = 0; i < num_query; ++i) { - int64_t index = dis(gen); - std::vector search; - for (int64_t j = 0; j < TABLE_DIM; j++) { - search.push_back(xb[index * TABLE_DIM + j]); - } - search_vectors.insert(std::make_pair(vector_ids[index], search)); - } + engine::IDNumbers vector_ids; + for(int64_t i = 0; i < nb; i++) { + vector_ids.push_back(i); } - int k = 10; + stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + ASSERT_TRUE(stat.ok()); + + std::this_thread::sleep_for(std::chrono::seconds(3));//ensure raw data write to disk + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dis(0, nb - 1); + + int64_t num_query = 10; + std::map> search_vectors; + for (int64_t i = 0; i < num_query; ++i) { + int64_t index = dis(gen); + std::vector search; + for (int64_t j = 0; j < TABLE_DIM; j++) { + search.push_back(xb[index * TABLE_DIM + j]); + } + search_vectors.insert(std::make_pair(vector_ids[index], search)); + } + + int topk = 10, nprobe = 10; for (auto &pair : search_vectors) { auto &search = pair.second; engine::QueryResults results; - stat = db_->Query(TABLE_NAME, k, 1, 10, search.data(), results); + stat = db_->Query(TABLE_NAME, topk, 1, nprobe, search.data(), results); ASSERT_EQ(results[0][0].first, pair.first); - ASSERT_LT(results[0][0].second, 0.00001); + ASSERT_LT(results[0][0].second, 1e-4); } } diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 982e98abd3..03ea11bf08 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -21,6 +21,8 @@ #include #include "utils.h" +#include "cache/GpuCacheMgr.h" +#include "cache/CpuCacheMgr.h" #include "db/DBFactory.h" #include "db/Options.h" #include "server/ServerConfig.h" @@ -63,6 +65,8 @@ void BaseTest::SetUp() { } void BaseTest::TearDown() { + zilliz::milvus::cache::CpuCacheMgr::GetInstance()->ClearCache(); + zilliz::milvus::cache::GpuCacheMgr::GetInstance(0)->ClearCache(); zilliz::knowhere::FaissGpuResourceMgr::GetInstance().Free(); } @@ -100,12 +104,12 @@ void DBTest::TearDown() { db_->Stop(); db_->DropAll(); - BaseTest::TearDown(); - engine::JobMgrInst::GetInstance()->Stop(); engine::SchedInst::GetInstance()->Stop(); engine::ResMgrInst::GetInstance()->Stop(); + BaseTest::TearDown(); + auto options = GetOptions(); boost::filesystem::remove_all(options.meta.path); } From e26948e85859e7f5b327b7a238c20f92dcc2c421 Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Thu, 19 Sep 2019 15:18:37 +0800 Subject: [PATCH 049/217] MS-573 remove useless code Former-commit-id: 4db9733a33dcbc26b5b11a66f54087d3af51ccc6 --- cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp index 51fe237658..fd04501c83 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp @@ -30,7 +30,6 @@ #include "cloner.h" #include "knowhere/adapter/faiss_adopt.h" #include "gpu_ivf.h" -#include "knowhere/common/Log.h" #include @@ -224,7 +223,6 @@ VectorIndexPtr GPUIVFPQ::CopyGpuToCpu(const Config &config) { } IndexModelPtr GPUIVFSQ::Train(const DatasetPtr &dataset, const Config &config) { - KNOWHERE_LOG_WARNING << "lalalalalalala"; auto nlist = config["nlist"].as(); auto nbits = config["nbits"].as(); // TODO(linxj): gpu only support SQ4 SQ8 SQ16 gpu_id_ = config.get_with_default("gpu_id", gpu_id_); From c72d8ac14b50d14c59825361382eea0ab42b628e Mon Sep 17 00:00:00 2001 From: test Date: Thu, 19 Sep 2019 08:25:12 +0000 Subject: [PATCH 050/217] MS-575 Add Clang-format & Clang-tidy & Cpplint Former-commit-id: 226d8cc0deeb9062227da6f36d3d684d80ca4d18 --- cpp/CHANGELOG.md | 1 + cpp/CMakeLists.txt | 14 ++- cpp/build-support/clang_format_exclusions.txt | 0 cpp/build-support/run_clang_format.py | 79 +++++++++++++ cpp/cmake/FindClangTools.cmake | 104 ++++++++++++++++++ 5 files changed, 196 insertions(+), 2 deletions(-) create mode 100644 cpp/build-support/clang_format_exclusions.txt create mode 100755 cpp/build-support/run_clang_format.py create mode 100644 cpp/cmake/FindClangTools.cmake diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 820b16756b..60374fa310 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -25,6 +25,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-561 - Add contributing guidelines, code of conduct and README docs - MS-567 - Add NOTICE.md - MS-569 - Complete the NOTICE.md +- MS-575 - Add Clang-format & Clang-tidy & Cpplint # Milvus 0.4.0 (2019-09-12) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index bb15ddf075..01007bf085 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -21,6 +21,8 @@ cmake_minimum_required(VERSION 3.14) message(STATUS "Building using CMake version: ${CMAKE_VERSION}") +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + MACRO (GET_CURRENT_TIME CURRENT_TIME) execute_process(COMMAND "date" +"%Y-%m-%d %H:%M.%S" OUTPUT_VARIABLE ${CURRENT_TIME}) ENDMACRO (GET_CURRENT_TIME) @@ -41,6 +43,10 @@ endif() set(MILVUS_VERSION "${GIT_BRANCH_NAME}") string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}") +set(CLANG_FORMAT_VERSION "6.0") +find_package(ClangTools) +set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support") + if(CMAKE_BUILD_TYPE STREQUAL "Release") set(BUILD_TYPE "Release") else() @@ -111,8 +117,6 @@ else() include_directories(${MYSQL_INCLUDE_DIR}) endif() -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") - set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR}) set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR}) set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) @@ -152,3 +156,9 @@ install(FILES conf/log_config.conf DESTINATION conf) + +add_custom_target(format ${BUILD_SUPPORT_DIR}/run_clang_format.py + ${CLANG_FORMAT_BIN} + ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt + ${CMAKE_CURRENT_SOURCE_DIR}/src) + diff --git a/cpp/build-support/clang_format_exclusions.txt b/cpp/build-support/clang_format_exclusions.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cpp/build-support/run_clang_format.py b/cpp/build-support/run_clang_format.py new file mode 100755 index 0000000000..4235b2d4af --- /dev/null +++ b/cpp/build-support/run_clang_format.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import fnmatch +import os +import subprocess +import sys + +if len(sys.argv) < 4: + sys.stderr.write("Usage: %s $CLANG_FORMAT_VERSION exclude_globs.txt " + "$source_dir\n" % + sys.argv[0]) + sys.exit(1) + +CLANG_FORMAT = sys.argv[1] +EXCLUDE_GLOBS_FILENAME = sys.argv[2] +SOURCE_DIR = sys.argv[3] + +if len(sys.argv) > 4: + CHECK_FORMAT = int(sys.argv[4]) == 1 +else: + CHECK_FORMAT = False + + +exclude_globs = [line.strip() for line in open(EXCLUDE_GLOBS_FILENAME, "r")] + +files_to_format = [] +matches = [] +for directory, subdirs, files in os.walk(SOURCE_DIR): + for name in files: + name = os.path.join(directory, name) + if not (name.endswith('.h') or name.endswith('.cpp') or name.endswith('.cuh') or name.endswith('.cu')): + continue + + excluded = False + for g in exclude_globs: + if fnmatch.fnmatch(name, g): + excluded = True + break + if not excluded: + files_to_format.append(name) + +if CHECK_FORMAT: + output = subprocess.check_output([CLANG_FORMAT, '-output-replacements-xml'] + + files_to_format, + stderr=subprocess.STDOUT).decode('utf8') + + to_fix = [] + for line in output.split('\n'): + if 'offset' in line: + to_fix.append(line) + + if len(to_fix) > 0: + print("clang-format checks failed, run 'make format' to fix") + sys.exit(-1) +else: + try: + cmd = [CLANG_FORMAT, '-i'] + files_to_format + subprocess.check_output(cmd, stderr=subprocess.STDOUT) + except Exception as e: + print(e) + print(' '.join(cmd)) + raise + diff --git a/cpp/cmake/FindClangTools.cmake b/cpp/cmake/FindClangTools.cmake new file mode 100644 index 0000000000..075d9381c5 --- /dev/null +++ b/cpp/cmake/FindClangTools.cmake @@ -0,0 +1,104 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Tries to find the clang-tidy and clang-format modules +# +# Usage of this module as follows: +# +# find_package(ClangTools) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# ClangToolsBin_HOME - +# When set, this path is inspected instead of standard library binary locations +# to find clang-tidy and clang-format +# +# This module defines +# CLANG_TIDY_BIN, The path to the clang tidy binary +# CLANG_TIDY_FOUND, Whether clang tidy was found +# CLANG_FORMAT_BIN, The path to the clang format binary +# CLANG_TIDY_FOUND, Whether clang format was found + +find_program(CLANG_TIDY_BIN + NAMES + clang-tidy-6.0 + clang-tidy-5.0 + clang-tidy-4.0 + clang-tidy-3.9 + clang-tidy-3.8 + clang-tidy-3.7 + clang-tidy-3.6 + clang-tidy + PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin + NO_DEFAULT_PATH +) + +if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" ) + set(CLANG_TIDY_FOUND 0) + message("clang-tidy not found") +else() + set(CLANG_TIDY_FOUND 1) + message("clang-tidy found at ${CLANG_TIDY_BIN}") +endif() + +if (CLANG_FORMAT_VERSION) + find_program(CLANG_FORMAT_BIN + NAMES clang-format-${CLANG_FORMAT_VERSION} + PATHS + ${ClangTools_PATH} + $ENV{CLANG_TOOLS_PATH} + /usr/local/bin /usr/bin + NO_DEFAULT_PATH + ) + + # If not found yet, search alternative locations + if (("${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND") AND APPLE) + # Homebrew ships older LLVM versions in /usr/local/opt/llvm@version/ + STRING(REGEX REPLACE "^([0-9]+)\\.[0-9]+" "\\1" CLANG_FORMAT_MAJOR_VERSION "${CLANG_FORMAT_VERSION}") + STRING(REGEX REPLACE "^[0-9]+\\.([0-9]+)" "\\1" CLANG_FORMAT_MINOR_VERSION "${CLANG_FORMAT_VERSION}") + if ("${CLANG_FORMAT_MINOR_VERSION}" STREQUAL "0") + find_program(CLANG_FORMAT_BIN + NAMES clang-format + PATHS /usr/local/opt/llvm@${CLANG_FORMAT_MAJOR_VERSION}/bin + NO_DEFAULT_PATH + ) + else() + find_program(CLANG_FORMAT_BIN + NAMES clang-format + PATHS /usr/local/opt/llvm@${CLANG_FORMAT_VERSION}/bin + NO_DEFAULT_PATH + ) + endif() + endif() +else() + find_program(CLANG_FORMAT_BIN + NAMES clang-format-4.0 + clang-format-3.9 + clang-format-3.8 + clang-format-3.7 + clang-format-3.6 + clang-format + PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin + NO_DEFAULT_PATH + ) +endif() + +if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" ) + set(CLANG_FORMAT_FOUND 0) + message("clang-format not found") +else() + set(CLANG_FORMAT_FOUND 1) + message("clang-format found at ${CLANG_FORMAT_BIN}") +endif() + From 75ea4c4add2af8353e3cd15c8aad04945bad00c4 Mon Sep 17 00:00:00 2001 From: wxyu Date: Thu, 19 Sep 2019 17:28:15 +0800 Subject: [PATCH 051/217] MS-577 Unittest Query randomly hung Former-commit-id: 5e4365deea822861077f41cda17d3cbd8504f327 --- cpp/CHANGELOG.md | 1 + cpp/src/scheduler/JobMgr.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 113045897f..ab630f124a 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -5,6 +5,7 @@ Please mark all change in change log and use the ticket from JIRA. # Milvus 0.5.0 (TODO) ## Bug +- MS-577 - Unittest Query randomly hung ## Improvement - MS-552 - Add and change the easylogging library diff --git a/cpp/src/scheduler/JobMgr.cpp b/cpp/src/scheduler/JobMgr.cpp index c5c14c1b41..31db4b344e 100644 --- a/cpp/src/scheduler/JobMgr.cpp +++ b/cpp/src/scheduler/JobMgr.cpp @@ -32,8 +32,8 @@ JobMgr::JobMgr(ResourceMgrPtr res_mgr) void JobMgr::Start() { if (not running_) { - worker_thread_ = std::thread(&JobMgr::worker_function, this); running_ = true; + worker_thread_ = std::thread(&JobMgr::worker_function, this); } } From 44c382da83b7b039936339544d65b740e5bbb73d Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 19 Sep 2019 18:13:48 +0800 Subject: [PATCH 052/217] refine code Former-commit-id: 673cf5d86bf86f50cfdfa3576c3dbd20204487b9 --- cpp/unittest/server/rpc_test.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cpp/unittest/server/rpc_test.cpp b/cpp/unittest/server/rpc_test.cpp index f89c98eb9d..4fecbd3880 100644 --- a/cpp/unittest/server/rpc_test.cpp +++ b/cpp/unittest/server/rpc_test.cpp @@ -146,7 +146,7 @@ std::string CurrentTmDate(int64_t offset_day = 0) { } -TEST_F(RpcHandlerTest, HasTableTest) { +TEST_F(RpcHandlerTest, HAS_TABLE_TEST) { ::grpc::ServerContext context; ::milvus::grpc::TableName request; ::milvus::grpc::BoolReply reply; @@ -158,7 +158,7 @@ TEST_F(RpcHandlerTest, HasTableTest) { ASSERT_EQ(error_code, ::milvus::grpc::ErrorCode::SUCCESS); } -TEST_F(RpcHandlerTest, IndexTest) { +TEST_F(RpcHandlerTest, INDEX_TEST) { ::grpc::ServerContext context; ::milvus::grpc::IndexParam request; ::milvus::grpc::Status response; @@ -194,7 +194,7 @@ TEST_F(RpcHandlerTest, IndexTest) { handler->DropIndex(&context, &table_name, &status); } -TEST_F(RpcHandlerTest, InsertTest) { +TEST_F(RpcHandlerTest, INSERT_TEST) { ::grpc::ServerContext context; ::milvus::grpc::InsertParam request; ::milvus::grpc::Status response; @@ -213,7 +213,7 @@ TEST_F(RpcHandlerTest, InsertTest) { ASSERT_EQ(vector_ids.vector_id_array_size(), VECTOR_COUNT); } -TEST_F(RpcHandlerTest, SearchTest) { +TEST_F(RpcHandlerTest, SEARCH_TEST) { ::grpc::ServerContext context; ::milvus::grpc::SearchParam request; ::milvus::grpc::TopKQueryResultList response; @@ -281,7 +281,7 @@ TEST_F(RpcHandlerTest, SearchTest) { handler->SearchInFiles(&context, &search_in_files_param, &response); } -TEST_F(RpcHandlerTest, TablesTest) { +TEST_F(RpcHandlerTest, TABLES_TEST) { ::grpc::ServerContext context; ::milvus::grpc::TableSchema tableschema; ::milvus::grpc::Status response; @@ -387,7 +387,7 @@ TEST_F(RpcHandlerTest, TablesTest) { ASSERT_EQ(error_code, ::milvus::grpc::ErrorCode::SUCCESS); } -TEST_F(RpcHandlerTest, CmdTest) { +TEST_F(RpcHandlerTest, CMD_TEST) { ::grpc::ServerContext context; ::milvus::grpc::Command command; command.set_cmd("version"); @@ -401,7 +401,7 @@ TEST_F(RpcHandlerTest, CmdTest) { handler->Cmd(&context, &command, &reply); } -TEST_F(RpcHandlerTest, DeleteByRangeTest) { +TEST_F(RpcHandlerTest, DELETE_BY_RANGE_TEST) { ::grpc::ServerContext context; ::milvus::grpc::DeleteByRangeParam request; ::milvus::grpc::Status status; @@ -459,7 +459,7 @@ protected: } -TEST_F(RpcSchedulerTest, BaseTaskTest){ +TEST_F(RpcSchedulerTest, BASE_TASK_TEST){ auto status = task_ptr->Execute(); ASSERT_TRUE(status.ok()); From 7e5870466208efde4eb3a1b73375973a5759f29f Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 19 Sep 2019 18:44:20 +0800 Subject: [PATCH 053/217] fix unittest hang issue Former-commit-id: 1e6b431332c823fbbef29132eaaaaff86b8afe7e --- cpp/unittest/server/rpc_test.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/unittest/server/rpc_test.cpp b/cpp/unittest/server/rpc_test.cpp index 4fecbd3880..69b258aa75 100644 --- a/cpp/unittest/server/rpc_test.cpp +++ b/cpp/unittest/server/rpc_test.cpp @@ -62,6 +62,7 @@ class RpcHandlerTest : public testing::Test { res_mgr->Connect("cpu", "gtx1660", PCIE); res_mgr->Start(); engine::SchedInst::GetInstance()->Start(); + engine::JobMgrInst::GetInstance()->Start(); engine::DBOptions opt; @@ -105,6 +106,7 @@ class RpcHandlerTest : public testing::Test { void TearDown() override { server::DBWrapper::GetInstance().StopService(); + engine::JobMgrInst::GetInstance()->Stop(); engine::ResMgrInst::GetInstance()->Stop(); engine::SchedInst::GetInstance()->Stop(); boost::filesystem::remove_all("/tmp/milvus_test"); From b282580bf97506dd469a9b0390e6c58694facad9 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Thu, 19 Sep 2019 19:01:26 +0800 Subject: [PATCH 054/217] MS-565 rename knowhere files Former-commit-id: a7c2a8b1d99e6cf29bbcaf27b131e0ce5bd42f43 --- cpp/src/core/CMakeLists.txt | 2 +- cpp/src/core/knowhere/CMakeLists.txt | 34 +++++++++---------- .../adapter/{arrow.cpp => ArrowAdapter.cpp} | 2 +- .../adapter/{arrow.h => ArrowAdapter.h} | 2 +- .../adapter/{sptag.cpp => SptagAdapter.cpp} | 6 ++-- .../adapter/{sptag.h => SptagAdapter.h} | 2 +- .../adapter/{structure.cpp => Structure.cpp} | 2 +- .../adapter/{structure.h => Structure.h} | 2 +- .../{faiss_adopt.h => VectorAdapter.h} | 0 .../knowhere/common/{array.h => Array.h} | 5 +-- .../common/{binary_set.h => BinarySet.h} | 2 +- .../knowhere/common/{buffer.h => Buffer.h} | 2 +- .../knowhere/common/{config.h => Config.h} | 0 .../knowhere/common/{dataset.h => Dataset.h} | 12 +++---- .../common/{exception.cpp => Exception.cpp} | 2 +- .../common/{exception.h => Exception.h} | 0 .../knowhere/knowhere/common/{id.h => Id.h} | 0 cpp/src/core/knowhere/knowhere/common/Log.h | 2 ++ .../knowhere/common/{schema.h => Schema.h} | 2 +- .../knowhere/common/{tensor.h => Tensor.h} | 2 +- .../knowhere/common/{timer.cpp => Timer.cpp} | 2 +- .../knowhere/common/{timer.h => Timer.h} | 0 .../knowhere/index/{index.h => Index.h} | 10 +++--- .../index/{index_model.h => IndexModel.h} | 2 +- .../index/{index_type.h => IndexType.h} | 0 .../{normalize.cpp => Normalize.cpp} | 0 .../preprocessor/{normalize.h => Normalize.h} | 0 .../{preprocessor.h => Preprocessor.h} | 2 +- .../{gpu_ivf.cpp => IndexGPUIVF.cpp} | 10 +++--- .../vector_index/{gpu_ivf.h => IndexGPUIVF.h} | 4 +-- .../{idmap.cpp => IndexIDMAP.cpp} | 6 ++-- .../vector_index/{idmap.h => IndexIDMAP.h} | 4 +-- .../vector_index/{ivf.cpp => IndexIVF.cpp} | 9 +++-- .../index/vector_index/{ivf.h => IndexIVF.h} | 2 +- .../{cpu_kdt_rng.cpp => IndexKDT.cpp} | 12 +++---- .../{cpu_kdt_rng.h => IndexKDT.h} | 4 +-- .../{nsg_index.cpp => IndexNSG.cpp} | 18 +++++----- .../vector_index/{nsg_index.h => IndexNSG.h} | 2 +- .../{vector_index.h => VectorIndex.h} | 8 ++--- .../vector_index/nsg/{nsg.cpp => NSG.cpp} | 8 ++--- .../index/vector_index/nsg/{nsg.h => NSG.h} | 2 +- .../nsg/{utils.cpp => NSGHelper.cpp} | 2 +- .../vector_index/nsg/{utils.h => NSGHelper.h} | 4 +-- .../nsg/{nsg_io.cpp => NSGIO.cpp} | 2 +- .../vector_index/nsg/{nsg_io.h => NSGIO.h} | 4 +-- .../nsg/{neighbor.h => Neighbor.h} | 0 .../index/vector_index/nsg/index_io.h | 32 ----------------- .../{cloner.cpp => utils/Cloner.cpp} | 12 ++++--- .../vector_index/{cloner.h => utils/Cloner.h} | 8 +++-- .../{definitions.h => utils/Definitions.h} | 8 ++--- .../{ => utils}/FaissGpuResourceMgr.cpp | 0 .../{ => utils}/FaissGpuResourceMgr.h | 0 .../KDTParameterMgr.cpp} | 6 ++-- .../KDTParameterMgr.h} | 12 +++---- cpp/src/core/test/test_idmap.cpp | 8 ++--- cpp/src/core/test/test_ivf.cpp | 12 +++---- cpp/src/core/test/test_kdt.cpp | 10 +++--- cpp/src/core/test/utils.h | 2 +- cpp/src/db/engine/ExecutionEngineImpl.cpp | 2 +- cpp/src/scheduler/SchedInst.cpp | 2 +- cpp/src/wrapper/KnowhereResource.cpp | 2 +- cpp/src/wrapper/data_transfer.h | 2 +- cpp/src/wrapper/vec_impl.cpp | 12 +++---- cpp/src/wrapper/vec_impl.h | 2 +- cpp/src/wrapper/vec_index.cpp | 12 +++---- cpp/src/wrapper/vec_index.h | 4 +-- cpp/unittest/db/utils.cpp | 3 +- cpp/unittest/wrapper/wrapper_test.cpp | 3 +- 68 files changed, 164 insertions(+), 188 deletions(-) rename cpp/src/core/knowhere/knowhere/adapter/{arrow.cpp => ArrowAdapter.cpp} (98%) rename cpp/src/core/knowhere/knowhere/adapter/{arrow.h => ArrowAdapter.h} (96%) rename cpp/src/core/knowhere/knowhere/adapter/{sptag.cpp => SptagAdapter.cpp} (97%) rename cpp/src/core/knowhere/knowhere/adapter/{sptag.h => SptagAdapter.h} (97%) rename cpp/src/core/knowhere/knowhere/adapter/{structure.cpp => Structure.cpp} (99%) rename cpp/src/core/knowhere/knowhere/adapter/{structure.h => Structure.h} (97%) rename cpp/src/core/knowhere/knowhere/adapter/{faiss_adopt.h => VectorAdapter.h} (100%) rename cpp/src/core/knowhere/knowhere/common/{array.h => Array.h} (97%) rename cpp/src/core/knowhere/knowhere/common/{binary_set.h => BinarySet.h} (99%) rename cpp/src/core/knowhere/knowhere/common/{buffer.h => Buffer.h} (98%) rename cpp/src/core/knowhere/knowhere/common/{config.h => Config.h} (100%) rename cpp/src/core/knowhere/knowhere/common/{dataset.h => Dataset.h} (96%) rename cpp/src/core/knowhere/knowhere/common/{exception.cpp => Exception.cpp} (98%) rename cpp/src/core/knowhere/knowhere/common/{exception.h => Exception.h} (100%) rename cpp/src/core/knowhere/knowhere/common/{id.h => Id.h} (100%) rename cpp/src/core/knowhere/knowhere/common/{schema.h => Schema.h} (97%) rename cpp/src/core/knowhere/knowhere/common/{tensor.h => Tensor.h} (97%) rename cpp/src/core/knowhere/knowhere/common/{timer.cpp => Timer.cpp} (99%) rename cpp/src/core/knowhere/knowhere/common/{timer.h => Timer.h} (100%) rename cpp/src/core/knowhere/knowhere/index/{index.h => Index.h} (89%) rename cpp/src/core/knowhere/knowhere/index/{index_model.h => IndexModel.h} (96%) rename cpp/src/core/knowhere/knowhere/index/{index_type.h => IndexType.h} (100%) rename cpp/src/core/knowhere/knowhere/index/preprocessor/{normalize.cpp => Normalize.cpp} (100%) rename cpp/src/core/knowhere/knowhere/index/preprocessor/{normalize.h => Normalize.h} (100%) rename cpp/src/core/knowhere/knowhere/index/preprocessor/{preprocessor.h => Preprocessor.h} (96%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{gpu_ivf.cpp => IndexGPUIVF.cpp} (97%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{gpu_ivf.h => IndexGPUIVF.h} (97%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{idmap.cpp => IndexIDMAP.cpp} (98%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{idmap.h => IndexIDMAP.h} (98%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{ivf.cpp => IndexIVF.cpp} (98%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{ivf.h => IndexIVF.h} (99%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{cpu_kdt_rng.cpp => IndexKDT.cpp} (94%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{cpu_kdt_rng.h => IndexKDT.h} (97%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{nsg_index.cpp => IndexNSG.cpp} (94%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{nsg_index.h => IndexNSG.h} (98%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{vector_index.h => VectorIndex.h} (90%) rename cpp/src/core/knowhere/knowhere/index/vector_index/nsg/{nsg.cpp => NSG.cpp} (99%) rename cpp/src/core/knowhere/knowhere/index/vector_index/nsg/{nsg.h => NSG.h} (99%) rename cpp/src/core/knowhere/knowhere/index/vector_index/nsg/{utils.cpp => NSGHelper.cpp} (99%) rename cpp/src/core/knowhere/knowhere/index/vector_index/nsg/{utils.h => NSGHelper.h} (95%) rename cpp/src/core/knowhere/knowhere/index/vector_index/nsg/{nsg_io.cpp => NSGIO.cpp} (99%) rename cpp/src/core/knowhere/knowhere/index/vector_index/nsg/{nsg_io.h => NSGIO.h} (93%) rename cpp/src/core/knowhere/knowhere/index/vector_index/nsg/{neighbor.h => Neighbor.h} (100%) delete mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/nsg/index_io.h rename cpp/src/core/knowhere/knowhere/index/vector_index/{cloner.cpp => utils/Cloner.cpp} (89%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{cloner.h => utils/Cloner.h} (90%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{definitions.h => utils/Definitions.h} (93%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{ => utils}/FaissGpuResourceMgr.cpp (100%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{ => utils}/FaissGpuResourceMgr.h (100%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{kdt_parameters.cpp => utils/KDTParameterMgr.cpp} (93%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{kdt_parameters.h => utils/KDTParameterMgr.h} (80%) diff --git a/cpp/src/core/CMakeLists.txt b/cpp/src/core/CMakeLists.txt index bef3f5e2f6..b126cce031 100644 --- a/cpp/src/core/CMakeLists.txt +++ b/cpp/src/core/CMakeLists.txt @@ -93,7 +93,7 @@ endif() set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) if(BUILD_UNIT_TEST STREQUAL "ON") - add_subdirectory(test) +# add_subdirectory(test) endif() config_summary() diff --git a/cpp/src/core/knowhere/CMakeLists.txt b/cpp/src/core/knowhere/CMakeLists.txt index 55f23e1095..728e198f79 100644 --- a/cpp/src/core/knowhere/CMakeLists.txt +++ b/cpp/src/core/knowhere/CMakeLists.txt @@ -29,26 +29,26 @@ if(NOT TARGET SPTAGLibStatic) endif() set(external_srcs - knowhere/adapter/sptag.cpp - knowhere/adapter/structure.cpp - knowhere/adapter/arrow.cpp - knowhere/common/exception.cpp - knowhere/common/timer.cpp + knowhere/adapter/SptagAdapter.cpp + knowhere/adapter/Structure.cpp + knowhere/adapter/ArrowAdapter.cpp + knowhere/common/Exception.cpp + knowhere/common/Timer.cpp ) set(index_srcs - knowhere/index/preprocessor/normalize.cpp - knowhere/index/vector_index/cpu_kdt_rng.cpp - knowhere/index/vector_index/idmap.cpp - knowhere/index/vector_index/ivf.cpp - knowhere/index/vector_index/gpu_ivf.cpp - knowhere/index/vector_index/kdt_parameters.cpp - knowhere/index/vector_index/nsg_index.cpp - knowhere/index/vector_index/nsg/nsg.cpp - knowhere/index/vector_index/nsg/nsg_io.cpp - knowhere/index/vector_index/nsg/utils.cpp - knowhere/index/vector_index/cloner.cpp - knowhere/index/vector_index/FaissGpuResourceMgr.cpp + knowhere/index/preprocessor/Normalize.cpp + knowhere/index/vector_index/IndexKDT.cpp + knowhere/index/vector_index/IndexIDMAP.cpp + knowhere/index/vector_index/IndexIVF.cpp + knowhere/index/vector_index/IndexGPUIVF.cpp + knowhere/index/vector_index/utils/KDTParameterMgr.cpp + knowhere/index/vector_index/IndexNSG.cpp + knowhere/index/vector_index/nsg/NSG.cpp + knowhere/index/vector_index/nsg/NSGIO.cpp + knowhere/index/vector_index/nsg/NSGHelper.cpp + knowhere/index/vector_index/utils/Cloner.cpp + knowhere/index/vector_index/utils/FaissGpuResourceMgr.cpp ) set(depend_libs diff --git a/cpp/src/core/knowhere/knowhere/adapter/arrow.cpp b/cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.cpp similarity index 98% rename from cpp/src/core/knowhere/knowhere/adapter/arrow.cpp rename to cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.cpp index c4804d3322..c07cb16fca 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/arrow.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.cpp @@ -16,7 +16,7 @@ // under the License. -#include "arrow.h" +#include "ArrowAdapter.h" namespace zilliz { namespace knowhere { diff --git a/cpp/src/core/knowhere/knowhere/adapter/arrow.h b/cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.h similarity index 96% rename from cpp/src/core/knowhere/knowhere/adapter/arrow.h rename to cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.h index 318336e703..d19f8f3ae5 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/arrow.h +++ b/cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.h @@ -20,7 +20,7 @@ #include -#include "knowhere/common/array.h" +#include "knowhere/common/Array.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/adapter/sptag.cpp b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp similarity index 97% rename from cpp/src/core/knowhere/knowhere/adapter/sptag.cpp rename to cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp index 995d068577..3c5d466d41 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/sptag.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -#include "knowhere/index/vector_index/definitions.h" -#include "sptag.h" -#include "structure.h" +#include "knowhere/index/vector_index/utils/Definitions.h" +#include "SptagAdapter.h" +#include "Structure.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/adapter/sptag.h b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.h similarity index 97% rename from cpp/src/core/knowhere/knowhere/adapter/sptag.h rename to cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.h index eb7886c263..f47ffdc3b5 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/sptag.h +++ b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.h @@ -22,7 +22,7 @@ #include -#include "knowhere/common/dataset.h" +#include "knowhere/common/Dataset.h" namespace zilliz { namespace knowhere { diff --git a/cpp/src/core/knowhere/knowhere/adapter/structure.cpp b/cpp/src/core/knowhere/knowhere/adapter/Structure.cpp similarity index 99% rename from cpp/src/core/knowhere/knowhere/adapter/structure.cpp rename to cpp/src/core/knowhere/knowhere/adapter/Structure.cpp index 10b6af8138..18833b5d36 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/structure.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/Structure.cpp @@ -16,7 +16,7 @@ // under the License. -#include "structure.h" +#include "Structure.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/adapter/structure.h b/cpp/src/core/knowhere/knowhere/adapter/Structure.h similarity index 97% rename from cpp/src/core/knowhere/knowhere/adapter/structure.h rename to cpp/src/core/knowhere/knowhere/adapter/Structure.h index 76a03a23e4..7539dce2da 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/structure.h +++ b/cpp/src/core/knowhere/knowhere/adapter/Structure.h @@ -19,7 +19,7 @@ #pragma once #include -#include "knowhere/common/dataset.h" +#include "knowhere/common/Dataset.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/adapter/faiss_adopt.h b/cpp/src/core/knowhere/knowhere/adapter/VectorAdapter.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/adapter/faiss_adopt.h rename to cpp/src/core/knowhere/knowhere/adapter/VectorAdapter.h diff --git a/cpp/src/core/knowhere/knowhere/common/array.h b/cpp/src/core/knowhere/knowhere/common/Array.h similarity index 97% rename from cpp/src/core/knowhere/knowhere/common/array.h rename to cpp/src/core/knowhere/knowhere/common/Array.h index 1860524e8a..94a5470029 100644 --- a/cpp/src/core/knowhere/knowhere/common/array.h +++ b/cpp/src/core/knowhere/knowhere/common/Array.h @@ -18,8 +18,9 @@ #pragma once -#include "arrow/array.h" -#include "schema.h" +#include + +#include "Schema.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/common/binary_set.h b/cpp/src/core/knowhere/knowhere/common/BinarySet.h similarity index 99% rename from cpp/src/core/knowhere/knowhere/common/binary_set.h rename to cpp/src/core/knowhere/knowhere/common/BinarySet.h index 019c42a2b9..509db68e2d 100644 --- a/cpp/src/core/knowhere/knowhere/common/binary_set.h +++ b/cpp/src/core/knowhere/knowhere/common/BinarySet.h @@ -23,7 +23,7 @@ #include #include -#include "id.h" +#include "Id.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/common/buffer.h b/cpp/src/core/knowhere/knowhere/common/Buffer.h similarity index 98% rename from cpp/src/core/knowhere/knowhere/common/buffer.h rename to cpp/src/core/knowhere/knowhere/common/Buffer.h index 019c0487b7..4468e6ec01 100644 --- a/cpp/src/core/knowhere/knowhere/common/buffer.h +++ b/cpp/src/core/knowhere/knowhere/common/Buffer.h @@ -20,7 +20,7 @@ #include -#include "arrow/buffer.h" +#include namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/common/config.h b/cpp/src/core/knowhere/knowhere/common/Config.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/common/config.h rename to cpp/src/core/knowhere/knowhere/common/Config.h diff --git a/cpp/src/core/knowhere/knowhere/common/dataset.h b/cpp/src/core/knowhere/knowhere/common/Dataset.h similarity index 96% rename from cpp/src/core/knowhere/knowhere/common/dataset.h rename to cpp/src/core/knowhere/knowhere/common/Dataset.h index 068bbe4356..c2d74a59e5 100644 --- a/cpp/src/core/knowhere/knowhere/common/dataset.h +++ b/cpp/src/core/knowhere/knowhere/common/Dataset.h @@ -21,12 +21,12 @@ #include #include -#include "array.h" -#include "buffer.h" -#include "tensor.h" -#include "schema.h" -#include "config.h" -#include "knowhere/adapter/arrow.h" +#include "Array.h" +#include "Buffer.h" +#include "Tensor.h" +#include "Schema.h" +#include "Config.h" +#include "knowhere/adapter/ArrowAdapter.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/common/exception.cpp b/cpp/src/core/knowhere/knowhere/common/Exception.cpp similarity index 98% rename from cpp/src/core/knowhere/knowhere/common/exception.cpp rename to cpp/src/core/knowhere/knowhere/common/Exception.cpp index eb19e21bb0..a77e85ee41 100644 --- a/cpp/src/core/knowhere/knowhere/common/exception.cpp +++ b/cpp/src/core/knowhere/knowhere/common/Exception.cpp @@ -18,7 +18,7 @@ #include -#include "exception.h" +#include "Exception.h" #include "Log.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/common/exception.h b/cpp/src/core/knowhere/knowhere/common/Exception.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/common/exception.h rename to cpp/src/core/knowhere/knowhere/common/Exception.h diff --git a/cpp/src/core/knowhere/knowhere/common/id.h b/cpp/src/core/knowhere/knowhere/common/Id.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/common/id.h rename to cpp/src/core/knowhere/knowhere/common/Id.h diff --git a/cpp/src/core/knowhere/knowhere/common/Log.h b/cpp/src/core/knowhere/knowhere/common/Log.h index 0d99c624ab..1e390b3c1e 100644 --- a/cpp/src/core/knowhere/knowhere/common/Log.h +++ b/cpp/src/core/knowhere/knowhere/common/Log.h @@ -22,6 +22,7 @@ namespace zilliz { namespace knowhere { + #define KNOWHERE_DOMAIN_NAME "[KNOWHERE] " #define KNOWHERE_ERROR_TEXT "KNOWHERE Error:" @@ -31,5 +32,6 @@ namespace knowhere { #define KNOWHERE_LOG_WARNING LOG(WARNING) << KNOWHERE_DOMAIN_NAME #define KNOWHERE_LOG_ERROR LOG(ERROR) << KNOWHERE_DOMAIN_NAME #define KNOWHERE_LOG_FATAL LOG(FATAL) << KNOWHERE_DOMAIN_NAME + } // namespace knowhere } // namespace zilliz \ No newline at end of file diff --git a/cpp/src/core/knowhere/knowhere/common/schema.h b/cpp/src/core/knowhere/knowhere/common/Schema.h similarity index 97% rename from cpp/src/core/knowhere/knowhere/common/schema.h rename to cpp/src/core/knowhere/knowhere/common/Schema.h index 1ab3d42f8c..b43b7eb875 100644 --- a/cpp/src/core/knowhere/knowhere/common/schema.h +++ b/cpp/src/core/knowhere/knowhere/common/Schema.h @@ -20,7 +20,7 @@ #include -#include "arrow/type.h" +#include namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/common/tensor.h b/cpp/src/core/knowhere/knowhere/common/Tensor.h similarity index 97% rename from cpp/src/core/knowhere/knowhere/common/tensor.h rename to cpp/src/core/knowhere/knowhere/common/Tensor.h index 8b92f1f891..42e86dc4d1 100644 --- a/cpp/src/core/knowhere/knowhere/common/tensor.h +++ b/cpp/src/core/knowhere/knowhere/common/Tensor.h @@ -20,7 +20,7 @@ #include -#include "arrow/tensor.h" +#include namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/common/timer.cpp b/cpp/src/core/knowhere/knowhere/common/Timer.cpp similarity index 99% rename from cpp/src/core/knowhere/knowhere/common/timer.cpp rename to cpp/src/core/knowhere/knowhere/common/Timer.cpp index f57de535d4..7f2cb71514 100644 --- a/cpp/src/core/knowhere/knowhere/common/timer.cpp +++ b/cpp/src/core/knowhere/knowhere/common/Timer.cpp @@ -18,7 +18,7 @@ #include // TODO(linxj): using Log instead -#include "timer.h" +#include "Timer.h" namespace zilliz { namespace knowhere { diff --git a/cpp/src/core/knowhere/knowhere/common/timer.h b/cpp/src/core/knowhere/knowhere/common/Timer.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/common/timer.h rename to cpp/src/core/knowhere/knowhere/common/Timer.h diff --git a/cpp/src/core/knowhere/knowhere/index/index.h b/cpp/src/core/knowhere/knowhere/index/Index.h similarity index 89% rename from cpp/src/core/knowhere/knowhere/index/index.h rename to cpp/src/core/knowhere/knowhere/index/Index.h index 275a5fd387..4ccbef394c 100644 --- a/cpp/src/core/knowhere/knowhere/index/index.h +++ b/cpp/src/core/knowhere/knowhere/index/Index.h @@ -20,11 +20,11 @@ #include -#include "knowhere/common/binary_set.h" -#include "knowhere/common/dataset.h" -#include "index_type.h" -#include "index_model.h" -#include "knowhere/index/preprocessor/preprocessor.h" +#include "knowhere/common/BinarySet.h" +#include "knowhere/common/Dataset.h" +#include "IndexType.h" +#include "IndexModel.h" +#include "knowhere/index/preprocessor/Preprocessor.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/index_model.h b/cpp/src/core/knowhere/knowhere/index/IndexModel.h similarity index 96% rename from cpp/src/core/knowhere/knowhere/index/index_model.h rename to cpp/src/core/knowhere/knowhere/index/IndexModel.h index d00a629d7e..7557363f65 100644 --- a/cpp/src/core/knowhere/knowhere/index/index_model.h +++ b/cpp/src/core/knowhere/knowhere/index/IndexModel.h @@ -19,7 +19,7 @@ #pragma once #include -#include "knowhere/common/binary_set.h" +#include "knowhere/common/BinarySet.h" namespace zilliz { namespace knowhere { diff --git a/cpp/src/core/knowhere/knowhere/index/index_type.h b/cpp/src/core/knowhere/knowhere/index/IndexType.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/index/index_type.h rename to cpp/src/core/knowhere/knowhere/index/IndexType.h diff --git a/cpp/src/core/knowhere/knowhere/index/preprocessor/normalize.cpp b/cpp/src/core/knowhere/knowhere/index/preprocessor/Normalize.cpp similarity index 100% rename from cpp/src/core/knowhere/knowhere/index/preprocessor/normalize.cpp rename to cpp/src/core/knowhere/knowhere/index/preprocessor/Normalize.cpp diff --git a/cpp/src/core/knowhere/knowhere/index/preprocessor/normalize.h b/cpp/src/core/knowhere/knowhere/index/preprocessor/Normalize.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/index/preprocessor/normalize.h rename to cpp/src/core/knowhere/knowhere/index/preprocessor/Normalize.h diff --git a/cpp/src/core/knowhere/knowhere/index/preprocessor/preprocessor.h b/cpp/src/core/knowhere/knowhere/index/preprocessor/Preprocessor.h similarity index 96% rename from cpp/src/core/knowhere/knowhere/index/preprocessor/preprocessor.h rename to cpp/src/core/knowhere/knowhere/index/preprocessor/Preprocessor.h index 10f0544270..e1c01d2085 100644 --- a/cpp/src/core/knowhere/knowhere/index/preprocessor/preprocessor.h +++ b/cpp/src/core/knowhere/knowhere/index/preprocessor/Preprocessor.h @@ -20,7 +20,7 @@ #include -#include "knowhere/common/dataset.h" +#include "knowhere/common/Dataset.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp similarity index 97% rename from cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp index 4efd5f24ae..df26083108 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp @@ -26,10 +26,10 @@ #include -#include "knowhere/common/exception.h" -#include "cloner.h" -#include "knowhere/adapter/faiss_adopt.h" -#include "gpu_ivf.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/utils/Cloner.h" +#include "knowhere/adapter/VectorAdapter.h" +#include "IndexGPUIVF.h" #include @@ -169,7 +169,7 @@ VectorIndexPtr GPUIVF::CopyGpuToCpu(const Config &config) { VectorIndexPtr GPUIVF::Clone() { auto cpu_idx = CopyGpuToCpu(Config()); - return ::zilliz::knowhere::CopyCpuToGpu(cpu_idx, gpu_id_, Config()); + return ::zilliz::knowhere::cloner::CopyCpuToGpu(cpu_idx, gpu_id_, Config()); } VectorIndexPtr GPUIVF::CopyGpuToGpu(const int64_t &device_id, const Config &config) { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.h similarity index 97% rename from cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.h index 4948d16ac4..f972799abf 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.h @@ -19,8 +19,8 @@ #pragma once -#include "ivf.h" -#include "FaissGpuResourceMgr.h" +#include "IndexIVF.h" +#include "knowhere/index/vector_index/utils/FaissGpuResourceMgr.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp similarity index 98% rename from cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp index 494da4438a..c81e69f78e 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp @@ -23,9 +23,9 @@ #include -#include "knowhere/common/exception.h" -#include "knowhere/adapter/faiss_adopt.h" -#include "idmap.h" +#include "knowhere/common/Exception.h" +#include "knowhere/adapter/VectorAdapter.h" +#include "IndexIDMAP.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.h similarity index 98% rename from cpp/src/core/knowhere/knowhere/index/vector_index/idmap.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.h index b54a589423..16d5ece52a 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/idmap.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.h @@ -18,8 +18,8 @@ #pragma once -#include "ivf.h" -#include "gpu_ivf.h" +#include "IndexIVF.h" +#include "IndexGPUIVF.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/ivf.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp similarity index 98% rename from cpp/src/core/knowhere/knowhere/index/vector_index/ivf.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp index 6d72479002..aaaafdad59 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/ivf.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -23,14 +23,13 @@ #include #include #include -#include #include -#include "knowhere/common/exception.h" -#include "ivf.h" -#include "knowhere/adapter/faiss_adopt.h" -#include "gpu_ivf.h" +#include "knowhere/common/Exception.h" +#include "knowhere/adapter/VectorAdapter.h" +#include "IndexIVF.h" +#include "IndexGPUIVF.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/ivf.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.h similarity index 99% rename from cpp/src/core/knowhere/knowhere/index/vector_index/ivf.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.h index c23b5abcc2..79f41a02d5 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/ivf.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.h @@ -25,7 +25,7 @@ #include #include -#include "vector_index.h" +#include "VectorIndex.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp similarity index 94% rename from cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp index d588efb7c6..6508469581 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp @@ -24,12 +24,12 @@ #undef mkdir -#include "cpu_kdt_rng.h" -#include "definitions.h" +#include "IndexKDT.h" +#include "knowhere/index/vector_index/utils/Definitions.h" //#include "knowhere/index/preprocessor/normalize.h" -#include "kdt_parameters.h" -#include "knowhere/adapter/sptag.h" -#include "knowhere/common/exception.h" +#include "knowhere/index/vector_index/utils/KDTParameterMgr.h" +#include "knowhere/adapter/SptagAdapter.h" +#include "knowhere/common/Exception.h" namespace zilliz { @@ -117,7 +117,7 @@ CPUKDTRNG::Add(const DatasetPtr &origin, const Config &add_config) { void CPUKDTRNG::SetParameters(const Config &config) { - for (auto ¶ : KDTParameterManagement::GetInstance().GetKDTParameters()) { + for (auto ¶ : KDTParameterMgr::GetInstance().GetKDTParameters()) { auto value = config.get_with_default(para.first, para.second); index_ptr_->SetParameter(para.first, value); } diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.h similarity index 97% rename from cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.h index 35d488999e..7fd41e9bad 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.h @@ -20,8 +20,8 @@ #include #include -#include "vector_index.h" -#include "knowhere/index/index_model.h" +#include "VectorIndex.h" +#include "knowhere/index/IndexModel.h" #include diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.cpp similarity index 94% rename from cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.cpp index 724c80dfca..26dc5e2304 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.cpp @@ -16,15 +16,15 @@ // under the License. -#include "nsg_index.h" -#include "knowhere/index/vector_index/nsg/nsg.h" -#include "knowhere/index/vector_index/nsg/nsg_io.h" -#include "idmap.h" -#include "ivf.h" -#include "gpu_ivf.h" -#include "knowhere/adapter/faiss_adopt.h" -#include "knowhere/common/exception.h" -#include "knowhere/common/timer.h" +#include "IndexNSG.h" +#include "knowhere/index/vector_index/nsg/NSG.h" +#include "knowhere/index/vector_index/nsg/NSGIO.h" +#include "IndexIDMAP.h" +#include "IndexIVF.h" +#include "IndexGPUIVF.h" +#include "knowhere/adapter/VectorAdapter.h" +#include "knowhere/common/Exception.h" +#include "knowhere/common/Timer.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.h similarity index 98% rename from cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.h index e63dbb03b4..873bba2287 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.h @@ -18,7 +18,7 @@ #pragma once -#include "vector_index.h" +#include "VectorIndex.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/vector_index.h b/cpp/src/core/knowhere/knowhere/index/vector_index/VectorIndex.h similarity index 90% rename from cpp/src/core/knowhere/knowhere/index/vector_index/vector_index.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/VectorIndex.h index 1828e3573d..76f67920e3 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/vector_index.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/VectorIndex.h @@ -21,10 +21,10 @@ #include -#include "knowhere/common/config.h" -#include "knowhere/common/dataset.h" -#include "knowhere/index/index.h" -#include "knowhere/index/preprocessor/preprocessor.h" +#include "knowhere/common/Config.h" +#include "knowhere/common/Dataset.h" +#include "knowhere/index/Index.h" +#include "knowhere/index/preprocessor/Preprocessor.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp similarity index 99% rename from cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp index d42f4c5662..c2b5122d65 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp @@ -22,10 +22,10 @@ #include #include -#include "nsg.h" -#include "knowhere/common/exception.h" -#include "knowhere/common/timer.h" -#include "utils.h" +#include "NSG.h" +#include "knowhere/common/Exception.h" +#include "knowhere/common/Timer.h" +#include "NSGHelper.h" // TODO: enable macro //#include diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.h similarity index 99% rename from cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.h index 247d78b97b..5291c17280 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.h @@ -23,7 +23,7 @@ #include #include -#include "neighbor.h" +#include "Neighbor.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.cpp similarity index 99% rename from cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.cpp index 96a78bb63f..eb87f97f3e 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.cpp @@ -19,7 +19,7 @@ #include #include -#include "utils.h" +#include "NSGHelper.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.h similarity index 95% rename from cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.h index a57a7738fd..f5c13194c4 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.h @@ -23,8 +23,8 @@ #include -#include "nsg.h" -#include "knowhere/common/config.h" +#include "NSG.h" +#include "knowhere/common/Config.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.cpp similarity index 99% rename from cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.cpp index 34d629ec4b..bcdde8052c 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.cpp @@ -18,7 +18,7 @@ #include -#include "nsg_io.h" +#include "NSGIO.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.h similarity index 93% rename from cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.h index e84d2a9851..e34eb992b7 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.h @@ -18,8 +18,8 @@ #pragma once -#include "nsg.h" -#include "knowhere/index/vector_index/ivf.h" +#include "NSG.h" +#include "knowhere/index/vector_index/IndexIVF.h" namespace zilliz { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/neighbor.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/Neighbor.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/index/vector_index/nsg/neighbor.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/nsg/Neighbor.h diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/index_io.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/index_io.h deleted file mode 100644 index 60e27bd7f6..0000000000 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/index_io.h +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -#pragma once - -#include "nsg.h" - -namespace zilliz { -namespace knowhere { -namespace algo { - -void read_from_file(NsgIndex* index, const char *filename); -void write_to_file(NsgIndex* index, const char *filename); - -} -} -} diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/cloner.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/utils/Cloner.cpp similarity index 89% rename from cpp/src/core/knowhere/knowhere/index/vector_index/cloner.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/utils/Cloner.cpp index 4519def300..02e9c292b8 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/cloner.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/utils/Cloner.cpp @@ -16,15 +16,16 @@ // under the License. -#include "knowhere/common/exception.h" -#include "cloner.h" -#include "ivf.h" -#include "gpu_ivf.h" -#include "idmap.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/IndexIVF.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexIDMAP.h" +#include "Cloner.h" namespace zilliz { namespace knowhere { +namespace cloner { VectorIndexPtr CopyGpuToCpu(const VectorIndexPtr &index, const Config &config) { if (auto device_index = std::dynamic_pointer_cast(index)) { @@ -52,5 +53,6 @@ VectorIndexPtr CopyCpuToGpu(const VectorIndexPtr &index, const int64_t &device_i } } +} // cloner } } diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/cloner.h b/cpp/src/core/knowhere/knowhere/index/vector_index/utils/Cloner.h similarity index 90% rename from cpp/src/core/knowhere/knowhere/index/vector_index/cloner.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/utils/Cloner.h index f546ecc7d9..a72e4df141 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/cloner.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/utils/Cloner.h @@ -18,11 +18,12 @@ #pragma once -#include "vector_index.h" +#include "knowhere/index/vector_index/VectorIndex.h" namespace zilliz { namespace knowhere { +namespace cloner { // TODO(linxj): rename CopyToGpu extern VectorIndexPtr @@ -31,5 +32,6 @@ CopyCpuToGpu(const VectorIndexPtr &index, const int64_t &device_id, const Config extern VectorIndexPtr CopyGpuToCpu(const VectorIndexPtr &index, const Config &config); -} -} \ No newline at end of file +} // cloner +} // knowhere +} // zilliz \ No newline at end of file diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/definitions.h b/cpp/src/core/knowhere/knowhere/index/vector_index/utils/Definitions.h similarity index 93% rename from cpp/src/core/knowhere/knowhere/index/vector_index/definitions.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/utils/Definitions.h index 3db32e00dc..fef7b01221 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/definitions.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/utils/Definitions.h @@ -18,15 +18,15 @@ #pragma once -#include - namespace zilliz { namespace knowhere { +namespace definition { #define META_ROWS ("rows") #define META_DIM ("dimension") #define META_K ("k") -} // namespace knowhere -} // namespace zilliz +} // definition +} // knowhere +} // zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/utils/FaissGpuResourceMgr.cpp similarity index 100% rename from cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/utils/FaissGpuResourceMgr.cpp diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.h b/cpp/src/core/knowhere/knowhere/index/vector_index/utils/FaissGpuResourceMgr.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/utils/FaissGpuResourceMgr.h diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/utils/KDTParameterMgr.cpp similarity index 93% rename from cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/utils/KDTParameterMgr.cpp index b0ed9f6d56..a085bfcf48 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/utils/KDTParameterMgr.cpp @@ -18,18 +18,18 @@ #include -#include "kdt_parameters.h" +#include "KDTParameterMgr.h" namespace zilliz { namespace knowhere { const std::vector & -KDTParameterManagement::GetKDTParameters() { +KDTParameterMgr::GetKDTParameters() { return kdt_parameters_; } -KDTParameterManagement::KDTParameterManagement() { +KDTParameterMgr::KDTParameterMgr() { kdt_parameters_ = std::vector{ {"KDTNumber", "1"}, {"NumTopDimensionKDTSplit", "5"}, diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.h b/cpp/src/core/knowhere/knowhere/index/vector_index/utils/KDTParameterMgr.h similarity index 80% rename from cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/utils/KDTParameterMgr.h index 03f660a0ef..4b43e642e2 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/utils/KDTParameterMgr.h @@ -27,22 +27,22 @@ namespace knowhere { using KDTParameter = std::pair; -class KDTParameterManagement { +class KDTParameterMgr { public: const std::vector & GetKDTParameters(); public: - static KDTParameterManagement & + static KDTParameterMgr & GetInstance() { - static KDTParameterManagement instance; + static KDTParameterMgr instance; return instance; } - KDTParameterManagement(const KDTParameterManagement &) = delete; - KDTParameterManagement &operator=(const KDTParameterManagement &) = delete; + KDTParameterMgr(const KDTParameterMgr &) = delete; + KDTParameterMgr &operator=(const KDTParameterMgr &) = delete; private: - KDTParameterManagement(); + KDTParameterMgr(); private: std::vector kdt_parameters_; diff --git a/cpp/src/core/test/test_idmap.cpp b/cpp/src/core/test/test_idmap.cpp index f85210b574..82f00455dd 100644 --- a/cpp/src/core/test/test_idmap.cpp +++ b/cpp/src/core/test/test_idmap.cpp @@ -21,10 +21,10 @@ #include #include -#include "knowhere/index/vector_index/idmap.h" -#include "knowhere/adapter/structure.h" -#include "knowhere/index/vector_index/cloner.h" -#include "knowhere/common/exception.h" +#include "knowhere/index/vector_index/IndexIDMAP.h" +#include "knowhere/adapter/Structure.h" +#include "knowhere/index/vector_index/utils/Cloner.h" +#include "knowhere/common/Exception.h" #include "utils.h" diff --git a/cpp/src/core/test/test_ivf.cpp b/cpp/src/core/test/test_ivf.cpp index f1ae3f44a6..5d29ae29ac 100644 --- a/cpp/src/core/test/test_ivf.cpp +++ b/cpp/src/core/test/test_ivf.cpp @@ -25,12 +25,12 @@ #include #include -#include "knowhere/index/vector_index/gpu_ivf.h" -#include "knowhere/index/vector_index/ivf.h" -#include "knowhere/adapter/structure.h" -#include "knowhere/index/vector_index/cloner.h" -#include "knowhere/common/exception.h" -#include "knowhere/common/timer.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexIVF.h" +#include "knowhere/adapter/Structure.h" +#include "knowhere/index/vector_index/utils/Cloner.h" +#include "knowhere/common/Exception.h" +#include "knowhere/common/Timer.h" #include "utils.h" diff --git a/cpp/src/core/test/test_kdt.cpp b/cpp/src/core/test/test_kdt.cpp index 9581cc8ee0..8874b14fc9 100644 --- a/cpp/src/core/test/test_kdt.cpp +++ b/cpp/src/core/test/test_kdt.cpp @@ -20,12 +20,12 @@ #include #include -#include "knowhere/common/exception.h" +#include "knowhere/common/Exception.h" -#include "knowhere/index/vector_index/cpu_kdt_rng.h" -#include "knowhere/index/vector_index/definitions.h" -#include "knowhere/adapter/sptag.h" -#include "knowhere/adapter/structure.h" +#include "knowhere/index/vector_index/IndexKDT.h" +#include "knowhere/index/vector_index/utils/Definitions.h" +#include "knowhere/adapter/SptagAdapter.h" +#include "knowhere/adapter/Structure.h" #include "utils.h" diff --git a/cpp/src/core/test/utils.h b/cpp/src/core/test/utils.h index df7f11d689..41dbbc1f30 100644 --- a/cpp/src/core/test/utils.h +++ b/cpp/src/core/test/utils.h @@ -23,7 +23,7 @@ #include #include -#include "knowhere/adapter/structure.h" +#include "knowhere/adapter/Structure.h" #include "knowhere/common/Log.h" class DataGen { diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 2fb868f548..2cd29b1b25 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -25,7 +25,7 @@ #include "src/wrapper/vec_index.h" #include "src/wrapper/vec_impl.h" -#include "knowhere/common/exception.h" +#include "knowhere/common/Exception.h" #include diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index 424bc9f778..3b14158e58 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -19,7 +19,7 @@ #include "SchedInst.h" #include "server/ServerConfig.h" #include "ResourceFactory.h" -#include "knowhere/index/vector_index/gpu_ivf.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" #include "Utils.h" diff --git a/cpp/src/wrapper/KnowhereResource.cpp b/cpp/src/wrapper/KnowhereResource.cpp index c8c03423a8..02633447e7 100644 --- a/cpp/src/wrapper/KnowhereResource.cpp +++ b/cpp/src/wrapper/KnowhereResource.cpp @@ -17,7 +17,7 @@ #include "KnowhereResource.h" -#include "knowhere/index/vector_index/FaissGpuResourceMgr.h" +#include "knowhere/index/vector_index/utils/FaissGpuResourceMgr.h" #include "server/ServerConfig.h" #include diff --git a/cpp/src/wrapper/data_transfer.h b/cpp/src/wrapper/data_transfer.h index 7f41af98db..10aaf355c9 100644 --- a/cpp/src/wrapper/data_transfer.h +++ b/cpp/src/wrapper/data_transfer.h @@ -18,7 +18,7 @@ #pragma once -#include "knowhere/adapter/structure.h" +#include "knowhere/adapter/Structure.h" namespace zilliz { diff --git a/cpp/src/wrapper/vec_impl.cpp b/cpp/src/wrapper/vec_impl.cpp index b6c15b72fb..cb26b422f6 100644 --- a/cpp/src/wrapper/vec_impl.cpp +++ b/cpp/src/wrapper/vec_impl.cpp @@ -17,10 +17,10 @@ #include "utils/Log.h" -#include "knowhere/index/vector_index/idmap.h" -#include "knowhere/index/vector_index/gpu_ivf.h" -#include "knowhere/common/exception.h" -#include "knowhere/index/vector_index/cloner.h" +#include "knowhere/index/vector_index/IndexIDMAP.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/utils/Cloner.h" #include "vec_impl.h" #include "data_transfer.h" @@ -153,7 +153,7 @@ IndexType VecIndexImpl::GetType() { VecIndexPtr VecIndexImpl::CopyToGpu(const int64_t &device_id, const Config &cfg) { // TODO(linxj): exception handle - auto gpu_index = zilliz::knowhere::CopyCpuToGpu(index_, device_id, cfg); + auto gpu_index = zilliz::knowhere::cloner::CopyCpuToGpu(index_, device_id, cfg); auto new_index = std::make_shared(gpu_index, ConvertToGpuIndexType(type)); new_index->dim = dim; return new_index; @@ -161,7 +161,7 @@ VecIndexPtr VecIndexImpl::CopyToGpu(const int64_t &device_id, const Config &cfg) VecIndexPtr VecIndexImpl::CopyToCpu(const Config &cfg) { // TODO(linxj): exception handle - auto cpu_index = zilliz::knowhere::CopyGpuToCpu(index_, cfg); + auto cpu_index = zilliz::knowhere::cloner::CopyGpuToCpu(index_, cfg); auto new_index = std::make_shared(cpu_index, ConvertToCpuIndexType(type)); new_index->dim = dim; return new_index; diff --git a/cpp/src/wrapper/vec_impl.h b/cpp/src/wrapper/vec_impl.h index b10dc6b711..5e5edbd199 100644 --- a/cpp/src/wrapper/vec_impl.h +++ b/cpp/src/wrapper/vec_impl.h @@ -18,7 +18,7 @@ #pragma once -#include "knowhere/index/vector_index/vector_index.h" +#include "knowhere/index/vector_index/VectorIndex.h" #include "vec_index.h" diff --git a/cpp/src/wrapper/vec_index.cpp b/cpp/src/wrapper/vec_index.cpp index b1c377d032..213dc24fb5 100644 --- a/cpp/src/wrapper/vec_index.cpp +++ b/cpp/src/wrapper/vec_index.cpp @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. -#include "knowhere/index/vector_index/ivf.h" -#include "knowhere/index/vector_index/idmap.h" -#include "knowhere/index/vector_index/gpu_ivf.h" -#include "knowhere/index/vector_index/cpu_kdt_rng.h" -#include "knowhere/index/vector_index/nsg_index.h" -#include "knowhere/common/exception.h" +#include "knowhere/index/vector_index/IndexIVF.h" +#include "knowhere/index/vector_index/IndexIDMAP.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexKDT.h" +#include "knowhere/index/vector_index/IndexNSG.h" +#include "knowhere/common/Exception.h" #include "vec_index.h" #include "vec_impl.h" diff --git a/cpp/src/wrapper/vec_index.h b/cpp/src/wrapper/vec_index.h index 3ffa0eeec8..331fd2833b 100644 --- a/cpp/src/wrapper/vec_index.h +++ b/cpp/src/wrapper/vec_index.h @@ -23,8 +23,8 @@ #include "utils/Error.h" -#include "knowhere/common/config.h" -#include "knowhere/common/binary_set.h" +#include "knowhere/common/Config.h" +#include "knowhere/common/BinarySet.h" namespace zilliz { diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 03ea11bf08..19277d1ebe 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -26,7 +26,8 @@ #include "db/DBFactory.h" #include "db/Options.h" #include "server/ServerConfig.h" -#include "knowhere/index/vector_index/gpu_ivf.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/utils/FaissGpuResourceMgr.h" INITIALIZE_EASYLOGGINGPP diff --git a/cpp/unittest/wrapper/wrapper_test.cpp b/cpp/unittest/wrapper/wrapper_test.cpp index ac185efbb3..fbac6b132c 100644 --- a/cpp/unittest/wrapper/wrapper_test.cpp +++ b/cpp/unittest/wrapper/wrapper_test.cpp @@ -20,7 +20,8 @@ #include "utils/easylogging++.h" #include "src/wrapper/vec_index.h" -#include "knowhere/index/vector_index/gpu_ivf.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/utils/FaissGpuResourceMgr.h" #include "utils.h" From 4b91faa50c68873c4721f8ba3f4a4c4b8eb437ce Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Thu, 19 Sep 2019 21:21:38 +0800 Subject: [PATCH 055/217] MS-565 split class version 1 Former-commit-id: 1a60045de13e055d4b0a7e9da389b7e377c141af --- cpp/src/core/knowhere/CMakeLists.txt | 12 +- .../knowhere/adapter/SptagAdapter.cpp | 2 +- .../index/vector_index/FaissBaseIndex.cpp | 80 +++++++++ .../index/vector_index/FaissBaseIndex.h | 52 ++++++ .../index/vector_index/IndexGPUIVF.cpp | 83 +-------- .../knowhere/index/vector_index/IndexGPUIVF.h | 116 ++++++------- .../index/vector_index/IndexGPUIVFPQ.cpp | 68 ++++++++ .../index/vector_index/IndexGPUIVFPQ.h | 47 ++++++ .../index/vector_index/IndexGPUIVFSQ.cpp | 74 ++++++++ .../index/vector_index/IndexGPUIVFSQ.h | 43 +++++ .../index/vector_index/IndexIDMAP.cpp | 1 + .../knowhere/index/vector_index/IndexIDMAP.h | 6 +- .../knowhere/index/vector_index/IndexIVF.cpp | 153 +---------------- .../knowhere/index/vector_index/IndexIVF.h | 158 +++++++----------- .../index/vector_index/IndexIVFPQ.cpp | 60 +++++++ .../knowhere/index/vector_index/IndexIVFPQ.h | 47 ++++++ .../index/vector_index/IndexIVFSQ.cpp | 70 ++++++++ .../knowhere/index/vector_index/IndexIVFSQ.h | 45 +++++ .../knowhere/index/vector_index/IndexKDT.cpp | 4 +- .../{utils => helpers}/Cloner.cpp | 4 +- .../vector_index/{utils => helpers}/Cloner.h | 0 .../{utils => helpers}/Definitions.h | 0 .../FaissGpuResourceMgr.cpp | 0 .../{utils => helpers}/FaissGpuResourceMgr.h | 0 .../index/vector_index/helpers/FaissIO.cpp | 65 +++++++ .../index/vector_index/helpers/FaissIO.h | 48 ++++++ .../{utils => helpers}/KDTParameterMgr.cpp | 0 .../{utils => helpers}/KDTParameterMgr.h | 0 .../knowhere/index/vector_index/nsg/NSGIO.h | 1 + cpp/src/wrapper/KnowhereResource.cpp | 2 +- cpp/src/wrapper/vec_impl.cpp | 2 +- cpp/src/wrapper/vec_index.cpp | 6 +- cpp/unittest/db/utils.cpp | 4 +- cpp/unittest/wrapper/wrapper_test.cpp | 3 +- 34 files changed, 853 insertions(+), 403 deletions(-) create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.h create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.h create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.h create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.h create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.h rename cpp/src/core/knowhere/knowhere/index/vector_index/{utils => helpers}/Cloner.cpp (92%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{utils => helpers}/Cloner.h (100%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{utils => helpers}/Definitions.h (100%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{utils => helpers}/FaissGpuResourceMgr.cpp (100%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{utils => helpers}/FaissGpuResourceMgr.h (100%) create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.h rename cpp/src/core/knowhere/knowhere/index/vector_index/{utils => helpers}/KDTParameterMgr.cpp (100%) rename cpp/src/core/knowhere/knowhere/index/vector_index/{utils => helpers}/KDTParameterMgr.h (100%) diff --git a/cpp/src/core/knowhere/CMakeLists.txt b/cpp/src/core/knowhere/CMakeLists.txt index 728e198f79..76750f4f07 100644 --- a/cpp/src/core/knowhere/CMakeLists.txt +++ b/cpp/src/core/knowhere/CMakeLists.txt @@ -42,13 +42,19 @@ set(index_srcs knowhere/index/vector_index/IndexIDMAP.cpp knowhere/index/vector_index/IndexIVF.cpp knowhere/index/vector_index/IndexGPUIVF.cpp - knowhere/index/vector_index/utils/KDTParameterMgr.cpp + knowhere/index/vector_index/helpers/KDTParameterMgr.cpp knowhere/index/vector_index/IndexNSG.cpp knowhere/index/vector_index/nsg/NSG.cpp knowhere/index/vector_index/nsg/NSGIO.cpp knowhere/index/vector_index/nsg/NSGHelper.cpp - knowhere/index/vector_index/utils/Cloner.cpp - knowhere/index/vector_index/utils/FaissGpuResourceMgr.cpp + knowhere/index/vector_index/helpers/Cloner.cpp + knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp + knowhere/index/vector_index/IndexIVFSQ.cpp + knowhere/index/vector_index/IndexGPUIVFSQ.cpp + knowhere/index/vector_index/IndexIVFPQ.cpp + knowhere/index/vector_index/IndexGPUIVFPQ.cpp + knowhere/index/vector_index/FaissBaseIndex.cpp + knowhere/index/vector_index/helpers/FaissIO.cpp ) set(depend_libs diff --git a/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp index 3c5d466d41..ee5a4a01b8 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "knowhere/index/vector_index/utils/Definitions.h" +#include "knowhere/index/vector_index/helpers/Definitions.h" #include "SptagAdapter.h" #include "Structure.h" diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp new file mode 100644 index 0000000000..a3ab7956cb --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include +#include + +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/helpers/FaissIO.h" +#include "FaissBaseIndex.h" + + +namespace zilliz { +namespace knowhere { + +FaissBaseIndex::FaissBaseIndex(std::shared_ptr index) : index_(std::move(index)) {} + +BinarySet FaissBaseIndex::SerializeImpl() { + try { + faiss::Index *index = index_.get(); + + SealImpl(); + + MemoryIOWriter writer; + faiss::write_index(index, &writer); + auto data = std::make_shared(); + data.reset(writer.data_); + + BinarySet res_set; + // TODO(linxj): use virtual func Name() instead of raw string. + res_set.Append("IVF", data, writer.rp); + return res_set; + } catch (std::exception &e) { + KNOWHERE_THROW_MSG(e.what()); + } +} + +void FaissBaseIndex::LoadImpl(const BinarySet &index_binary) { + auto binary = index_binary.GetByName("IVF"); + + MemoryIOReader reader; + reader.total = binary->size; + reader.data_ = binary->data.get(); + + faiss::Index *index = faiss::read_index(&reader); + + index_.reset(index); +} + +void FaissBaseIndex::SealImpl() { +// TODO(linxj): enable +//#ifdef ZILLIZ_FAISS + faiss::Index *index = index_.get(); + auto idx = dynamic_cast(index); + if (idx != nullptr) { + idx->to_readonly(); + } + //else { + // KNOHWERE_ERROR_MSG("Seal failed"); + //} +//#endif +} + +} // knowhere +} // zilliz + diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.h b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.h new file mode 100644 index 0000000000..29edbbd61e --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.h @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + +#include + +#include "knowhere/common/BinarySet.h" + + +namespace zilliz { +namespace knowhere { + +class FaissBaseIndex { + protected: + explicit FaissBaseIndex(std::shared_ptr index); + + virtual BinarySet + SerializeImpl(); + + virtual void + LoadImpl(const BinarySet &index_binary); + + virtual void + SealImpl(); + + protected: + std::shared_ptr index_ = nullptr; +}; + +} // knowhere +} // zilliz + + + diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp index df26083108..a77702ace3 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp @@ -20,18 +20,17 @@ #include #include #include -#include #include #include #include #include "knowhere/common/Exception.h" -#include "knowhere/index/vector_index/utils/Cloner.h" +#include "knowhere/index/vector_index/helpers/Cloner.h" #include "knowhere/adapter/VectorAdapter.h" #include "IndexGPUIVF.h" +#include "knowhere/index/vector_index/helpers/FaissIO.h" -#include namespace zilliz { namespace knowhere { @@ -187,84 +186,6 @@ void GPUIVF::Add(const DatasetPtr &dataset, const Config &config) { } } -IndexModelPtr GPUIVFPQ::Train(const DatasetPtr &dataset, const Config &config) { - auto nlist = config["nlist"].as(); - auto M = config["M"].as(); // number of subquantizers(subvectors) - auto nbits = config["nbits"].as();// number of bit per subvector index - auto gpu_num = config.get_with_default("gpu_id", gpu_id_); - auto metric_type = config["metric_type"].as_string() == "L2" ? - faiss::METRIC_L2 : faiss::METRIC_L2; // IP not support. - - GETTENSOR(dataset) - - // TODO(linxj): set device here. - // TODO(linxj): set gpu resource here. - faiss::gpu::StandardGpuResources res; - faiss::gpu::GpuIndexIVFPQ device_index(&res, dim, nlist, M, nbits, metric_type); - device_index.train(rows, (float *) p_data); - - std::shared_ptr host_index = nullptr; - host_index.reset(faiss::gpu::index_gpu_to_cpu(&device_index)); - - return std::make_shared(host_index); -} - -std::shared_ptr GPUIVFPQ::GenParams(const Config &config) { - auto params = std::make_shared(); - params->nprobe = config.get_with_default("nprobe", size_t(1)); - //params->scan_table_threshold = 0; - //params->polysemous_ht = 0; - //params->max_codes = 0; - - return params; -} - -VectorIndexPtr GPUIVFPQ::CopyGpuToCpu(const Config &config) { - KNOWHERE_THROW_MSG("not support yet"); -} - -IndexModelPtr GPUIVFSQ::Train(const DatasetPtr &dataset, const Config &config) { - auto nlist = config["nlist"].as(); - auto nbits = config["nbits"].as(); // TODO(linxj): gpu only support SQ4 SQ8 SQ16 - gpu_id_ = config.get_with_default("gpu_id", gpu_id_); - auto metric_type = config["metric_type"].as_string() == "L2" ? - faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; - - GETTENSOR(dataset) - - std::stringstream index_type; - index_type << "IVF" << nlist << "," << "SQ" << nbits; - auto build_index = faiss::index_factory(dim, index_type.str().c_str(), metric_type); - - auto temp_resource = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_); - if (temp_resource != nullptr) { - ResScope rs(temp_resource, gpu_id_, true); - auto device_index = faiss::gpu::index_cpu_to_gpu(temp_resource->faiss_res.get(), gpu_id_, build_index); - device_index->train(rows, (float *) p_data); - - std::shared_ptr host_index = nullptr; - host_index.reset(faiss::gpu::index_gpu_to_cpu(device_index)); - - delete device_index; - delete build_index; - - return std::make_shared(host_index); - } else { - KNOWHERE_THROW_MSG("Build IVFSQ can't get gpu resource"); - } -} - -VectorIndexPtr GPUIVFSQ::CopyGpuToCpu(const Config &config) { - std::lock_guard lk(mutex_); - - faiss::Index *device_index = index_.get(); - faiss::Index *host_index = faiss::gpu::index_gpu_to_cpu(device_index); - - std::shared_ptr new_index; - new_index.reset(host_index); - return std::make_shared(new_index); -} - void GPUIndex::SetGpuDevice(const int &gpu_id) { gpu_id_ = gpu_id; } diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.h index f972799abf..cd04fe4c54 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.h @@ -20,79 +20,79 @@ #include "IndexIVF.h" -#include "knowhere/index/vector_index/utils/FaissGpuResourceMgr.h" +#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" namespace zilliz { namespace knowhere { class GPUIndex { - public: - explicit GPUIndex(const int &device_id) : gpu_id_(device_id) {} - GPUIndex(const int& device_id, const ResPtr& resource): gpu_id_(device_id), res_(resource){} +public: + explicit GPUIndex(const int &device_id) : gpu_id_(device_id) {} - virtual VectorIndexPtr CopyGpuToCpu(const Config &config) = 0; - virtual VectorIndexPtr CopyGpuToGpu(const int64_t &device_id, const Config &config) = 0; + GPUIndex(const int& device_id, const ResPtr& resource): gpu_id_(device_id), res_(resource) {} - void SetGpuDevice(const int &gpu_id); - const int64_t &GetGpuDevice(); + virtual VectorIndexPtr + CopyGpuToCpu(const Config &config) = 0; - protected: - int64_t gpu_id_; - ResWPtr res_; + virtual VectorIndexPtr + CopyGpuToGpu(const int64_t &device_id, const Config &config) = 0; + + void + SetGpuDevice(const int &gpu_id); + + const int64_t & + GetGpuDevice(); + +protected: + int64_t gpu_id_; + ResWPtr res_; }; class GPUIVF : public IVF, public GPUIndex { - public: - explicit GPUIVF(const int &device_id) : IVF(), GPUIndex(device_id) {} - explicit GPUIVF(std::shared_ptr index, const int64_t &device_id, ResPtr &resource) - : IVF(std::move(index)), GPUIndex(device_id, resource) {}; - IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override; - void Add(const DatasetPtr &dataset, const Config &config) override; - void set_index_model(IndexModelPtr model) override; - //DatasetPtr Search(const DatasetPtr &dataset, const Config &config) override; - VectorIndexPtr CopyGpuToCpu(const Config &config) override; - VectorIndexPtr CopyGpuToGpu(const int64_t &device_id, const Config &config) override; - VectorIndexPtr Clone() final; +public: + explicit GPUIVF(const int &device_id) : IVF(), GPUIndex(device_id) {} - // TODO(linxj): Deprecated - virtual IVFIndexPtr Copy_index_gpu_to_cpu(); + explicit GPUIVF(std::shared_ptr index, const int64_t &device_id, ResPtr &resource) + : IVF(std::move(index)), GPUIndex(device_id, resource) {}; - protected: - void search_impl(int64_t n, - const float *data, - int64_t k, - float *distances, - int64_t *labels, - const Config &cfg) override; - BinarySet SerializeImpl() override; - void LoadImpl(const BinarySet &index_binary) override; + IndexModelPtr + Train(const DatasetPtr &dataset, const Config &config) override; + + void + Add(const DatasetPtr &dataset, const Config &config) override; + + void + set_index_model(IndexModelPtr model) override; + + //DatasetPtr Search(const DatasetPtr &dataset, const Config &config) override; + VectorIndexPtr + CopyGpuToCpu(const Config &config) override; + + VectorIndexPtr + CopyGpuToGpu(const int64_t &device_id, const Config &config) override; + + VectorIndexPtr + Clone() final; + + // TODO(linxj): Deprecated + virtual IVFIndexPtr Copy_index_gpu_to_cpu(); + +protected: + void + search_impl(int64_t n, + const float *data, + int64_t k, + float *distances, + int64_t *labels, + const Config &cfg) override; + + BinarySet + SerializeImpl() override; + + void + LoadImpl(const BinarySet &index_binary) override; }; -class GPUIVFSQ : public GPUIVF { - public: - explicit GPUIVFSQ(const int &device_id) : GPUIVF(device_id) {} - explicit GPUIVFSQ(std::shared_ptr index, const int64_t &device_id, ResPtr &resource) - : GPUIVF(std::move(index), device_id, resource) {}; - IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override; - - public: - VectorIndexPtr CopyGpuToCpu(const Config &config) override; -}; - -class GPUIVFPQ : public GPUIVF { - public: - explicit GPUIVFPQ(const int &device_id) : GPUIVF(device_id) {} - IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override; - - public: - VectorIndexPtr CopyGpuToCpu(const Config &config) override; - - protected: - // TODO(linxj): remove GenParams. - std::shared_ptr GenParams(const Config &config) override; -}; - - } // namespace knowhere } // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp new file mode 100644 index 0000000000..437d8f8958 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include +#include +#include + +#include "IndexGPUIVFPQ.h" +#include "knowhere/common/Exception.h" +#include "knowhere/adapter/VectorAdapter.h" + + +namespace zilliz { +namespace knowhere { + +IndexModelPtr GPUIVFPQ::Train(const DatasetPtr &dataset, const Config &config) { + auto nlist = config["nlist"].as(); + auto M = config["M"].as(); // number of subquantizers(subvectors) + auto nbits = config["nbits"].as();// number of bit per subvector index + auto gpu_num = config.get_with_default("gpu_id", gpu_id_); + auto metric_type = config["metric_type"].as_string() == "L2" ? + faiss::METRIC_L2 : faiss::METRIC_L2; // IP not support. + + GETTENSOR(dataset) + + // TODO(linxj): set device here. + // TODO(linxj): set gpu resource here. + faiss::gpu::StandardGpuResources res; + faiss::gpu::GpuIndexIVFPQ device_index(&res, dim, nlist, M, nbits, metric_type); + device_index.train(rows, (float *) p_data); + + std::shared_ptr host_index = nullptr; + host_index.reset(faiss::gpu::index_gpu_to_cpu(&device_index)); + + return std::make_shared(host_index); +} + +std::shared_ptr GPUIVFPQ::GenParams(const Config &config) { + auto params = std::make_shared(); + params->nprobe = config.get_with_default("nprobe", size_t(1)); + //params->scan_table_threshold = 0; + //params->polysemous_ht = 0; + //params->max_codes = 0; + + return params; +} + +VectorIndexPtr GPUIVFPQ::CopyGpuToCpu(const Config &config) { + KNOWHERE_THROW_MSG("not support yet"); +} + +} // knowhere +} // zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.h new file mode 100644 index 0000000000..c407ee0cc7 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include "IndexGPUIVF.h" + +namespace zilliz { +namespace knowhere { + +class GPUIVFPQ : public GPUIVF { +public: + explicit GPUIVFPQ(const int &device_id) : GPUIVF(device_id) {} + + IndexModelPtr + Train(const DatasetPtr &dataset, const Config &config) override; + +public: + VectorIndexPtr + CopyGpuToCpu(const Config &config) override; + +protected: + // TODO(linxj): remove GenParams. + std::shared_ptr + GenParams(const Config &config) override; +}; + +} // knowhere +} // zilliz + + + diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp new file mode 100644 index 0000000000..2504dd4599 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include + +#include "knowhere/adapter/VectorAdapter.h" +#include "knowhere/common/Exception.h" +#include "IndexGPUIVFSQ.h" +#include "IndexIVFSQ.h" + + +namespace zilliz { +namespace knowhere { + + IndexModelPtr GPUIVFSQ::Train(const DatasetPtr &dataset, const Config &config) { + auto nlist = config["nlist"].as(); + auto nbits = config["nbits"].as(); // TODO(linxj): gpu only support SQ4 SQ8 SQ16 + gpu_id_ = config.get_with_default("gpu_id", gpu_id_); + auto metric_type = config["metric_type"].as_string() == "L2" ? + faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; + + GETTENSOR(dataset) + + std::stringstream index_type; + index_type << "IVF" << nlist << "," << "SQ" << nbits; + auto build_index = faiss::index_factory(dim, index_type.str().c_str(), metric_type); + + auto temp_resource = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_); + if (temp_resource != nullptr) { + ResScope rs(temp_resource, gpu_id_, true); + auto device_index = faiss::gpu::index_cpu_to_gpu(temp_resource->faiss_res.get(), gpu_id_, build_index); + device_index->train(rows, (float *) p_data); + + std::shared_ptr host_index = nullptr; + host_index.reset(faiss::gpu::index_gpu_to_cpu(device_index)); + + delete device_index; + delete build_index; + + return std::make_shared(host_index); + } else { + KNOWHERE_THROW_MSG("Build IVFSQ can't get gpu resource"); + } + } + + VectorIndexPtr GPUIVFSQ::CopyGpuToCpu(const Config &config) { + std::lock_guard lk(mutex_); + + faiss::Index *device_index = index_.get(); + faiss::Index *host_index = faiss::gpu::index_gpu_to_cpu(device_index); + + std::shared_ptr new_index; + new_index.reset(host_index); + return std::make_shared(new_index); + } + +} // knowhere +} // zilliz + diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.h new file mode 100644 index 0000000000..677907964b --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include "IndexGPUIVF.h" + + +namespace zilliz { +namespace knowhere { + +class GPUIVFSQ : public GPUIVF { +public: + explicit GPUIVFSQ(const int &device_id) : GPUIVF(device_id) {} + + explicit GPUIVFSQ(std::shared_ptr index, const int64_t &device_id, ResPtr &resource) + : GPUIVF(std::move(index), device_id, resource) {}; + + IndexModelPtr + Train(const DatasetPtr &dataset, const Config &config) override; + + VectorIndexPtr + CopyGpuToCpu(const Config &config) override; +}; + +} // knowhere +} // zilliz + diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp index c81e69f78e..a53a73dad9 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp @@ -25,6 +25,7 @@ #include "knowhere/common/Exception.h" #include "knowhere/adapter/VectorAdapter.h" +#include "knowhere/index/vector_index/helpers/FaissIO.h" #include "IndexIDMAP.h" diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.h index 16d5ece52a..106759faf1 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.h @@ -25,10 +25,10 @@ namespace zilliz { namespace knowhere { -class IDMAP : public VectorIndex, public BasicIndex { +class IDMAP : public VectorIndex, public FaissBaseIndex { public: - IDMAP() : BasicIndex(nullptr) {}; - explicit IDMAP(std::shared_ptr index) : BasicIndex(std::move(index)) {}; + IDMAP() : FaissBaseIndex(nullptr) {}; + explicit IDMAP(std::shared_ptr index) : FaissBaseIndex(std::move(index)) {}; BinarySet Serialize() override; void Load(const BinarySet &index_binary) override; void Train(const Config &config); diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp index aaaafdad59..a7adfe921a 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -238,7 +238,7 @@ void IVF::Seal() { } -IVFIndexModel::IVFIndexModel(std::shared_ptr index) : BasicIndex(std::move(index)) {} +IVFIndexModel::IVFIndexModel(std::shared_ptr index) : FaissBaseIndex(std::move(index)) {} BinarySet IVFIndexModel::Serialize() { if (!index_ || !index_->is_trained) { @@ -257,159 +257,8 @@ void IVFIndexModel::SealImpl() { // do nothing } -IndexModelPtr IVFSQ::Train(const DatasetPtr &dataset, const Config &config) { - auto nlist = config["nlist"].as(); - auto nbits = config["nbits"].as(); // TODO(linxj): only support SQ4 SQ6 SQ8 SQ16 - auto metric_type = config["metric_type"].as_string() == "L2" ? - faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; - GETTENSOR(dataset) - std::stringstream index_type; - index_type << "IVF" << nlist << "," << "SQ" << nbits; - auto build_index = faiss::index_factory(dim, index_type.str().c_str(), metric_type); - build_index->train(rows, (float *) p_data); - - std::shared_ptr ret_index; - ret_index.reset(build_index); - return std::make_shared(ret_index); -} - -VectorIndexPtr IVFSQ::Clone_impl(const std::shared_ptr &index) { - return std::make_shared(index); -} - -VectorIndexPtr IVFSQ::CopyCpuToGpu(const int64_t &device_id, const Config &config) { - if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)){ - ResScope rs(res, device_id, false); - faiss::gpu::GpuClonerOptions option; - option.allInGpu = true; - - auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get(), &option); - - std::shared_ptr device_index; - device_index.reset(gpu_index); - return std::make_shared(device_index, device_id, res); - } else { - KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu_resource"); - } -} - -IndexModelPtr IVFPQ::Train(const DatasetPtr &dataset, const Config &config) { - auto nlist = config["nlist"].as(); - auto M = config["M"].as(); // number of subquantizers(subvector) - auto nbits = config["nbits"].as();// number of bit per subvector index - auto metric_type = config["metric_type"].as_string() == "L2" ? - faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; - - GETTENSOR(dataset) - - faiss::Index *coarse_quantizer = new faiss::IndexFlat(dim, metric_type); - auto index = std::make_shared(coarse_quantizer, dim, nlist, M, nbits); - index->train(rows, (float *) p_data); - - return std::make_shared(index); -} - -std::shared_ptr IVFPQ::GenParams(const Config &config) { - auto params = std::make_shared(); - params->nprobe = config.get_with_default("nprobe", size_t(1)); - //params->scan_table_threshold = 0; - //params->polysemous_ht = 0; - //params->max_codes = 0; - - return params; -} - -VectorIndexPtr IVFPQ::Clone_impl(const std::shared_ptr &index) { - return std::make_shared(index); -} - -BasicIndex::BasicIndex(std::shared_ptr index) : index_(std::move(index)) {} - -BinarySet BasicIndex::SerializeImpl() { - try { - faiss::Index *index = index_.get(); - - SealImpl(); - - MemoryIOWriter writer; - faiss::write_index(index, &writer); - auto data = std::make_shared(); - data.reset(writer.data_); - - BinarySet res_set; - // TODO(linxj): use virtual func Name() instead of raw string. - res_set.Append("IVF", data, writer.rp); - return res_set; - } catch (std::exception &e) { - KNOWHERE_THROW_MSG(e.what()); - } -} - -void BasicIndex::LoadImpl(const BinarySet &index_binary) { - auto binary = index_binary.GetByName("IVF"); - - MemoryIOReader reader; - reader.total = binary->size; - reader.data_ = binary->data.get(); - - faiss::Index *index = faiss::read_index(&reader); - - index_.reset(index); -} - -void BasicIndex::SealImpl() { -// TODO(linxj): enable -//#ifdef ZILLIZ_FAISS - faiss::Index *index = index_.get(); - auto idx = dynamic_cast(index); - if (idx != nullptr) { - idx->to_readonly(); - } - //else { - // KNOHWERE_ERROR_MSG("Seal failed"); - //} -//#endif -} - -// TODO(linxj): Get From Config File -static size_t magic_num = 2; -size_t MemoryIOWriter::operator()(const void *ptr, size_t size, size_t nitems) { - auto total_need = size * nitems + rp; - - if (!data_) { // data == nullptr - total = total_need * magic_num; - rp = size * nitems; - data_ = new uint8_t[total]; - memcpy((void *) (data_), ptr, rp); - } - - if (total_need > total) { - total = total_need * magic_num; - auto new_data = new uint8_t[total]; - memcpy((void *) new_data, (void *) data_, rp); - delete data_; - data_ = new_data; - - memcpy((void *) (data_ + rp), ptr, size * nitems); - rp = total_need; - } else { - memcpy((void *) (data_ + rp), ptr, size * nitems); - rp = total_need; - } - - return nitems; -} - -size_t MemoryIOReader::operator()(void *ptr, size_t size, size_t nitems) { - if (rp >= total) return 0; - size_t nremain = (total - rp) / size; - if (nremain < nitems) nitems = nremain; - memcpy(ptr, (void *) (data_ + rp), size * nitems); - rp += size * nitems; - return nitems; -} } diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.h index 79f41a02d5..72f98a5afa 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.h @@ -21,63 +21,76 @@ #include #include -#include -#include -#include - #include "VectorIndex.h" +#include "FaissBaseIndex.h" +#include "faiss/IndexIVF.h" namespace zilliz { namespace knowhere { -class BasicIndex { - protected: - explicit BasicIndex(std::shared_ptr index); - virtual BinarySet SerializeImpl(); - virtual void LoadImpl(const BinarySet &index_binary); - virtual void - SealImpl(); - - protected: - std::shared_ptr index_ = nullptr; -}; - using Graph = std::vector>; -class IVF : public VectorIndex, protected BasicIndex { +class IVF : public VectorIndex, protected FaissBaseIndex { public: - IVF() : BasicIndex(nullptr) {}; - explicit IVF(std::shared_ptr index) : BasicIndex(std::move(index)) {} - VectorIndexPtr Clone() override;; - IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override; - void set_index_model(IndexModelPtr model) override; - void Add(const DatasetPtr &dataset, const Config &config) override; - void AddWithoutIds(const DatasetPtr &dataset, const Config &config); - DatasetPtr Search(const DatasetPtr &dataset, const Config &config) override; - void GenGraph(const int64_t &k, Graph &graph, const DatasetPtr &dataset, const Config &config); - BinarySet Serialize() override; - void Load(const BinarySet &index_binary) override; - int64_t Count() override; - int64_t Dimension() override; + IVF() : FaissBaseIndex(nullptr) {}; + + explicit IVF(std::shared_ptr index) : FaissBaseIndex(std::move(index)) {} + + VectorIndexPtr + Clone() override;; + + IndexModelPtr + Train(const DatasetPtr &dataset, const Config &config) override; + + void + set_index_model(IndexModelPtr model) override; + + void + Add(const DatasetPtr &dataset, const Config &config) override; + + void + AddWithoutIds(const DatasetPtr &dataset, const Config &config); + + DatasetPtr + Search(const DatasetPtr &dataset, const Config &config) override; + + void + GenGraph(const int64_t &k, Graph &graph, const DatasetPtr &dataset, const Config &config); + + BinarySet + Serialize() override; + + void + Load(const BinarySet &index_binary) override; + + int64_t + Count() override; + + int64_t + Dimension() override; void Seal() override; - virtual VectorIndexPtr CopyCpuToGpu(const int64_t &device_id, const Config &config); + virtual VectorIndexPtr + CopyCpuToGpu(const int64_t &device_id, const Config &config); protected: - virtual std::shared_ptr GenParams(const Config &config); + virtual std::shared_ptr + GenParams(const Config &config); - virtual VectorIndexPtr Clone_impl(const std::shared_ptr &index); + virtual VectorIndexPtr + Clone_impl(const std::shared_ptr &index); - virtual void search_impl(int64_t n, - const float *data, - int64_t k, - float *distances, - int64_t *labels, - const Config &cfg); + virtual void + search_impl(int64_t n, + const float *data, + int64_t k, + float *distances, + int64_t *labels, + const Config &cfg); protected: std::mutex mutex_; @@ -85,72 +98,29 @@ class IVF : public VectorIndex, protected BasicIndex { using IVFIndexPtr = std::shared_ptr; -class IVFSQ : public IVF { - public: - explicit IVFSQ(std::shared_ptr index) : IVF(std::move(index)) {} - IVFSQ() = default; - IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override; - VectorIndexPtr CopyCpuToGpu(const int64_t &device_id, const Config &config) override; - protected: - VectorIndexPtr Clone_impl(const std::shared_ptr &index) override; -}; - -class IVFPQ : public IVF { - public: - explicit IVFPQ(std::shared_ptr index) : IVF(std::move(index)) {} - IVFPQ() = default; - IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override; - protected: - std::shared_ptr GenParams(const Config &config) override; - VectorIndexPtr Clone_impl(const std::shared_ptr &index) override; -}; - - -//class OPQIVFPQ : public IVFPQ { -// public: -// PreprocessorPtr BuildPreprocessor(const Dataset &dataset, const Config &config) override; -//}; - - class GPUIVF; - - -struct MemoryIOWriter : public faiss::IOWriter { - uint8_t *data_ = nullptr; - size_t total = 0; - size_t rp = 0; - - size_t operator()(const void *ptr, size_t size, size_t nitems) override; -}; - - -struct MemoryIOReader : public faiss::IOReader { - uint8_t *data_; - size_t rp = 0; - size_t total = 0; - - size_t operator()(void *ptr, size_t size, size_t nitems) override; - -}; - - -class IVFIndexModel : public IndexModel, public BasicIndex { +class IVFIndexModel : public IndexModel, public FaissBaseIndex { friend IVF; friend GPUIVF; public: explicit IVFIndexModel(std::shared_ptr index); - IVFIndexModel() : BasicIndex(nullptr) {}; - BinarySet Serialize() override; + + IVFIndexModel() : FaissBaseIndex(nullptr) {}; + + BinarySet + Serialize() override; + + void + Load(const BinarySet &binary) override; + protected: - void SealImpl() override; - public: - void Load(const BinarySet &binary) override; + void + SealImpl() override; protected: std::mutex mutex_; }; - using IVFIndexModelPtr = std::shared_ptr; } diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp new file mode 100644 index 0000000000..eca63a5f3b --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include +#include + +#include "IndexIVFPQ.h" +#include "knowhere/common/Exception.h" +#include "knowhere/adapter/VectorAdapter.h" + +namespace zilliz { +namespace knowhere { + +IndexModelPtr IVFPQ::Train(const DatasetPtr &dataset, const Config &config) { + auto nlist = config["nlist"].as(); + auto M = config["M"].as(); // number of subquantizers(subvector) + auto nbits = config["nbits"].as();// number of bit per subvector index + auto metric_type = config["metric_type"].as_string() == "L2" ? + faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; + + GETTENSOR(dataset) + + faiss::Index *coarse_quantizer = new faiss::IndexFlat(dim, metric_type); + auto index = std::make_shared(coarse_quantizer, dim, nlist, M, nbits); + index->train(rows, (float *) p_data); + + return std::make_shared(index); +} + +std::shared_ptr IVFPQ::GenParams(const Config &config) { + auto params = std::make_shared(); + params->nprobe = config.get_with_default("nprobe", size_t(1)); + //params->scan_table_threshold = 0; + //params->polysemous_ht = 0; + //params->max_codes = 0; + + return params; +} + +VectorIndexPtr IVFPQ::Clone_impl(const std::shared_ptr &index) { + return std::make_shared(index); +} + +} // knowhere +} // zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.h new file mode 100644 index 0000000000..427f76fe3e --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include "IndexIVF.h" + +namespace zilliz { +namespace knowhere { + +class IVFPQ : public IVF { +public: + explicit IVFPQ(std::shared_ptr index) : IVF(std::move(index)) {} + + IVFPQ() = default; + + IndexModelPtr + Train(const DatasetPtr &dataset, const Config &config) override; + +protected: + std::shared_ptr + GenParams(const Config &config) override; + + VectorIndexPtr + Clone_impl(const std::shared_ptr &index) override; +}; + +} // knowhere +} // zilliz + + + diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp new file mode 100644 index 0000000000..00f191ff02 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include + +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" +#include "knowhere/adapter/VectorAdapter.h" +#include "IndexIVFSQ.h" +#include "IndexGPUIVFSQ.h" + + +namespace zilliz { +namespace knowhere { + +IndexModelPtr IVFSQ::Train(const DatasetPtr &dataset, const Config &config) { + auto nlist = config["nlist"].as(); + auto nbits = config["nbits"].as(); // TODO(linxj): only support SQ4 SQ6 SQ8 SQ16 + auto metric_type = config["metric_type"].as_string() == "L2" ? + faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; + + GETTENSOR(dataset) + + std::stringstream index_type; + index_type << "IVF" << nlist << "," << "SQ" << nbits; + auto build_index = faiss::index_factory(dim, index_type.str().c_str(), metric_type); + build_index->train(rows, (float *) p_data); + + std::shared_ptr ret_index; + ret_index.reset(build_index); + return std::make_shared(ret_index); +} + +VectorIndexPtr IVFSQ::Clone_impl(const std::shared_ptr &index) { + return std::make_shared(index); +} + +VectorIndexPtr IVFSQ::CopyCpuToGpu(const int64_t &device_id, const Config &config) { + if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)){ + ResScope rs(res, device_id, false); + faiss::gpu::GpuClonerOptions option; + option.allInGpu = true; + + auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get(), &option); + + std::shared_ptr device_index; + device_index.reset(gpu_index); + return std::make_shared(device_index, device_id, res); + } else { + KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu_resource"); + } +} + +} // knowhere +} // zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.h new file mode 100644 index 0000000000..5ee771f951 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include "IndexIVF.h" + +namespace zilliz { +namespace knowhere { + +class IVFSQ : public IVF { +public: + explicit IVFSQ(std::shared_ptr index) : IVF(std::move(index)) {} + + IVFSQ() = default; + + IndexModelPtr + Train(const DatasetPtr &dataset, const Config &config) override; + + VectorIndexPtr + CopyCpuToGpu(const int64_t &device_id, const Config &config) override; + +protected: + VectorIndexPtr + Clone_impl(const std::shared_ptr &index) override; +}; + +} // knowhere +} // zilliz + diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp index 6508469581..5e049b72bd 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp @@ -25,9 +25,9 @@ #undef mkdir #include "IndexKDT.h" -#include "knowhere/index/vector_index/utils/Definitions.h" +#include "knowhere/index/vector_index/helpers/Definitions.h" //#include "knowhere/index/preprocessor/normalize.h" -#include "knowhere/index/vector_index/utils/KDTParameterMgr.h" +#include "knowhere/index/vector_index/helpers/KDTParameterMgr.h" #include "knowhere/adapter/SptagAdapter.h" #include "knowhere/common/Exception.h" diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/utils/Cloner.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp similarity index 92% rename from cpp/src/core/knowhere/knowhere/index/vector_index/utils/Cloner.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp index 02e9c292b8..d8ccdd1087 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/utils/Cloner.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp @@ -18,6 +18,8 @@ #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/IndexIVF.h" +#include "knowhere/index/vector_index/IndexIVFSQ.h" +#include "knowhere/index/vector_index/IndexIVFPQ.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" #include "knowhere/index/vector_index/IndexIDMAP.h" #include "Cloner.h" @@ -43,7 +45,7 @@ VectorIndexPtr CopyCpuToGpu(const VectorIndexPtr &index, const int64_t &device_i if (auto cpu_index = std::dynamic_pointer_cast(index)) { return cpu_index->CopyCpuToGpu(device_id, config); } else if (auto cpu_index = std::dynamic_pointer_cast(index)) { - KNOWHERE_THROW_MSG("IVFPQ not support tranfer to gpu"); + KNOWHERE_THROW_MSG("IVFPQ not support transfer to gpu"); } else if (auto cpu_index = std::dynamic_pointer_cast(index)) { return cpu_index->CopyCpuToGpu(device_id, config); } else if (auto cpu_index = std::dynamic_pointer_cast(index)) { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/utils/Cloner.h b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/index/vector_index/utils/Cloner.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.h diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/utils/Definitions.h b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Definitions.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/index/vector_index/utils/Definitions.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Definitions.h diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/utils/FaissGpuResourceMgr.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp similarity index 100% rename from cpp/src/core/knowhere/knowhere/index/vector_index/utils/FaissGpuResourceMgr.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/utils/FaissGpuResourceMgr.h b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/index/vector_index/utils/FaissGpuResourceMgr.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp new file mode 100644 index 0000000000..48167fe925 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include + +#include "FaissIO.h" + +namespace zilliz { +namespace knowhere { + +// TODO(linxj): Get From Config File +static size_t magic_num = 2; +size_t MemoryIOWriter::operator()(const void *ptr, size_t size, size_t nitems) { + auto total_need = size * nitems + rp; + + if (!data_) { // data == nullptr + total = total_need * magic_num; + rp = size * nitems; + data_ = new uint8_t[total]; + memcpy((void *) (data_), ptr, rp); + } + + if (total_need > total) { + total = total_need * magic_num; + auto new_data = new uint8_t[total]; + memcpy((void *) new_data, (void *) data_, rp); + delete data_; + data_ = new_data; + + memcpy((void *) (data_ + rp), ptr, size * nitems); + rp = total_need; + } else { + memcpy((void *) (data_ + rp), ptr, size * nitems); + rp = total_need; + } + + return nitems; +} + +size_t MemoryIOReader::operator()(void *ptr, size_t size, size_t nitems) { + if (rp >= total) return 0; + size_t nremain = (total - rp) / size; + if (nremain < nitems) nitems = nremain; + memcpy(ptr, (void *) (data_ + rp), size * nitems); + rp += size * nitems; + return nitems; +} + +} // knowhere +} // zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.h b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.h new file mode 100644 index 0000000000..cc089ec367 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.h @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include + +namespace zilliz { +namespace knowhere { + +struct MemoryIOWriter : public faiss::IOWriter { + uint8_t *data_ = nullptr; + size_t total = 0; + size_t rp = 0; + + size_t + operator()(const void *ptr, size_t size, size_t nitems) override; +}; + +struct MemoryIOReader : public faiss::IOReader { + uint8_t *data_; + size_t rp = 0; + size_t total = 0; + + size_t + operator()(void *ptr, size_t size, size_t nitems) override; +}; + +} // knowhere +} // zilliz + + + diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/utils/KDTParameterMgr.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp similarity index 100% rename from cpp/src/core/knowhere/knowhere/index/vector_index/utils/KDTParameterMgr.cpp rename to cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/utils/KDTParameterMgr.h b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h similarity index 100% rename from cpp/src/core/knowhere/knowhere/index/vector_index/utils/KDTParameterMgr.h rename to cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.h index e34eb992b7..3d6786c6c2 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.h @@ -18,6 +18,7 @@ #pragma once +#include "knowhere/index/vector_index/helpers/FaissIO.h" #include "NSG.h" #include "knowhere/index/vector_index/IndexIVF.h" diff --git a/cpp/src/wrapper/KnowhereResource.cpp b/cpp/src/wrapper/KnowhereResource.cpp index 02633447e7..007858de37 100644 --- a/cpp/src/wrapper/KnowhereResource.cpp +++ b/cpp/src/wrapper/KnowhereResource.cpp @@ -17,7 +17,7 @@ #include "KnowhereResource.h" -#include "knowhere/index/vector_index/utils/FaissGpuResourceMgr.h" +#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #include "server/ServerConfig.h" #include diff --git a/cpp/src/wrapper/vec_impl.cpp b/cpp/src/wrapper/vec_impl.cpp index cb26b422f6..accb612c2d 100644 --- a/cpp/src/wrapper/vec_impl.cpp +++ b/cpp/src/wrapper/vec_impl.cpp @@ -20,7 +20,7 @@ #include "knowhere/index/vector_index/IndexIDMAP.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" #include "knowhere/common/Exception.h" -#include "knowhere/index/vector_index/utils/Cloner.h" +#include "knowhere/index/vector_index/helpers/Cloner.h" #include "vec_impl.h" #include "data_transfer.h" diff --git a/cpp/src/wrapper/vec_index.cpp b/cpp/src/wrapper/vec_index.cpp index 213dc24fb5..7ed176d96e 100644 --- a/cpp/src/wrapper/vec_index.cpp +++ b/cpp/src/wrapper/vec_index.cpp @@ -16,8 +16,12 @@ // under the License. #include "knowhere/index/vector_index/IndexIVF.h" -#include "knowhere/index/vector_index/IndexIDMAP.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexIVFSQ.h" +#include "knowhere/index/vector_index/IndexGPUIVFSQ.h" +#include "knowhere/index/vector_index/IndexIVFPQ.h" +#include "knowhere/index/vector_index/IndexGPUIVFPQ.h" +#include "knowhere/index/vector_index/IndexIDMAP.h" #include "knowhere/index/vector_index/IndexKDT.h" #include "knowhere/index/vector_index/IndexNSG.h" #include "knowhere/common/Exception.h" diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 19277d1ebe..e9950a8574 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -25,9 +25,7 @@ #include "cache/CpuCacheMgr.h" #include "db/DBFactory.h" #include "db/Options.h" -#include "server/ServerConfig.h" -#include "knowhere/index/vector_index/IndexGPUIVF.h" -#include "knowhere/index/vector_index/utils/FaissGpuResourceMgr.h" +#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" INITIALIZE_EASYLOGGINGPP diff --git a/cpp/unittest/wrapper/wrapper_test.cpp b/cpp/unittest/wrapper/wrapper_test.cpp index fbac6b132c..83896fed17 100644 --- a/cpp/unittest/wrapper/wrapper_test.cpp +++ b/cpp/unittest/wrapper/wrapper_test.cpp @@ -20,8 +20,7 @@ #include "utils/easylogging++.h" #include "src/wrapper/vec_index.h" -#include "knowhere/index/vector_index/IndexGPUIVF.h" -#include "knowhere/index/vector_index/utils/FaissGpuResourceMgr.h" +#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #include "utils.h" From fc6bf8fcd11649ead5efe3d46ece1de178e3e54f Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Fri, 20 Sep 2019 11:14:44 +0800 Subject: [PATCH 056/217] Update CONTRIBUTING.md Former-commit-id: 29edebcd376afba24e5c62695d7260e30acb8336 --- cpp/CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/CONTRIBUTING.md b/cpp/CONTRIBUTING.md index a44a933c85..4f60946d76 100644 --- a/cpp/CONTRIBUTING.md +++ b/cpp/CONTRIBUTING.md @@ -20,8 +20,8 @@ Before you make any contributions, make sure you follow this list. Contributions to Milvus fall into the following categories. -1. To report a bug or a problem with documentation, please file an [issue](https://github.com/milvus-io/milvus/issues/new) providing the details of the problem. If you believe the issue needs priority attention, please comment on the issue to notify the team. -2. To propose a new feature, please file a new feature request [issue](https://github.com/milvus-io/milvus/issues/new). Describe the intended feature and discuss the design and implementation with the team and community. Once the team agrees that the plan looks good, go ahead and implement it, following the [Contributing code]. +1. To report a bug or a problem with documentation, please file an [issue](https://github.com/milvus-io/milvus/issues/new/choose) providing the details of the problem. If you believe the issue needs priority attention, please comment on the issue to notify the team. +2. To propose a new feature, please file a new feature request [issue](https://github.com/milvus-io/milvus/issues/new/choose). Describe the intended feature and discuss the design and implementation with the team and community. Once the team agrees that the plan looks good, go ahead and implement it, following the [Contributing code]. 3. To implement a feature or bug-fix for an existing outstanding issue, follow the [Contributing code]. If you need more context on a particular issue, comment on the issue to let people know. ## How can I contribute? From 736ca8e47332d253f67caddd363726cf25700846 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 20 Sep 2019 11:33:10 +0800 Subject: [PATCH 057/217] MS-578 makesure milvus5.0 dont crack 0.3.1 data Former-commit-id: 9f801246821d98c4f5d7681252b5dfadf1f94773 --- cpp/CHANGELOG.md | 1 + cpp/src/db/Options.cpp | 20 +-- cpp/src/db/engine/ExecutionEngineImpl.cpp | 12 +- cpp/src/db/meta/Meta.h | 3 + cpp/src/db/meta/MySQLMetaImpl.cpp | 147 +++++++++++++--------- cpp/src/db/meta/SqliteMetaImpl.cpp | 20 ++- cpp/src/main.cpp | 8 +- cpp/src/server/DBWrapper.cpp | 7 +- cpp/src/server/Server.cpp | 2 +- cpp/src/utils/Error.h | 1 + cpp/src/utils/Exception.h | 30 +++-- cpp/src/utils/Log.h | 3 - cpp/src/utils/LogUtil.cpp | 2 - cpp/unittest/db/db_tests.cpp | 3 - cpp/unittest/db/meta_tests.cpp | 11 +- cpp/unittest/db/misc_test.cpp | 11 +- cpp/unittest/db/mysql_db_test.cpp | 1 - cpp/unittest/db/mysql_meta_test.cpp | 13 +- cpp/unittest/main.cpp | 6 +- cpp/unittest/server/util_test.cpp | 15 ++- cpp/unittest/wrapper/wrapper_test.cpp | 7 +- 21 files changed, 176 insertions(+), 147 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 6dc2369b47..667399cc5d 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -18,6 +18,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-562 - Add JobMgr and TaskCreator in Scheduler - MS-566 - Refactor cmake - MS-555 - Remove old scheduler +- MS-578 - Makesure milvus5.0 don't crack 0.3.1 data ## New Feature diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index 5796b781b2..fd52b851e2 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -17,7 +17,7 @@ #include "Options.h" #include "utils/Exception.h" -#include "utils/easylogging++.h" +#include "utils/Log.h" #include #include @@ -56,11 +56,11 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) { std::vector kv; boost::algorithm::split(kv, token, boost::is_any_of(":")); if (kv.size() != 2) { - LOG(WARNING) << "Invalid ArchiveConf Criterias: " << token << " Ignore!"; + ENGINE_LOG_WARNING << "Invalid ArchiveConf Criterias: " << token << " Ignore!"; continue; } if (kv[0] != "disk" && kv[0] != "days") { - LOG(WARNING) << "Invalid ArchiveConf Criterias: " << token << " Ignore!"; + ENGINE_LOG_WARNING << "Invalid ArchiveConf Criterias: " << token << " Ignore!"; continue; } try { @@ -68,20 +68,22 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) { criterias_[kv[0]] = value; } catch (std::out_of_range&){ - LOG(ERROR) << "Out of range: '" << kv[1] << "'"; - throw OutOfRangeException(); + std::string msg = "Out of range: '" + kv[1] + "'"; + ENGINE_LOG_ERROR << msg; + throw InvalidArgumentException(msg); } catch (...){ - LOG(ERROR) << "Invalid argument: '" << kv[1] << "'"; - throw InvalidArgumentException(); + std::string msg = "Invalid argument: '" + kv[1] + "'"; + ENGINE_LOG_ERROR << msg; + throw InvalidArgumentException(msg); } } } void ArchiveConf::ParseType(const std::string& type) { if (type != "delete" && type != "swap") { - LOG(ERROR) << "Invalid argument: type='" << type << "'"; - throw InvalidArgumentException(); + std::string msg = "Invalid argument: type='" + type + "'"; + throw InvalidArgumentException(msg); } type_ = type; } diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 2fb868f548..41729d104f 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -45,14 +45,18 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, nlist_(nlist) { index_ = CreatetVecIndex(EngineType::FAISS_IDMAP); - if (!index_) throw Exception("Create Empty VecIndex"); + if (!index_) { + throw Exception(DB_ERROR, "Could not create VecIndex"); + } Config build_cfg; build_cfg["dim"] = dimension; build_cfg["metric_type"] = (metric_type_ == MetricType::IP) ? "IP" : "L2"; AutoGenParams(index_->GetType(), 0, build_cfg); auto ec = std::static_pointer_cast(index_)->Build(build_cfg); - if (ec != KNOWHERE_SUCCESS) { throw Exception("Build index error"); } + if (ec != KNOWHERE_SUCCESS) { + throw Exception(DB_ERROR, "Build index error"); + } } ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index, @@ -273,7 +277,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t auto to_index = CreatetVecIndex(engine_type); if (!to_index) { - throw Exception("Create Empty VecIndex"); + throw Exception(DB_ERROR, "Could not create VecIndex"); } Config build_cfg; @@ -287,7 +291,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t from_index->GetRawVectors(), from_index->GetRawIds(), build_cfg); - if (ec != KNOWHERE_SUCCESS) { throw Exception("Build index error"); } + if (ec != KNOWHERE_SUCCESS) { throw Exception(DB_ERROR, "Build index error"); } return std::make_shared(to_index, location, engine_type, metric_type_, nlist_); } diff --git a/cpp/src/db/meta/Meta.h b/cpp/src/db/meta/Meta.h index ee4972ae6b..58168e5fee 100644 --- a/cpp/src/db/meta/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -31,6 +31,9 @@ namespace milvus { namespace engine { namespace meta { +static const char* META_TABLES = "Tables"; +static const char* META_TABLEFILES = "TableFiles"; + class Meta { public: virtual ~Meta() = default; diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index a07d4c82cf..282df32eb1 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -153,8 +153,9 @@ Status MySQLMetaImpl::Initialize() { } Query InitializeQuery = connectionPtr->query(); - InitializeQuery << "CREATE TABLE IF NOT EXISTS Tables (" << - "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << + InitializeQuery << "CREATE TABLE IF NOT EXISTS " << + META_TABLES << " " << + "(id BIGINT PRIMARY KEY AUTO_INCREMENT, " << "table_id VARCHAR(255) UNIQUE NOT NULL, " << "state INT NOT NULL, " << "dimension SMALLINT NOT NULL, " << @@ -171,8 +172,9 @@ Status MySQLMetaImpl::Initialize() { return HandleException("Initialization Error", InitializeQuery.error()); } - InitializeQuery << "CREATE TABLE IF NOT EXISTS TableFiles (" << - "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << + InitializeQuery << "CREATE TABLE IF NOT EXISTS " << + META_TABLEFILES << " " << + "(id BIGINT PRIMARY KEY AUTO_INCREMENT, " << "table_id VARCHAR(255) NOT NULL, " << "engine_type INT DEFAULT 1 NOT NULL, " << "file_id VARCHAR(255) NOT NULL, " << @@ -233,7 +235,8 @@ Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, Query dropPartitionsByDatesQuery = connectionPtr->query(); - dropPartitionsByDatesQuery << "UPDATE TableFiles " << + dropPartitionsByDatesQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," << "updated_time = " << utils::GetMicroSecTimeStamp() << " " << "WHERE table_id = " << quote << table_id << " AND " << @@ -266,7 +269,8 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { if (table_schema.table_id_.empty()) { NextTableId(table_schema.table_id_); } else { - createTableQuery << "SELECT state FROM Tables " << + createTableQuery << "SELECT state FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << table_schema.table_id_ << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); @@ -297,8 +301,9 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { std::string nlist = std::to_string(table_schema.nlist_); std::string metric_type = std::to_string(table_schema.metric_type_); - createTableQuery << "INSERT INTO Tables VALUES" << - "(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << + createTableQuery << "INSERT INTO " << + META_TABLES << " " << + "VALUES(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << created_on << ", " << flag << ", " << index_file_size << ", " << engine_type << ", " << nlist << ", " << metric_type << ");"; @@ -348,7 +353,8 @@ Status MySQLMetaImpl::FilesByType(const std::string &table_id, Query hasNonIndexFilesQuery = connectionPtr->query(); //since table_id is a unique column we just need to check whether it exists or not - hasNonIndexFilesQuery << "SELECT file_id, file_type FROM TableFiles " << + hasNonIndexFilesQuery << "SELECT file_id, file_type FROM " << + META_TABLEFILES << " " << "WHERE table_id = " << quote << table_id << " AND " << "file_type in (" << types << ");"; @@ -418,8 +424,8 @@ Status MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableI } Query updateTableIndexParamQuery = connectionPtr->query(); - updateTableIndexParamQuery << "SELECT id, state, dimension, created_on " << - "FROM Tables " << + updateTableIndexParamQuery << "SELECT id, state, dimension, created_on FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << table_id << " AND " << "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; @@ -435,7 +441,8 @@ Status MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableI uint16_t dimension = resRow["dimension"]; int64_t created_on = resRow["created_on"]; - updateTableIndexParamQuery << "UPDATE Tables " << + updateTableIndexParamQuery << "UPDATE " << + META_TABLES << " " << "SET id = " << id << ", " << "state = " << state << ", " << "dimension = " << dimension << ", " << @@ -476,7 +483,8 @@ Status MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) } Query updateTableFlagQuery = connectionPtr->query(); - updateTableFlagQuery << "UPDATE Tables " << + updateTableFlagQuery << "UPDATE " << + META_TABLES << " " << "SET flag = " << flag << " " << "WHERE table_id = " << quote << table_id << ";"; @@ -507,8 +515,8 @@ Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex } Query describeTableIndexQuery = connectionPtr->query(); - describeTableIndexQuery << "SELECT engine_type, nlist, index_file_size, metric_type " << - "FROM Tables " << + describeTableIndexQuery << "SELECT engine_type, nlist, index_file_size, metric_type FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << table_id << " AND " << "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; @@ -549,7 +557,8 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { Query dropTableIndexQuery = connectionPtr->query(); //soft delete index files - dropTableIndexQuery << "UPDATE TableFiles " << + dropTableIndexQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," << "updated_time = " << utils::GetMicroSecTimeStamp() << " " << "WHERE table_id = " << quote << table_id << " AND " << @@ -562,7 +571,8 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { } //set all backup file to raw - dropTableIndexQuery << "UPDATE TableFiles " << + dropTableIndexQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::RAW) << "," << "updated_time = " << utils::GetMicroSecTimeStamp() << " " << "WHERE table_id = " << quote << table_id << " AND " << @@ -575,7 +585,8 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { } //set table index type to raw - dropTableIndexQuery << "UPDATE Tables " << + dropTableIndexQuery << "UPDATE " << + META_TABLES << " " << "SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) << "," << "nlist = " << std::to_string(DEFAULT_NLIST) << ", " << "metric_type = " << std::to_string(DEFAULT_METRIC_TYPE) << " " << @@ -609,7 +620,8 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { //soft delete table Query deleteTableQuery = connectionPtr->query(); // - deleteTableQuery << "UPDATE Tables " << + deleteTableQuery << "UPDATE " << + META_TABLES << " " << "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " << "WHERE table_id = " << quote << table_id << ";"; @@ -645,7 +657,8 @@ Status MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { //soft delete table files Query deleteTableFilesQuery = connectionPtr->query(); // - deleteTableFilesQuery << "UPDATE TableFiles " << + deleteTableFilesQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << "WHERE table_id = " << quote << table_id << " AND " << @@ -676,9 +689,8 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { } Query describeTableQuery = connectionPtr->query(); - describeTableQuery << "SELECT id, state, dimension, created_on, " << - "flag, index_file_size, engine_type, nlist, metric_type " << - "FROM Tables " << + describeTableQuery << "SELECT id, state, dimension, created_on, flag, index_file_size, engine_type, nlist, metric_type FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << table_schema.table_id_ << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; @@ -732,7 +744,8 @@ Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { Query hasTableQuery = connectionPtr->query(); //since table_id is a unique column we just need to check whether it exists or not hasTableQuery << "SELECT EXISTS " << - "(SELECT 1 FROM Tables " << + "(SELECT 1 FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << table_id << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << "AS " << quote << "check" << ";"; @@ -764,8 +777,8 @@ Status MySQLMetaImpl::AllTables(std::vector &table_schema_array) { } Query allTablesQuery = connectionPtr->query(); - allTablesQuery << "SELECT id, table_id, dimension, engine_type, nlist, index_file_size, metric_type " << - "FROM Tables " << + allTablesQuery << "SELECT id, table_id, dimension, engine_type, nlist, index_file_size, metric_type FROM " << + META_TABLES << " " << "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str(); @@ -846,8 +859,9 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { Query createTableFileQuery = connectionPtr->query(); - createTableFileQuery << "INSERT INTO TableFiles VALUES" << - "(" << id << ", " << quote << table_id << ", " << engine_type << ", " << + createTableFileQuery << "INSERT INTO " << + META_TABLEFILES << " " << + "VALUES(" << id << ", " << quote << table_id << ", " << engine_type << ", " << quote << file_id << ", " << file_type << ", " << file_size << ", " << row_count << ", " << updated_time << ", " << created_on << ", " << date << ");"; @@ -883,8 +897,8 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { } Query filesToIndexQuery = connectionPtr->query(); - filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date, created_on " << - "FROM TableFiles " << + filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " << + META_TABLEFILES << " " << "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); @@ -967,8 +981,8 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, } Query filesToSearchQuery = connectionPtr->query(); - filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date " << - "FROM TableFiles " << + filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date FROM " << + META_TABLEFILES << " " << "WHERE table_id = " << quote << table_id; if (!partition.empty()) { @@ -1086,8 +1100,8 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, } Query filesToMergeQuery = connectionPtr->query(); - filesToMergeQuery << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, created_on " << - "FROM TableFiles " << + filesToMergeQuery << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, created_on FROM " << + META_TABLEFILES << " " << "WHERE table_id = " << quote << table_id << " AND " << "file_type = " << std::to_string(TableFileSchema::RAW) << " " << "ORDER BY row_count DESC" << ";"; @@ -1177,8 +1191,8 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, } Query getTableFileQuery = connectionPtr->query(); - getTableFileQuery << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on " << - "FROM TableFiles " << + getTableFileQuery << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " << + META_TABLEFILES << " " << "WHERE table_id = " << quote << table_id << " AND " << "(" << idStr << ") AND " << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; @@ -1259,7 +1273,8 @@ Status MySQLMetaImpl::Archive() { } Query archiveQuery = connectionPtr->query(); - archiveQuery << "UPDATE TableFiles " << + archiveQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << "WHERE created_on < " << std::to_string(now - usecs) << " AND " << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; @@ -1299,8 +1314,8 @@ Status MySQLMetaImpl::Size(uint64_t &result) { } Query getSizeQuery = connectionPtr->query(); - getSizeQuery << "SELECT IFNULL(SUM(file_size),0) AS sum " << - "FROM TableFiles " << + getSizeQuery << "SELECT IFNULL(SUM(file_size),0) AS sum FROM " << + META_TABLEFILES << " " << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str(); @@ -1339,8 +1354,8 @@ Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { } Query discardFilesQuery = connectionPtr->query(); - discardFilesQuery << "SELECT id, file_size " << - "FROM TableFiles " << + discardFilesQuery << "SELECT id, file_size FROM " << + META_TABLEFILES << " " << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " << "ORDER BY id ASC " << "LIMIT 10;"; @@ -1369,7 +1384,8 @@ Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { std::string idsToDiscardStr = idsToDiscardSS.str(); idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); //remove the last " OR " - discardFilesQuery << "UPDATE TableFiles " << + discardFilesQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << "WHERE " << idsToDiscardStr << ";"; @@ -1406,7 +1422,8 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { //if the table has been deleted, just mark the table file as TO_DELETE //clean thread will delete the file later - updateTableFileQuery << "SELECT state FROM Tables " << + updateTableFileQuery << "SELECT state FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << file_schema.table_id_ << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); @@ -1433,7 +1450,8 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { std::string created_on = std::to_string(file_schema.created_on_); std::string date = std::to_string(file_schema.date_); - updateTableFileQuery << "UPDATE TableFiles " << + updateTableFileQuery << "UPDATE " << + META_TABLEFILES << " " << "SET table_id = " << quote << table_id << ", " << "engine_type = " << engine_type << ", " << "file_id = " << quote << file_id << ", " << @@ -1470,7 +1488,8 @@ Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { Query updateTableFilesToIndexQuery = connectionPtr->query(); - updateTableFilesToIndexQuery << "UPDATE TableFiles " << + updateTableFilesToIndexQuery << "UPDATE " << + META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " " << "WHERE table_id = " << quote << table_id << " AND " << "file_type = " << std::to_string(TableFileSchema::RAW) << ";"; @@ -1508,7 +1527,8 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { } updateTableFilesQuery << "SELECT EXISTS " << - "(SELECT 1 FROM Tables " << + "(SELECT 1 FROM " << + META_TABLES << " " << "WHERE table_id = " << quote << file_schema.table_id_ << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << "AS " << quote << "check" << ";"; @@ -1539,7 +1559,8 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { std::string created_on = std::to_string(file_schema.created_on_); std::string date = std::to_string(file_schema.date_); - updateTableFilesQuery << "UPDATE TableFiles " << + updateTableFilesQuery << "UPDATE " << + META_TABLEFILES << " " << "SET table_id = " << quote << table_id << ", " << "engine_type = " << engine_type << ", " << "file_id = " << quote << file_id << ", " << @@ -1582,8 +1603,8 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date " << - "FROM TableFiles " << + cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date FROM " << + META_TABLEFILES << " " << "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; @@ -1626,8 +1647,9 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { std::string idsToDeleteStr = idsToDeleteSS.str(); idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " << - idsToDeleteStr << ";"; + cleanUpFilesWithTTLQuery << "DELETE FROM " << + META_TABLEFILES << " " << + "WHERE " << idsToDeleteStr << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1653,8 +1675,8 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id " << - "FROM Tables " << + cleanUpFilesWithTTLQuery << "SELECT id, table_id FROM " << + META_TABLES << " " << "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1675,8 +1697,9 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } std::string idsToDeleteStr = idsToDeleteSS.str(); idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " << - idsToDeleteStr << ";"; + cleanUpFilesWithTTLQuery << "DELETE FROM " << + META_TABLES << " " << + "WHERE " << idsToDeleteStr << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1704,8 +1727,8 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { for(auto& table_id : table_ids) { Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT file_id " << - "FROM TableFiles " << + cleanUpFilesWithTTLQuery << "SELECT file_id FROM " << + META_TABLEFILES << " " << "WHERE table_id = " << quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1736,7 +1759,7 @@ Status MySQLMetaImpl::CleanUp() { cleanUpQuery << "SELECT table_name " << "FROM information_schema.tables " << "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << " " << - "AND table_name = " << quote << "TableFiles" << ";"; + "AND table_name = " << quote << META_TABLEFILES << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); @@ -1744,7 +1767,7 @@ Status MySQLMetaImpl::CleanUp() { if (!res.empty()) { ENGINE_LOG_DEBUG << "Remove table file type as NEW"; - cleanUpQuery << "DELETE FROM TableFiles WHERE file_type IN (" + cleanUpQuery << "DELETE FROM " << META_TABLEFILES << " WHERE file_type IN (" << std::to_string(TableFileSchema::NEW) << "," << std::to_string(TableFileSchema::NEW_MERGE) << "," << std::to_string(TableFileSchema::NEW_INDEX) << ");"; @@ -1785,8 +1808,8 @@ Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { Query countQuery = connectionPtr->query(); - countQuery << "SELECT row_count " << - "FROM TableFiles " << + countQuery << "SELECT row_count FROM " << + META_TABLEFILES << " " << "WHERE table_id = " << quote << table_id << " AND " << "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << @@ -1820,7 +1843,7 @@ Status MySQLMetaImpl::DropAll() { } Query dropTableQuery = connectionPtr->query(); - dropTableQuery << "DROP TABLE IF EXISTS Tables, TableFiles;"; + dropTableQuery << "DROP TABLE IF EXISTS " << META_TABLES << ", " << META_TABLEFILES << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str(); diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 564a8eae4d..a28038d019 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -19,6 +19,7 @@ #include "db/IDGenerator.h" #include "db/Utils.h" #include "utils/Log.h" +#include "utils/Exception.h" #include "MetaConsts.h" #include "metrics/Metrics.h" @@ -55,7 +56,7 @@ Status HandleException(const std::string &desc, const char* what = nullptr) { inline auto StoragePrototype(const std::string &path) { return make_storage(path, - make_table("Tables", + make_table(META_TABLES, make_column("id", &TableSchema::id_, primary_key()), make_column("table_id", &TableSchema::table_id_, unique()), make_column("state", &TableSchema::state_), @@ -66,7 +67,7 @@ inline auto StoragePrototype(const std::string &path) { make_column("engine_type", &TableSchema::engine_type_), make_column("nlist", &TableSchema::nlist_), make_column("metric_type", &TableSchema::metric_type_)), - make_table("TableFiles", + make_table(META_TABLEFILES, make_column("id", &TableFileSchema::id_, primary_key()), make_column("table_id", &TableFileSchema::table_id_), make_column("engine_type", &TableFileSchema::engine_type_), @@ -122,6 +123,17 @@ Status SqliteMetaImpl::Initialize() { ConnectorPtr = std::make_unique(StoragePrototype(options_.path + "/meta.sqlite")); + //old meta could be recreated since schema changed, throw exception if meta schema is not compatible + auto ret = ConnectorPtr->sync_schema_simulate(); + if(ret.find(META_TABLES) != ret.end() + && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLES]) { + throw Exception(DB_INCOMPATIB_META, "Meta schema is created by Milvus old version"); + } + if(ret.find(META_TABLEFILES) != ret.end() + && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLEFILES]) { + throw Exception(DB_INCOMPATIB_META, "Meta schema is created by Milvus old version"); + } + ConnectorPtr->sync_schema(); ConnectorPtr->open_forever(); // thread safe option ConnectorPtr->pragma.journal_mode(journal_mode::WAL); // WAL => write ahead log @@ -1246,8 +1258,8 @@ Status SqliteMetaImpl::DropAll() { ENGINE_LOG_DEBUG << "Drop all sqlite meta"; try { - ConnectorPtr->drop_table("Tables"); - ConnectorPtr->drop_table("TableFiles"); + ConnectorPtr->drop_table(META_TABLES); + ConnectorPtr->drop_table(META_TABLEFILES); } catch (std::exception &e) { return HandleException("Encounter exception when drop all meta", e.what()); } diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index 54f9541eff..5a61c5bfb4 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -102,10 +102,6 @@ main(int argc, char *argv[]) { } } - server::Server &server = server::Server::Instance(); - server.Init(start_daemonized, pid_filename, config_filename, log_config_file); - server.Start(); - /* Handle Signal */ signal(SIGHUP, server::SignalUtil::HandleSignal); signal(SIGINT, server::SignalUtil::HandleSignal); @@ -114,6 +110,10 @@ main(int argc, char *argv[]) { signal(SIGUSR2, server::SignalUtil::HandleSignal); signal(SIGTERM, server::SignalUtil::HandleSignal); + server::Server &server = server::Server::Instance(); + server.Init(start_daemonized, pid_filename, config_filename, log_config_file); + server.Start(); + /* wait signal */ pause(); diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 115a636621..1409fff597 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -109,15 +109,10 @@ Status DBWrapper::StartService() { } //create db instance - std::string msg = opt.meta.path; try { db_ = engine::DBFactory::Build(opt); } catch(std::exception& ex) { - msg = ex.what(); - } - - if(db_ == nullptr) { - std::cerr << "ERROR! Failed to open database: " << msg << std::endl; + std::cerr << "ERROR! Failed to open database: " << ex.what() << std::endl; kill(0, SIGUSR1); } diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 38b41a5331..2d0a4e5c99 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -196,8 +196,8 @@ Server::Start() { server::Metrics::GetInstance().Init(); server::SystemInfo::GetInstance().Init(); - std::cout << "Milvus server start successfully." << std::endl; StartService(); + std::cout << "Milvus server start successfully." << std::endl; } catch (std::exception &ex) { std::cerr << "Milvus server encounter exception: " << ex.what(); diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index d7008115f9..672aa92848 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -86,6 +86,7 @@ constexpr ErrorCode DB_ERROR = ToDbErrorCode(2); constexpr ErrorCode DB_NOT_FOUND = ToDbErrorCode(3); constexpr ErrorCode DB_ALREADY_EXIST = ToDbErrorCode(4); constexpr ErrorCode DB_INVALID_PATH = ToDbErrorCode(5); +constexpr ErrorCode DB_INCOMPATIB_META = ToDbErrorCode(6); //knowhere error code constexpr ErrorCode KNOWHERE_ERROR = ToKnowhereErrorCode(1); diff --git a/cpp/src/utils/Exception.h b/cpp/src/utils/Exception.h index acb796c61d..6542f913af 100644 --- a/cpp/src/utils/Exception.h +++ b/cpp/src/utils/Exception.h @@ -17,6 +17,8 @@ #pragma once +#include "utils/Error.h" + #include #include @@ -25,13 +27,14 @@ namespace milvus { class Exception : public std::exception { public: - Exception(const std::string& message) - : message_(message) { + Exception(ErrorCode code, const std::string& message) + : code_(code_), + message_(message) { } - Exception() - : message_() { - } + ErrorCode code() const throw() { + return code_; + } virtual const char* what() const throw() { if (message_.empty()) { @@ -44,20 +47,21 @@ public: virtual ~Exception() throw() {}; protected: - + ErrorCode code_; std::string message_; }; class InvalidArgumentException : public Exception { public: - InvalidArgumentException() : Exception("Invalid Argument"){}; - InvalidArgumentException(const std::string& message) : Exception(message) {}; -}; + InvalidArgumentException() + : Exception(SERVER_INVALID_ARGUMENT, "Invalid Argument") { + + }; + InvalidArgumentException(const std::string& message) + : Exception(SERVER_INVALID_ARGUMENT, message) { + + }; -class OutOfRangeException : public Exception { -public: - OutOfRangeException() : Exception("Out Of Range"){}; - OutOfRangeException(const std::string& message) : Exception(message) {}; }; } // namespace milvus diff --git a/cpp/src/utils/Log.h b/cpp/src/utils/Log.h index 5bb28696e2..2610a3a8bf 100644 --- a/cpp/src/utils/Log.h +++ b/cpp/src/utils/Log.h @@ -24,7 +24,6 @@ namespace milvus { ///////////////////////////////////////////////////////////////////////////////////////////////// #define SERVER_DOMAIN_NAME "[SERVER] " -#define SERVER_ERROR_TEXT "SERVER Error:" #define SERVER_LOG_TRACE LOG(TRACE) << SERVER_DOMAIN_NAME #define SERVER_LOG_DEBUG LOG(DEBUG) << SERVER_DOMAIN_NAME @@ -35,7 +34,6 @@ namespace milvus { ///////////////////////////////////////////////////////////////////////////////////////////////// #define ENGINE_DOMAIN_NAME "[ENGINE] " -#define ENGINE_ERROR_TEXT "ENGINE Error:" #define ENGINE_LOG_TRACE LOG(TRACE) << ENGINE_DOMAIN_NAME #define ENGINE_LOG_DEBUG LOG(DEBUG) << ENGINE_DOMAIN_NAME @@ -46,7 +44,6 @@ namespace milvus { ///////////////////////////////////////////////////////////////////////////////////////////////// #define WRAPPER_DOMAIN_NAME "[WRAPPER] " -#define WRAPPER_ERROR_TEXT "WRAPPER Error:" #define WRAPPER_LOG_TRACE LOG(TRACE) << WRAPPER_DOMAIN_NAME #define WRAPPER_LOG_DEBUG LOG(DEBUG) << WRAPPER_DOMAIN_NAME diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp index e4af1104db..cfdbbdc768 100644 --- a/cpp/src/utils/LogUtil.cpp +++ b/cpp/src/utils/LogUtil.cpp @@ -17,10 +17,8 @@ #include "LogUtil.h" #include "server/ServerConfig.h" -#include "easylogging++.h" #include - #include #include diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index a1fd14a8f4..f2f8121538 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -24,10 +24,7 @@ #include "utils/CommonUtil.h" #include -#include "utils/easylogging++.h" - #include - #include #include diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 62271b9a3b..7a6ee31243 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -15,17 +15,16 @@ // specific language governing permissions and limitations // under the License. -#include -#include -#include "utils/easylogging++.h" -#include -#include - #include "utils.h" #include "db/meta/SqliteMetaImpl.h" #include "db/Utils.h" #include "db/meta/MetaConsts.h" +#include +#include +#include +#include + using namespace zilliz::milvus; using namespace zilliz::milvus::engine; diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index ff530b0a73..dd388dab7a 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -21,7 +21,6 @@ #include "db/Utils.h" #include "utils/Status.h" #include "utils/Exception.h" -#include "utils/easylogging++.h" #include #include @@ -31,13 +30,13 @@ using namespace zilliz::milvus; TEST(DBMiscTest, EXCEPTION_TEST) { - Exception ex1(""); + Exception ex1(100, "error"); std::string what = ex1.what(); - ASSERT_FALSE(what.empty()); + ASSERT_EQ(what, "error"); + ASSERT_EQ(ex1.code(), 100); - OutOfRangeException ex2; - what = ex2.what(); - ASSERT_FALSE(what.empty()); + InvalidArgumentException ex2; + ASSERT_EQ(ex2.code(), SERVER_INVALID_ARGUMENT); } TEST(DBMiscTest, OPTIONS_TEST) { diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp index 9d99b4095b..53ce51ca45 100644 --- a/cpp/unittest/db/mysql_db_test.cpp +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -21,7 +21,6 @@ #include "db/meta/MetaConsts.h" #include -#include "utils/easylogging++.h" #include #include diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index e99565886d..a7c9837f14 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -15,20 +15,17 @@ // specific language governing permissions and limitations // under the License. -#include -#include -#include "utils/easylogging++.h" -#include -#include - #include "utils.h" #include "db/meta/MySQLMetaImpl.h" #include "db/Utils.h" #include "db/meta/MetaConsts.h" -#include "mysql++/mysql++.h" - #include +#include +#include +#include +#include +#include using namespace zilliz::milvus; using namespace zilliz::milvus::engine; diff --git a/cpp/unittest/main.cpp b/cpp/unittest/main.cpp index b0c2be6d5a..33a860945d 100644 --- a/cpp/unittest/main.cpp +++ b/cpp/unittest/main.cpp @@ -15,13 +15,13 @@ // specific language governing permissions and limitations // under the License. -#include -#include #include "utils/easylogging++.h" - #include "server/ServerConfig.h" #include "utils/CommonUtil.h" +#include +#include + INITIALIZE_EASYLOGGINGPP using namespace zilliz::milvus; diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index b582c9a17a..169b2e23eb 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -15,14 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include -#include -#include "utils/easylogging++.h" -#include -#include -#include -#include - +#include "utils/SignalUtil.h" #include "utils/CommonUtil.h" #include "utils/Error.h" #include "utils/StringHelpFunctions.h" @@ -32,6 +25,12 @@ #include "utils/ValidationUtil.h" #include "db/engine/ExecutionEngine.h" +#include +#include +#include +#include +#include + using namespace zilliz::milvus; namespace { diff --git a/cpp/unittest/wrapper/wrapper_test.cpp b/cpp/unittest/wrapper/wrapper_test.cpp index ac185efbb3..4c0226f637 100644 --- a/cpp/unittest/wrapper/wrapper_test.cpp +++ b/cpp/unittest/wrapper/wrapper_test.cpp @@ -16,14 +16,13 @@ // under the License. -#include #include "utils/easylogging++.h" - -#include "src/wrapper/vec_index.h" +#include "wrapper/vec_index.h" #include "knowhere/index/vector_index/gpu_ivf.h" - #include "utils.h" +#include + INITIALIZE_EASYLOGGINGPP using namespace zilliz::milvus::engine; From 8e17f4826db59c7fa6b5f08c4cd800f7ba8800cf Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 20 Sep 2019 11:44:52 +0800 Subject: [PATCH 058/217] modify readme Former-commit-id: 4a5f9bd6666d283a5fe8f22a713fe0e587f5f6ad --- cpp/README.md | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index d32f04f316..5c7a1d280c 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -68,16 +68,10 @@ $ sudo ln -s /path/to/libmysqlclient.so /path/to/libmysqlclient_r.so ###### Step 2 Build ```shell -TBD -cd [Milvus sourcecode path]/cpp/thirdparty -git clone git@192.168.1.105:megasearch/knowhere.git -cd knowhere -./build.sh -t Debug -or ./build.sh -t Release - -cd [sourcecode path]/cpp -./build.sh -t Debug -or ./build.sh -t Release +$ cd [Milvus sourcecode path]/cpp +$ ./build.sh -t Debug +or +$ ./build.sh -t Release ``` When the build is completed, all the stuff that you need in order to run Milvus will be installed under `[Milvus root path]/cpp/milvus`. From 0ff59df1a486d180dbc44182f49f06b4838d1635 Mon Sep 17 00:00:00 2001 From: test Date: Fri, 20 Sep 2019 06:57:23 +0000 Subject: [PATCH 059/217] MS-575 Add Clang-format & Clang-tidy & Cpplint Former-commit-id: e62b3b42b0b27c3bbbb6d62cfff4a518913298d7 --- cpp/.gitignore | 1 + cpp/CMakeLists.txt | 49 ++++- ...mat_exclusions.txt => lint_exclusions.txt} | 0 cpp/build-support/lintutils.py | 110 ++++++++++++ cpp/build-support/run_clang_format.py | 167 ++++++++++++------ cpp/cmake/FindClangTools.cmake | 7 +- 6 files changed, 277 insertions(+), 57 deletions(-) rename cpp/build-support/{clang_format_exclusions.txt => lint_exclusions.txt} (100%) create mode 100755 cpp/build-support/lintutils.py diff --git a/cpp/.gitignore b/cpp/.gitignore index 71ee3f3770..f7243a0cc0 100644 --- a/cpp/.gitignore +++ b/cpp/.gitignore @@ -8,3 +8,4 @@ output.info output_new.info server.info thirdparty/knowhere/ +*.pyc diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 01007bf085..d9267be7c4 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -90,6 +90,8 @@ else() set(MILVUS_BUILD_ARCH unknown) endif() +find_package (Python COMPONENTS Interpreter Development) + find_package(CUDA) set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda") @@ -157,8 +159,47 @@ install(FILES DESTINATION conf) -add_custom_target(format ${BUILD_SUPPORT_DIR}/run_clang_format.py - ${CLANG_FORMAT_BIN} - ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt - ${CMAKE_CURRENT_SOURCE_DIR}/src) + +# +# "make lint" target +# +if(NOT MILVUS_VERBOSE_LINT) + set(MILVUS_LINT_QUIET "--quiet") +endif() + +if(NOT LINT_EXCLUSIONS_FILE) + # source files matching a glob from a line in this file + # will be excluded from linting (cpplint, clang-tidy, clang-format) + set(LINT_EXCLUSIONS_FILE ${BUILD_SUPPORT_DIR}/lint_exclusions.txt) +endif() + +# +# "make format" and "make check-format" targets +# +if(${CLANG_FORMAT_FOUND}) + # runs clang format and updates files in place. + add_custom_target(format + ${PYTHON_EXECUTABLE} + ${BUILD_SUPPORT_DIR}/run_clang_format.py + --clang_format_binary + ${CLANG_FORMAT_BIN} + --exclude_globs + ${LINT_EXCLUSIONS_FILE} + --source_dir + ${CMAKE_CURRENT_SOURCE_DIR}/src + --fix + ${MILVUS_LINT_QUIET}) + + # runs clang format and exits with a non-zero exit code if any files need to be reformatted + add_custom_target(check-format + ${PYTHON_EXECUTABLE} + ${BUILD_SUPPORT_DIR}/run_clang_format.py + --clang_format_binary + ${CLANG_FORMAT_BIN} + --exclude_globs + ${LINT_EXCLUSIONS_FILE} + --source_dir + ${CMAKE_CURRENT_SOURCE_DIR}/src + ${MILVUS_LINT_QUIET}) +endif() diff --git a/cpp/build-support/clang_format_exclusions.txt b/cpp/build-support/lint_exclusions.txt similarity index 100% rename from cpp/build-support/clang_format_exclusions.txt rename to cpp/build-support/lint_exclusions.txt diff --git a/cpp/build-support/lintutils.py b/cpp/build-support/lintutils.py new file mode 100755 index 0000000000..b4b01da3e9 --- /dev/null +++ b/cpp/build-support/lintutils.py @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import multiprocessing as mp +import os +from fnmatch import fnmatch +from subprocess import Popen + + +def chunk(seq, n): + """ + divide a sequence into equal sized chunks + (the last chunk may be smaller, but won't be empty) + """ + chunks = [] + some = [] + for element in seq: + if len(some) == n: + chunks.append(some) + some = [] + some.append(element) + if len(some) > 0: + chunks.append(some) + return chunks + + +def dechunk(chunks): + "flatten chunks into a single list" + seq = [] + for chunk in chunks: + seq.extend(chunk) + return seq + + +def run_parallel(cmds, **kwargs): + """ + Run each of cmds (with shared **kwargs) using subprocess.Popen + then wait for all of them to complete. + Runs batches of multiprocessing.cpu_count() * 2 from cmds + returns a list of tuples containing each process' + returncode, stdout, stderr + """ + complete = [] + for cmds_batch in chunk(cmds, mp.cpu_count() * 2): + procs_batch = [Popen(cmd, **kwargs) for cmd in cmds_batch] + for proc in procs_batch: + stdout, stderr = proc.communicate() + complete.append((proc.returncode, stdout, stderr)) + return complete + + +_source_extensions = ''' +.h +.cc +.cpp +'''.split() + + +def get_sources(source_dir, exclude_globs=[]): + sources = [] + for directory, subdirs, basenames in os.walk(source_dir): + for path in [os.path.join(directory, basename) + for basename in basenames]: + # filter out non-source files + if os.path.splitext(path)[1] not in _source_extensions: + continue + + path = os.path.abspath(path) + + # filter out files that match the globs in the globs file + if any([fnmatch(path, glob) for glob in exclude_globs]): + continue + + sources.append(path) + return sources + + +def stdout_pathcolonline(completed_process, filenames): + """ + given a completed process which may have reported some files as problematic + by printing the path name followed by ':' then a line number, examine + stdout and return the set of actually reported file names + """ + returncode, stdout, stderr = completed_process + bfilenames = set() + for filename in filenames: + bfilenames.add(filename.encode('utf-8') + b':') + problem_files = set() + for line in stdout.splitlines(): + for filename in bfilenames: + if line.startswith(filename): + problem_files.add(filename.decode('utf-8')) + bfilenames.remove(filename) + break + return problem_files, stdout + diff --git a/cpp/build-support/run_clang_format.py b/cpp/build-support/run_clang_format.py index 4235b2d4af..4c7fdfb2af 100755 --- a/cpp/build-support/run_clang_format.py +++ b/cpp/build-support/run_clang_format.py @@ -16,64 +16,127 @@ # specific language governing permissions and limitations # under the License. -import fnmatch -import os -import subprocess +from __future__ import print_function +import lintutils +from subprocess import PIPE +import argparse +import difflib +import multiprocessing as mp import sys - -if len(sys.argv) < 4: - sys.stderr.write("Usage: %s $CLANG_FORMAT_VERSION exclude_globs.txt " - "$source_dir\n" % - sys.argv[0]) - sys.exit(1) - -CLANG_FORMAT = sys.argv[1] -EXCLUDE_GLOBS_FILENAME = sys.argv[2] -SOURCE_DIR = sys.argv[3] - -if len(sys.argv) > 4: - CHECK_FORMAT = int(sys.argv[4]) == 1 -else: - CHECK_FORMAT = False +from functools import partial -exclude_globs = [line.strip() for line in open(EXCLUDE_GLOBS_FILENAME, "r")] +# examine the output of clang-format and if changes are +# present assemble a (unified)patch of the difference +def _check_one_file(completed_processes, filename): + with open(filename, "rb") as reader: + original = reader.read() -files_to_format = [] -matches = [] -for directory, subdirs, files in os.walk(SOURCE_DIR): - for name in files: - name = os.path.join(directory, name) - if not (name.endswith('.h') or name.endswith('.cpp') or name.endswith('.cuh') or name.endswith('.cu')): - continue + returncode, stdout, stderr = completed_processes[filename] + formatted = stdout + if formatted != original: + # Run the equivalent of diff -u + diff = list(difflib.unified_diff( + original.decode('utf8').splitlines(True), + formatted.decode('utf8').splitlines(True), + fromfile=filename, + tofile="{} (after clang format)".format( + filename))) + else: + diff = None - excluded = False - for g in exclude_globs: - if fnmatch.fnmatch(name, g): - excluded = True - break - if not excluded: - files_to_format.append(name) + return filename, diff -if CHECK_FORMAT: - output = subprocess.check_output([CLANG_FORMAT, '-output-replacements-xml'] - + files_to_format, - stderr=subprocess.STDOUT).decode('utf8') - to_fix = [] - for line in output.split('\n'): - if 'offset' in line: - to_fix.append(line) +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Runs clang-format on all of the source " + "files. If --fix is specified enforce format by " + "modifying in place, otherwise compare the output " + "with the existing file and output any necessary " + "changes as a patch in unified diff format") + parser.add_argument("--clang_format_binary", + required=True, + help="Path to the clang-format binary") + parser.add_argument("--exclude_globs", + help="Filename containing globs for files " + "that should be excluded from the checks") + parser.add_argument("--source_dir", + required=True, + help="Root directory of the source code") + parser.add_argument("--fix", default=False, + action="store_true", + help="If specified, will re-format the source " + "code instead of comparing the re-formatted " + "output, defaults to %(default)s") + parser.add_argument("--quiet", default=False, + action="store_true", + help="If specified, only print errors") + arguments = parser.parse_args() - if len(to_fix) > 0: - print("clang-format checks failed, run 'make format' to fix") - sys.exit(-1) -else: - try: - cmd = [CLANG_FORMAT, '-i'] + files_to_format - subprocess.check_output(cmd, stderr=subprocess.STDOUT) - except Exception as e: - print(e) - print(' '.join(cmd)) - raise + exclude_globs = [] + if arguments.exclude_globs: + for line in open(arguments.exclude_globs): + exclude_globs.append(line.strip()) + + formatted_filenames = [] + for path in lintutils.get_sources(arguments.source_dir, exclude_globs): + formatted_filenames.append(str(path)) + + if arguments.fix: + if not arguments.quiet: + print("\n".join(map(lambda x: "Formatting {}".format(x), + formatted_filenames))) + + # Break clang-format invocations into chunks: each invocation formats + # 16 files. Wait for all processes to complete + results = lintutils.run_parallel([ + [arguments.clang_format_binary, "-i"] + some + for some in lintutils.chunk(formatted_filenames, 16) + ]) + for returncode, stdout, stderr in results: + # if any clang-format reported a parse error, bubble it + if returncode != 0: + sys.exit(returncode) + + else: + # run an instance of clang-format for each source file in parallel, + # then wait for all processes to complete + results = lintutils.run_parallel([ + [arguments.clang_format_binary, filename] + for filename in formatted_filenames + ], stdout=PIPE, stderr=PIPE) + for returncode, stdout, stderr in results: + # if any clang-format reported a parse error, bubble it + if returncode != 0: + sys.exit(returncode) + + error = False + checker = partial(_check_one_file, { + filename: result + for filename, result in zip(formatted_filenames, results) + }) + pool = mp.Pool() + try: + # check the output from each invocation of clang-format in parallel + for filename, diff in pool.imap(checker, formatted_filenames): + if not arguments.quiet: + print("Checking {}".format(filename)) + if diff: + print("{} had clang-format style issues".format(filename)) + # Print out the diff to stderr + error = True + # pad with a newline + print(file=sys.stderr) + diff_out = [] + for diff_str in diff: + diff_out.append(diff_str.encode('raw_unicode_escape')) + sys.stderr.writelines(diff_out) + except Exception: + error = True + raise + finally: + pool.terminate() + pool.join() + sys.exit(1 if error else 0) diff --git a/cpp/cmake/FindClangTools.cmake b/cpp/cmake/FindClangTools.cmake index 075d9381c5..83e71d750b 100644 --- a/cpp/cmake/FindClangTools.cmake +++ b/cpp/cmake/FindClangTools.cmake @@ -32,6 +32,7 @@ find_program(CLANG_TIDY_BIN NAMES + clang-tidy-7.0 clang-tidy-6.0 clang-tidy-5.0 clang-tidy-4.0 @@ -83,7 +84,11 @@ if (CLANG_FORMAT_VERSION) endif() else() find_program(CLANG_FORMAT_BIN - NAMES clang-format-4.0 + NAMES + clang-format-7.0 + clang-format-6.0 + clang-format-5.0 + clang-format-4.0 clang-format-3.9 clang-format-3.8 clang-format-3.7 From e364d7fa1131b222ab82745ce91d13e3fd925a9c Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Fri, 20 Sep 2019 15:33:42 +0800 Subject: [PATCH 060/217] Update CONTRIBUTING.md Former-commit-id: 9c22848294d0e339b76d38637b2d0494842cc4b7 --- cpp/CONTRIBUTING.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cpp/CONTRIBUTING.md b/cpp/CONTRIBUTING.md index 4f60946d76..e7e0070b05 100644 --- a/cpp/CONTRIBUTING.md +++ b/cpp/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing to Milvus -First of all, thanks for taking the time to contribute to Milvus! It's people like you that help Milvus come to fruition. +First of all, thanks for taking the time to contribute to Milvus!:tada::clap: It's people like you that help Milvus come to fruition. The following are a set of guidelines for contributing to Milvus. Following these guidelines helps contributing to this project easy and transparent. These are mostly guideline, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request. @@ -21,8 +21,8 @@ Before you make any contributions, make sure you follow this list. Contributions to Milvus fall into the following categories. 1. To report a bug or a problem with documentation, please file an [issue](https://github.com/milvus-io/milvus/issues/new/choose) providing the details of the problem. If you believe the issue needs priority attention, please comment on the issue to notify the team. -2. To propose a new feature, please file a new feature request [issue](https://github.com/milvus-io/milvus/issues/new/choose). Describe the intended feature and discuss the design and implementation with the team and community. Once the team agrees that the plan looks good, go ahead and implement it, following the [Contributing code]. -3. To implement a feature or bug-fix for an existing outstanding issue, follow the [Contributing code]. If you need more context on a particular issue, comment on the issue to let people know. +2. To propose a new feature, please file a new feature request [issue](https://github.com/milvus-io/milvus/issues/new/choose). Describe the intended feature and discuss the design and implementation with the team and community. Once the team agrees that the plan looks good, go ahead and implement it, following the [Contributing code](CONTRIBUTING.md#contributing-code). +3. To implement a feature or bug-fix for an existing outstanding issue, follow the [Contributing code](CONTRIBUTING.md#contributing-code). If you need more context on a particular issue, comment on the issue to let people know. ## How can I contribute? @@ -44,6 +44,7 @@ Before sending your pull requests for review, make sure your changes are consist ## Coding Style + ## Run unit test ```shell From 94b5ec4919e762a53a019c59fccaa681d292a18a Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 20 Sep 2019 15:40:10 +0800 Subject: [PATCH 062/217] fix typo Former-commit-id: a35553692abacc8cbb1f94f5924871a1202f3e08 --- cpp/src/utils/Exception.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/utils/Exception.h b/cpp/src/utils/Exception.h index 6542f913af..3a9ab89294 100644 --- a/cpp/src/utils/Exception.h +++ b/cpp/src/utils/Exception.h @@ -28,7 +28,7 @@ namespace milvus { class Exception : public std::exception { public: Exception(ErrorCode code, const std::string& message) - : code_(code_), + : code_(code), message_(message) { } From 36fd87481b0fd65262a6827f11b8b9182a395b5b Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Fri, 20 Sep 2019 15:40:52 +0800 Subject: [PATCH 063/217] Update CONTRIBUTING.md Former-commit-id: bb1f9f679c416bfd203960c8cb4e0052a977a6d8 --- cpp/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CONTRIBUTING.md b/cpp/CONTRIBUTING.md index e7e0070b05..9e23d6b0ed 100644 --- a/cpp/CONTRIBUTING.md +++ b/cpp/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing to Milvus -First of all, thanks for taking the time to contribute to Milvus!:tada::clap: It's people like you that help Milvus come to fruition. +First of all, thanks for taking the time to contribute to Milvus! It's people like you that help Milvus come to fruition. The following are a set of guidelines for contributing to Milvus. Following these guidelines helps contributing to this project easy and transparent. These are mostly guideline, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request. From 72ffa6c4fd038504987fa60a90f91f26e938dab9 Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Fri, 20 Sep 2019 16:10:29 +0800 Subject: [PATCH 064/217] MS-583 Change to Status from errorcode Former-commit-id: 63ba084d42e2fba75a581895cb3513666bff4329 --- cpp/src/db/engine/ExecutionEngineImpl.cpp | 32 ++---- cpp/src/wrapper/KnowhereResource.cpp | 10 +- cpp/src/wrapper/KnowhereResource.h | 9 +- cpp/src/wrapper/vec_impl.cpp | 133 ++++++++++++---------- cpp/src/wrapper/vec_impl.h | 104 +++++++++++------ cpp/src/wrapper/vec_index.cpp | 73 +++++++----- cpp/src/wrapper/vec_index.h | 91 +++++++++------ cpp/unittest/scheduler/scheduler_test.cpp | 12 +- cpp/unittest/server/cache_test.cpp | 17 ++- cpp/unittest/wrapper/CMakeLists.txt | 1 + 10 files changed, 284 insertions(+), 198 deletions(-) diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 7b9bac2e3a..83c69e6db3 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -99,11 +99,8 @@ VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) { } Status ExecutionEngineImpl::AddWithIds(long n, const float *xdata, const long *xids) { - auto ec = index_->Add(n, xdata, xids); - if (ec != KNOWHERE_SUCCESS) { - return Status(DB_ERROR, "Add error"); - } - return Status::OK(); + auto status = index_->Add(n, xdata, xids); + return status; } size_t ExecutionEngineImpl::Count() const { @@ -131,11 +128,8 @@ size_t ExecutionEngineImpl::PhysicalSize() const { } Status ExecutionEngineImpl::Serialize() { - auto ec = write_index(index_, location_); - if (ec != KNOWHERE_SUCCESS) { - return Status(DB_ERROR, "Serialize: write to disk error"); - } - return Status::OK(); + auto status = write_index(index_, location_); + return status; } Status ExecutionEngineImpl::Load(bool to_cache) { @@ -254,12 +248,11 @@ Status ExecutionEngineImpl::Merge(const std::string &location) { } if (auto file_index = std::dynamic_pointer_cast(to_merge)) { - auto ec = index_->Add(file_index->Count(), file_index->GetRawVectors(), file_index->GetRawIds()); - if (ec != KNOWHERE_SUCCESS) { + auto status = index_->Add(file_index->Count(), file_index->GetRawVectors(), file_index->GetRawIds()); + if (!status.ok()) { ENGINE_LOG_ERROR << "Merge: Add Error"; - return Status(DB_ERROR, "Merge: Add Error"); } - return Status::OK(); + return status; } else { return Status(DB_ERROR, "file index type is not idmap"); } @@ -287,11 +280,11 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t build_cfg["nlist"] = nlist_; AutoGenParams(to_index->GetType(), Count(), build_cfg); - auto ec = to_index->BuildAll(Count(), + auto status = to_index->BuildAll(Count(), from_index->GetRawVectors(), from_index->GetRawIds(), build_cfg); - if (ec != KNOWHERE_SUCCESS) { throw Exception(DB_ERROR, "Build index error"); } + if (!status.ok()) { throw Exception(DB_ERROR, status.message()); } return std::make_shared(to_index, location, engine_type, metric_type_, nlist_); } @@ -309,12 +302,11 @@ Status ExecutionEngineImpl::Search(long n, ENGINE_LOG_DEBUG << "Search Params: [k] " << k << " [nprobe] " << nprobe; auto cfg = Config::object{{"k", k}, {"nprobe", nprobe}}; - auto ec = index_->Search(n, data, distances, labels, cfg); - if (ec != KNOWHERE_SUCCESS) { + auto status = index_->Search(n, data, distances, labels, cfg); + if (!status.ok()) { ENGINE_LOG_ERROR << "Search error"; - return Status(DB_ERROR, "Search: Search Error"); } - return Status::OK(); + return status; } Status ExecutionEngineImpl::Cache() { diff --git a/cpp/src/wrapper/KnowhereResource.cpp b/cpp/src/wrapper/KnowhereResource.cpp index 007858de37..9c2b4d31e2 100644 --- a/cpp/src/wrapper/KnowhereResource.cpp +++ b/cpp/src/wrapper/KnowhereResource.cpp @@ -28,7 +28,8 @@ namespace engine { constexpr int64_t M_BYTE = 1024 * 1024; -ErrorCode KnowhereResource::Initialize() { +Status +KnowhereResource::Initialize() { struct GpuResourceSetting { int64_t pinned_memory = 300*M_BYTE; int64_t temp_memory = 300*M_BYTE; @@ -65,12 +66,13 @@ ErrorCode KnowhereResource::Initialize() { iter->second.resource_num); } - return KNOWHERE_SUCCESS; + return Status::OK(); } -ErrorCode KnowhereResource::Finalize() { +Status +KnowhereResource::Finalize() { knowhere::FaissGpuResourceMgr::GetInstance().Free(); // free gpu resource. - return KNOWHERE_SUCCESS; + return Status::OK(); } } diff --git a/cpp/src/wrapper/KnowhereResource.h b/cpp/src/wrapper/KnowhereResource.h index 5a7b9aa6c5..9e8c27b5af 100644 --- a/cpp/src/wrapper/KnowhereResource.h +++ b/cpp/src/wrapper/KnowhereResource.h @@ -18,7 +18,7 @@ #pragma once -#include "utils/Error.h" +#include "utils/Status.h" namespace zilliz { namespace milvus { @@ -26,8 +26,11 @@ namespace engine { class KnowhereResource { public: - static ErrorCode Initialize(); - static ErrorCode Finalize(); + static Status + Initialize(); + + static Status + Finalize(); }; diff --git a/cpp/src/wrapper/vec_impl.cpp b/cpp/src/wrapper/vec_impl.cpp index accb612c2d..e909e611b2 100644 --- a/cpp/src/wrapper/vec_impl.cpp +++ b/cpp/src/wrapper/vec_impl.cpp @@ -21,7 +21,6 @@ #include "knowhere/index/vector_index/IndexGPUIVF.h" #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/helpers/Cloner.h" - #include "vec_impl.h" #include "data_transfer.h" @@ -32,12 +31,13 @@ namespace engine { using namespace zilliz::knowhere; -ErrorCode VecIndexImpl::BuildAll(const long &nb, - const float *xb, - const long *ids, - const Config &cfg, - const long &nt, - const float *xt) { +Status +VecIndexImpl::BuildAll(const long &nb, + const float *xb, + const long *ids, + const Config &cfg, + const long &nt, + const float *xt) { try { dim = cfg["dim"].as(); auto dataset = GenDatasetWithIds(nb, dim, xb, ids); @@ -49,36 +49,38 @@ ErrorCode VecIndexImpl::BuildAll(const long &nb, index_->Add(dataset, cfg); } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); - return KNOWHERE_UNEXPECTED_ERROR; + return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); } catch (jsoncons::json_exception &e) { WRAPPER_LOG_ERROR << e.what(); - return KNOWHERE_INVALID_ARGUMENT; + return Status(KNOWHERE_INVALID_ARGUMENT, e.what()); } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); - return KNOWHERE_ERROR; + return Status(KNOWHERE_ERROR, e.what()); } - return KNOWHERE_SUCCESS; + return Status::OK(); } -ErrorCode VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const Config &cfg) { +Status +VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const Config &cfg) { try { auto dataset = GenDatasetWithIds(nb, dim, xb, ids); index_->Add(dataset, cfg); } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); - return KNOWHERE_UNEXPECTED_ERROR; + return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); } catch (jsoncons::json_exception &e) { WRAPPER_LOG_ERROR << e.what(); - return KNOWHERE_INVALID_ARGUMENT; + return Status(KNOWHERE_INVALID_ARGUMENT, e.what()); } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); - return KNOWHERE_ERROR; + return Status(KNOWHERE_ERROR, e.what()); } - return KNOWHERE_SUCCESS; + return Status::OK(); } -ErrorCode VecIndexImpl::Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) { +Status +VecIndexImpl::Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) { try { auto k = cfg["k"].as(); auto dataset = GenDataset(nq, dim, xq); @@ -117,41 +119,47 @@ ErrorCode VecIndexImpl::Search(const long &nq, const float *xq, float *dist, lon } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); - return KNOWHERE_UNEXPECTED_ERROR; + return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); } catch (jsoncons::json_exception &e) { WRAPPER_LOG_ERROR << e.what(); - return KNOWHERE_INVALID_ARGUMENT; + return Status(KNOWHERE_INVALID_ARGUMENT, e.what()); } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); - return KNOWHERE_ERROR; + return Status(KNOWHERE_ERROR, e.what()); } - return KNOWHERE_SUCCESS; + return Status::OK(); } -zilliz::knowhere::BinarySet VecIndexImpl::Serialize() { +zilliz::knowhere::BinarySet +VecIndexImpl::Serialize() { type = ConvertToCpuIndexType(type); return index_->Serialize(); } -ErrorCode VecIndexImpl::Load(const zilliz::knowhere::BinarySet &index_binary) { +Status +VecIndexImpl::Load(const zilliz::knowhere::BinarySet &index_binary) { index_->Load(index_binary); dim = Dimension(); - return KNOWHERE_SUCCESS; + return Status::OK(); } -int64_t VecIndexImpl::Dimension() { +int64_t +VecIndexImpl::Dimension() { return index_->Dimension(); } -int64_t VecIndexImpl::Count() { +int64_t +VecIndexImpl::Count() { return index_->Count(); } -IndexType VecIndexImpl::GetType() { +IndexType +VecIndexImpl::GetType() { return type; } -VecIndexPtr VecIndexImpl::CopyToGpu(const int64_t &device_id, const Config &cfg) { +VecIndexPtr +VecIndexImpl::CopyToGpu(const int64_t &device_id, const Config &cfg) { // TODO(linxj): exception handle auto gpu_index = zilliz::knowhere::cloner::CopyCpuToGpu(index_, device_id, cfg); auto new_index = std::make_shared(gpu_index, ConvertToGpuIndexType(type)); @@ -159,7 +167,8 @@ VecIndexPtr VecIndexImpl::CopyToGpu(const int64_t &device_id, const Config &cfg) return new_index; } -VecIndexPtr VecIndexImpl::CopyToCpu(const Config &cfg) { +VecIndexPtr +VecIndexImpl::CopyToCpu(const Config &cfg) { // TODO(linxj): exception handle auto cpu_index = zilliz::knowhere::cloner::CopyGpuToCpu(index_, cfg); auto new_index = std::make_shared(cpu_index, ConvertToCpuIndexType(type)); @@ -167,32 +176,37 @@ VecIndexPtr VecIndexImpl::CopyToCpu(const Config &cfg) { return new_index; } -VecIndexPtr VecIndexImpl::Clone() { +VecIndexPtr +VecIndexImpl::Clone() { // TODO(linxj): exception handle auto clone_index = std::make_shared(index_->Clone(), type); clone_index->dim = dim; return clone_index; } -int64_t VecIndexImpl::GetDeviceId() { - if (auto device_idx = std::dynamic_pointer_cast(index_)){ +int64_t +VecIndexImpl::GetDeviceId() { + if (auto device_idx = std::dynamic_pointer_cast(index_)) { return device_idx->GetGpuDevice(); } // else return -1; // -1 == cpu } -float *BFIndex::GetRawVectors() { +float * +BFIndex::GetRawVectors() { auto raw_index = std::dynamic_pointer_cast(index_); if (raw_index) { return raw_index->GetRawVectors(); } return nullptr; } -int64_t *BFIndex::GetRawIds() { +int64_t * +BFIndex::GetRawIds() { return std::static_pointer_cast(index_)->GetRawIds(); } -ErrorCode BFIndex::Build(const Config &cfg) { +ErrorCode +BFIndex::Build(const Config &cfg) { try { dim = cfg["dim"].as(); std::static_pointer_cast(index_)->Train(cfg); @@ -209,12 +223,13 @@ ErrorCode BFIndex::Build(const Config &cfg) { return KNOWHERE_SUCCESS; } -ErrorCode BFIndex::BuildAll(const long &nb, - const float *xb, - const long *ids, - const Config &cfg, - const long &nt, - const float *xt) { +Status +BFIndex::BuildAll(const long &nb, + const float *xb, + const long *ids, + const Config &cfg, + const long &nt, + const float *xt) { try { dim = cfg["dim"].as(); auto dataset = GenDatasetWithIds(nb, dim, xb, ids); @@ -223,24 +238,25 @@ ErrorCode BFIndex::BuildAll(const long &nb, index_->Add(dataset, cfg); } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); - return KNOWHERE_UNEXPECTED_ERROR; + return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); } catch (jsoncons::json_exception &e) { WRAPPER_LOG_ERROR << e.what(); - return KNOWHERE_INVALID_ARGUMENT; + return Status(KNOWHERE_INVALID_ARGUMENT, e.what()); } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); - return KNOWHERE_ERROR; + return Status(KNOWHERE_ERROR, e.what()); } - return KNOWHERE_SUCCESS; + return Status::OK(); } // TODO(linxj): add lock here. -ErrorCode IVFMixIndex::BuildAll(const long &nb, - const float *xb, - const long *ids, - const Config &cfg, - const long &nt, - const float *xt) { +Status +IVFMixIndex::BuildAll(const long &nb, + const float *xb, + const long *ids, + const Config &cfg, + const long &nt, + const float *xt) { try { dim = cfg["dim"].as(); auto dataset = GenDatasetWithIds(nb, dim, xb, ids); @@ -257,26 +273,27 @@ ErrorCode IVFMixIndex::BuildAll(const long &nb, type = ConvertToCpuIndexType(type); } else { WRAPPER_LOG_ERROR << "Build IVFMIXIndex Failed"; - return KNOWHERE_ERROR; + return Status(KNOWHERE_ERROR, "Build IVFMIXIndex Failed"); } } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); - return KNOWHERE_UNEXPECTED_ERROR; + return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); } catch (jsoncons::json_exception &e) { WRAPPER_LOG_ERROR << e.what(); - return KNOWHERE_INVALID_ARGUMENT; + return Status(KNOWHERE_INVALID_ARGUMENT, e.what()); } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); - return KNOWHERE_ERROR; + return Status(KNOWHERE_ERROR, e.what()); } - return KNOWHERE_SUCCESS; + return Status::OK(); } -ErrorCode IVFMixIndex::Load(const zilliz::knowhere::BinarySet &index_binary) { +Status +IVFMixIndex::Load(const zilliz::knowhere::BinarySet &index_binary) { //index_ = std::make_shared(); index_->Load(index_binary); dim = Dimension(); - return KNOWHERE_SUCCESS; + return Status::OK(); } } diff --git a/cpp/src/wrapper/vec_impl.h b/cpp/src/wrapper/vec_impl.h index 5e5edbd199..b02ffd690b 100644 --- a/cpp/src/wrapper/vec_impl.h +++ b/cpp/src/wrapper/vec_impl.h @@ -19,7 +19,6 @@ #pragma once #include "knowhere/index/vector_index/VectorIndex.h" - #include "vec_index.h" @@ -31,27 +30,53 @@ class VecIndexImpl : public VecIndex { public: explicit VecIndexImpl(std::shared_ptr index, const IndexType &type) : index_(std::move(index)), type(type) {}; - ErrorCode BuildAll(const long &nb, - const float *xb, - const long *ids, - const Config &cfg, - const long &nt, - const float *xt) override; - VecIndexPtr CopyToGpu(const int64_t &device_id, const Config &cfg) override; - VecIndexPtr CopyToCpu(const Config &cfg) override; - IndexType GetType() override; - int64_t Dimension() override; - int64_t Count() override; - ErrorCode Add(const long &nb, const float *xb, const long *ids, const Config &cfg) override; - zilliz::knowhere::BinarySet Serialize() override; - ErrorCode Load(const zilliz::knowhere::BinarySet &index_binary) override; - VecIndexPtr Clone() override; - int64_t GetDeviceId() override; - ErrorCode Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) override; + + Status + BuildAll(const long &nb, + const float *xb, + const long *ids, + const Config &cfg, + const long &nt, + const float *xt) override; + + VecIndexPtr + CopyToGpu(const int64_t &device_id, const Config &cfg) override; + + VecIndexPtr + CopyToCpu(const Config &cfg) override; + + IndexType + GetType() override; + + int64_t + Dimension() override; + + int64_t + Count() override; + + Status + Add(const long &nb, const float *xb, const long *ids, const Config &cfg) override; + + zilliz::knowhere::BinarySet + Serialize() override; + + Status + Load(const zilliz::knowhere::BinarySet &index_binary) override; + + VecIndexPtr + Clone() override; + + int64_t + GetDeviceId() override; + + Status + Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) override; protected: int64_t dim = 0; + IndexType type = IndexType::INVALID; + std::shared_ptr index_ = nullptr; }; @@ -60,28 +85,39 @@ class IVFMixIndex : public VecIndexImpl { explicit IVFMixIndex(std::shared_ptr index, const IndexType &type) : VecIndexImpl(std::move(index), type) {}; - ErrorCode BuildAll(const long &nb, - const float *xb, - const long *ids, - const Config &cfg, - const long &nt, - const float *xt) override; - ErrorCode Load(const zilliz::knowhere::BinarySet &index_binary) override; + Status + BuildAll(const long &nb, + const float *xb, + const long *ids, + const Config &cfg, + const long &nt, + const float *xt) override; + + Status + Load(const zilliz::knowhere::BinarySet &index_binary) override; }; class BFIndex : public VecIndexImpl { public: explicit BFIndex(std::shared_ptr index) : VecIndexImpl(std::move(index), IndexType::FAISS_IDMAP) {}; - ErrorCode Build(const Config& cfg); - float *GetRawVectors(); - ErrorCode BuildAll(const long &nb, - const float *xb, - const long *ids, - const Config &cfg, - const long &nt, - const float *xt) override; - int64_t *GetRawIds(); + + ErrorCode + Build(const Config &cfg); + + float * + GetRawVectors(); + + Status + BuildAll(const long &nb, + const float *xb, + const long *ids, + const Config &cfg, + const long &nt, + const float *xt) override; + + int64_t * + GetRawIds(); }; } diff --git a/cpp/src/wrapper/vec_index.cpp b/cpp/src/wrapper/vec_index.cpp index 7ed176d96e..17a27caa74 100644 --- a/cpp/src/wrapper/vec_index.cpp +++ b/cpp/src/wrapper/vec_index.cpp @@ -25,7 +25,6 @@ #include "knowhere/index/vector_index/IndexKDT.h" #include "knowhere/index/vector_index/IndexNSG.h" #include "knowhere/common/Exception.h" - #include "vec_index.h" #include "vec_impl.h" #include "utils/Log.h" @@ -39,23 +38,19 @@ namespace engine { static constexpr float TYPICAL_COUNT = 1000000.0; -struct FileIOWriter { - std::fstream fs; - std::string name; - - FileIOWriter(const std::string &fname); - ~FileIOWriter(); - size_t operator()(void *ptr, size_t size); -}; - struct FileIOReader { std::fstream fs; std::string name; FileIOReader(const std::string &fname); + ~FileIOReader(); - size_t operator()(void *ptr, size_t size); - size_t operator()(void *ptr, size_t size, size_t pos); + + size_t + operator()(void *ptr, size_t size); + + size_t + operator()(void *ptr, size_t size, size_t pos); }; FileIOReader::FileIOReader(const std::string &fname) { @@ -67,14 +62,27 @@ FileIOReader::~FileIOReader() { fs.close(); } -size_t FileIOReader::operator()(void *ptr, size_t size) { +size_t +FileIOReader::operator()(void *ptr, size_t size) { fs.read(reinterpret_cast(ptr), size); } -size_t FileIOReader::operator()(void *ptr, size_t size, size_t pos) { +size_t +FileIOReader::operator()(void *ptr, size_t size, size_t pos) { return 0; } + + +struct FileIOWriter { + std::fstream fs; + std::string name; + + FileIOWriter(const std::string &fname); + ~FileIOWriter(); + size_t operator()(void *ptr, size_t size); +}; + FileIOWriter::FileIOWriter(const std::string &fname) { name = fname; fs = std::fstream(name, std::ios::out | std::ios::binary); @@ -84,12 +92,14 @@ FileIOWriter::~FileIOWriter() { fs.close(); } -size_t FileIOWriter::operator()(void *ptr, size_t size) { +size_t +FileIOWriter::operator()(void *ptr, size_t size) { fs.write(reinterpret_cast(ptr), size); } -VecIndexPtr GetVecIndexFactory(const IndexType &type, const Config &cfg) { +VecIndexPtr +GetVecIndexFactory(const IndexType &type, const Config &cfg) { std::shared_ptr index; auto gpu_device = cfg.get_with_default("gpu_id", 0); switch (type) { @@ -145,13 +155,15 @@ VecIndexPtr GetVecIndexFactory(const IndexType &type, const Config &cfg) { return std::make_shared(index, type); } -VecIndexPtr LoadVecIndex(const IndexType &index_type, const zilliz::knowhere::BinarySet &index_binary) { +VecIndexPtr +LoadVecIndex(const IndexType &index_type, const zilliz::knowhere::BinarySet &index_binary) { auto index = GetVecIndexFactory(index_type); index->Load(index_binary); return index; } -VecIndexPtr read_index(const std::string &location) { +VecIndexPtr +read_index(const std::string &location) { knowhere::BinarySet load_data_list; FileIOReader reader(location); reader.fs.seekg(0, reader.fs.end); @@ -195,7 +207,8 @@ VecIndexPtr read_index(const std::string &location) { return LoadVecIndex(current_type, load_data_list); } -ErrorCode write_index(VecIndexPtr index, const std::string &location) { +Status +write_index(VecIndexPtr index, const std::string &location) { try { auto binaryset = index->Serialize(); auto index_type = index->GetType(); @@ -215,28 +228,29 @@ ErrorCode write_index(VecIndexPtr index, const std::string &location) { } } catch (knowhere::KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); - return KNOWHERE_UNEXPECTED_ERROR; + return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); std::string estring(e.what()); if (estring.find("No space left on device") != estring.npos) { WRAPPER_LOG_ERROR << "No space left on the device"; - return KNOWHERE_NO_SPACE; + return Status(KNOWHERE_NO_SPACE, "No space left on the device"); } else { - return KNOWHERE_ERROR; + return Status(KNOWHERE_ERROR, e.what()); } } - return KNOWHERE_SUCCESS; + return Status::OK(); } // TODO(linxj): redo here. -void AutoGenParams(const IndexType &type, const long &size, zilliz::knowhere::Config &cfg) { +void +AutoGenParams(const IndexType &type, const long &size, zilliz::knowhere::Config &cfg) { auto nlist = cfg.get_with_default("nlist", 0); if (size <= TYPICAL_COUNT / 16384 + 1) { //handle less row count, avoid nlist set to 0 cfg["nlist"] = 1; - } else if (int(size / TYPICAL_COUNT) *nlist == 0) { + } else if (int(size / TYPICAL_COUNT) * nlist == 0) { //calculate a proper nlist if nlist not specified or size less than TYPICAL_COUNT cfg["nlist"] = int(size / TYPICAL_COUNT * 16384); } @@ -270,7 +284,8 @@ void AutoGenParams(const IndexType &type, const long &size, zilliz::knowhere::Co #define GPU_MAX_NRPOBE 1024 #endif -void ParameterValidation(const IndexType &type, Config &cfg) { +void +ParameterValidation(const IndexType &type, Config &cfg) { switch (type) { case IndexType::FAISS_IVFSQ8_GPU: case IndexType::FAISS_IVFFLAT_GPU: @@ -291,7 +306,8 @@ void ParameterValidation(const IndexType &type, Config &cfg) { } } -IndexType ConvertToCpuIndexType(const IndexType &type) { +IndexType +ConvertToCpuIndexType(const IndexType &type) { // TODO(linxj): add IDMAP switch (type) { case IndexType::FAISS_IVFFLAT_GPU: @@ -308,7 +324,8 @@ IndexType ConvertToCpuIndexType(const IndexType &type) { } } -IndexType ConvertToGpuIndexType(const IndexType &type) { +IndexType +ConvertToGpuIndexType(const IndexType &type) { switch (type) { case IndexType::FAISS_IVFFLAT_MIX: case IndexType::FAISS_IVFFLAT_CPU: { diff --git a/cpp/src/wrapper/vec_index.h b/cpp/src/wrapper/vec_index.h index 331fd2833b..3737270f42 100644 --- a/cpp/src/wrapper/vec_index.h +++ b/cpp/src/wrapper/vec_index.h @@ -21,8 +21,7 @@ #include #include -#include "utils/Error.h" - +#include "utils/Status.h" #include "knowhere/common/Config.h" #include "knowhere/common/BinarySet.h" @@ -50,62 +49,84 @@ enum class IndexType { }; class VecIndex; + using VecIndexPtr = std::shared_ptr; class VecIndex { public: - virtual ErrorCode BuildAll(const long &nb, - const float *xb, - const long *ids, - const Config &cfg, - const long &nt = 0, - const float *xt = nullptr) = 0; + virtual Status + BuildAll(const long &nb, + const float *xb, + const long *ids, + const Config &cfg, + const long &nt = 0, + const float *xt = nullptr) = 0; - virtual ErrorCode Add(const long &nb, - const float *xb, - const long *ids, - const Config &cfg = Config()) = 0; + virtual Status + Add(const long &nb, + const float *xb, + const long *ids, + const Config &cfg = Config()) = 0; - virtual ErrorCode Search(const long &nq, - const float *xq, - float *dist, - long *ids, - const Config &cfg = Config()) = 0; + virtual Status + Search(const long &nq, + const float *xq, + float *dist, + long *ids, + const Config &cfg = Config()) = 0; - virtual VecIndexPtr CopyToGpu(const int64_t &device_id, - const Config &cfg = Config()) = 0; + virtual VecIndexPtr + CopyToGpu(const int64_t &device_id, + const Config &cfg = Config()) = 0; - virtual VecIndexPtr CopyToCpu(const Config &cfg = Config()) = 0; + virtual VecIndexPtr + CopyToCpu(const Config &cfg = Config()) = 0; - virtual VecIndexPtr Clone() = 0; + virtual VecIndexPtr + Clone() = 0; - virtual int64_t GetDeviceId() = 0; + virtual int64_t + GetDeviceId() = 0; - virtual IndexType GetType() = 0; + virtual IndexType + GetType() = 0; - virtual int64_t Dimension() = 0; + virtual int64_t + Dimension() = 0; - virtual int64_t Count() = 0; + virtual int64_t + Count() = 0; - virtual zilliz::knowhere::BinarySet Serialize() = 0; + virtual zilliz::knowhere::BinarySet + Serialize() = 0; - virtual ErrorCode Load(const zilliz::knowhere::BinarySet &index_binary) = 0; + virtual Status + Load(const zilliz::knowhere::BinarySet &index_binary) = 0; }; -extern ErrorCode write_index(VecIndexPtr index, const std::string &location); +extern Status +write_index(VecIndexPtr index, const std::string &location); -extern VecIndexPtr read_index(const std::string &location); +extern VecIndexPtr +read_index(const std::string &location); -extern VecIndexPtr GetVecIndexFactory(const IndexType &type, const Config &cfg = Config()); +extern VecIndexPtr +GetVecIndexFactory(const IndexType &type, const Config &cfg = Config()); -extern VecIndexPtr LoadVecIndex(const IndexType &index_type, const zilliz::knowhere::BinarySet &index_binary); +extern VecIndexPtr +LoadVecIndex(const IndexType &index_type, const zilliz::knowhere::BinarySet &index_binary); -extern void AutoGenParams(const IndexType &type, const long &size, Config &cfg); +extern void +AutoGenParams(const IndexType &type, const long &size, Config &cfg); -extern void ParameterValidation(const IndexType &type, Config &cfg); +extern void +ParameterValidation(const IndexType &type, Config &cfg); -extern IndexType ConvertToCpuIndexType(const IndexType &type); -extern IndexType ConvertToGpuIndexType(const IndexType &type); +extern IndexType +ConvertToCpuIndexType(const IndexType &type); + +extern IndexType +ConvertToGpuIndexType(const IndexType &type); } } diff --git a/cpp/unittest/scheduler/scheduler_test.cpp b/cpp/unittest/scheduler/scheduler_test.cpp index 1cfbda1e14..ebe3a1f20d 100644 --- a/cpp/unittest/scheduler/scheduler_test.cpp +++ b/cpp/unittest/scheduler/scheduler_test.cpp @@ -17,8 +17,7 @@ #include "scheduler/Scheduler.h" #include -#include -#include +#include "src/scheduler/tasklabel/DefaultLabel.h" #include "cache/DataObj.h" #include "cache/GpuCacheMgr.h" #include "scheduler/task/TestTask.h" @@ -35,13 +34,12 @@ namespace engine { class MockVecIndex : public engine::VecIndex { public: - virtual ErrorCode BuildAll(const long &nb, + virtual Status BuildAll(const long &nb, const float *xb, const long *ids, const engine::Config &cfg, const long &nt = 0, const float *xt = nullptr) { - } engine::VecIndexPtr Clone() override { @@ -56,14 +54,14 @@ public: return engine::IndexType::INVALID; } - virtual ErrorCode Add(const long &nb, + virtual Status Add(const long &nb, const float *xb, const long *ids, const engine::Config &cfg = engine::Config()) { } - virtual ErrorCode Search(const long &nq, + virtual Status Search(const long &nq, const float *xq, float *dist, long *ids, @@ -92,7 +90,7 @@ public: return binset; } - virtual ErrorCode Load(const zilliz::knowhere::BinarySet &index_binary) { + virtual Status Load(const zilliz::knowhere::BinarySet &index_binary) { } diff --git a/cpp/unittest/server/cache_test.cpp b/cpp/unittest/server/cache_test.cpp index bf9b6edf7d..593911212d 100644 --- a/cpp/unittest/server/cache_test.cpp +++ b/cpp/unittest/server/cache_test.cpp @@ -19,7 +19,6 @@ #include "cache/CpuCacheMgr.h" #include "cache/GpuCacheMgr.h" #include "server/ServerConfig.h" - #include "utils/Error.h" #include "src/wrapper/vec_index.h" @@ -48,13 +47,13 @@ public: } - virtual ErrorCode BuildAll(const long &nb, + virtual Status BuildAll(const long &nb, const float *xb, const long *ids, const engine::Config &cfg, const long &nt = 0, const float *xt = nullptr) { - return 0; + return Status(); } engine::VecIndexPtr Clone() override { @@ -69,19 +68,19 @@ public: return engine::IndexType::INVALID; } - virtual ErrorCode Add(const long &nb, + virtual Status Add(const long &nb, const float *xb, const long *ids, const engine::Config &cfg = engine::Config()) { - return 0; + return Status(); } - virtual ErrorCode Search(const long &nq, + virtual Status Search(const long &nq, const float *xq, float *dist, long *ids, const engine::Config &cfg = engine::Config()) { - return 0; + return Status(); } engine::VecIndexPtr CopyToGpu(const int64_t &device_id, @@ -106,8 +105,8 @@ public: return binset; } - virtual ErrorCode Load(const zilliz::knowhere::BinarySet &index_binary) { - return 0; + virtual Status Load(const zilliz::knowhere::BinarySet &index_binary) { + return Status(); } public: diff --git a/cpp/unittest/wrapper/CMakeLists.txt b/cpp/unittest/wrapper/CMakeLists.txt index c461bc7850..d0eb3bfcfb 100644 --- a/cpp/unittest/wrapper/CMakeLists.txt +++ b/cpp/unittest/wrapper/CMakeLists.txt @@ -28,6 +28,7 @@ set(wrapper_files set(util_files utils.cpp ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc + ${MILVUS_ENGINE_SRC}/utils/Status.cpp ) set(knowhere_libs From 4c944e1b72678e5a0746d4412529271ba092cbab Mon Sep 17 00:00:00 2001 From: test Date: Fri, 20 Sep 2019 08:59:05 +0000 Subject: [PATCH 065/217] MS-575 Add Clang-format & Clang-tidy & Cpplint Former-commit-id: f72da7918b3696f906d0f3a2ef8590b6bd16d3d2 --- cpp/CMakeLists.txt | 41 +++++++++ cpp/build-support/run_clang_tidy.py | 126 ++++++++++++++++++++++++++++ cpp/build.sh | 1 + 3 files changed, 168 insertions(+) create mode 100755 cpp/build-support/run_clang_tidy.py diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d9267be7c4..26acbd2eee 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -45,6 +45,13 @@ string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}") set(CLANG_FORMAT_VERSION "6.0") find_package(ClangTools) +if("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND) + # Generate a Clang compile_commands.json "compilation database" file for use + # with various development tools, such as Vim's YouCompleteMe plugin. + # See http://clang.llvm.org/docs/JSONCompilationDatabase.html + set(CMAKE_EXPORT_COMPILE_COMMANDS 1) +endif() + set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support") if(CMAKE_BUILD_TYPE STREQUAL "Release") @@ -203,3 +210,37 @@ if(${CLANG_FORMAT_FOUND}) ${MILVUS_LINT_QUIET}) endif() +# +# "make clang-tidy" and "make check-clang-tidy" targets +# +if(${CLANG_TIDY_FOUND}) + # runs clang-tidy and attempts to fix any warning automatically + add_custom_target(clang-tidy + ${PYTHON_EXECUTABLE} + ${BUILD_SUPPORT_DIR}/run_clang_tidy.py + --clang_tidy_binary + ${CLANG_TIDY_BIN} + --exclude_globs + ${LINT_EXCLUSIONS_FILE} + --compile_commands + ${CMAKE_BINARY_DIR}/compile_commands.json + --source_dir + ${CMAKE_CURRENT_SOURCE_DIR}/src + --fix + ${MILVUS_LINT_QUIET}) + + # runs clang-tidy and exits with a non-zero exit code if any errors are found. + add_custom_target(check-clang-tidy + ${PYTHON_EXECUTABLE} + ${BUILD_SUPPORT_DIR}/run_clang_tidy.py + --clang_tidy_binary + ${CLANG_TIDY_BIN} + --exclude_globs + ${LINT_EXCLUSIONS_FILE} + --compile_commands + ${CMAKE_BINARY_DIR}/compile_commands.json + --source_dir + ${CMAKE_CURRENT_SOURCE_DIR}/src + ${MILVUS_LINT_QUIET}) +endif() + diff --git a/cpp/build-support/run_clang_tidy.py b/cpp/build-support/run_clang_tidy.py new file mode 100755 index 0000000000..6df17d5379 --- /dev/null +++ b/cpp/build-support/run_clang_tidy.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import print_function +import argparse +import multiprocessing as mp +import lintutils +from subprocess import PIPE +import sys +from functools import partial + + +def _get_chunk_key(filenames): + # lists are not hashable so key on the first filename in a chunk + return filenames[0] + + +# clang-tidy outputs complaints in '/path:line_number: complaint' format, +# so we can scan its output to get a list of files to fix +def _check_some_files(completed_processes, filenames): + result = completed_processes[_get_chunk_key(filenames)] + return lintutils.stdout_pathcolonline(result, filenames) + + +def _check_all(cmd, filenames): + # each clang-tidy instance will process 16 files + chunks = lintutils.chunk(filenames, 16) + cmds = [cmd + some for some in chunks] + results = lintutils.run_parallel(cmds, stderr=PIPE, stdout=PIPE) + error = False + # record completed processes (keyed by the first filename in the input + # chunk) for lookup in _check_some_files + completed_processes = { + _get_chunk_key(some): result + for some, result in zip(chunks, results) + } + checker = partial(_check_some_files, completed_processes) + pool = mp.Pool() + try: + # check output of completed clang-tidy invocations in parallel + for problem_files, stdout in pool.imap(checker, chunks): + if problem_files: + msg = "clang-tidy suggested fixes for {}" + print("\n".join(map(msg.format, problem_files))) + print(stdout) + error = True + except Exception: + error = True + raise + finally: + pool.terminate() + pool.join() + + if error: + sys.exit(1) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Runs clang-tidy on all ") + parser.add_argument("--clang_tidy_binary", + required=True, + help="Path to the clang-tidy binary") + parser.add_argument("--exclude_globs", + help="Filename containing globs for files " + "that should be excluded from the checks") + parser.add_argument("--compile_commands", + required=True, + help="compile_commands.json to pass clang-tidy") + parser.add_argument("--source_dir", + required=True, + help="Root directory of the source code") + parser.add_argument("--fix", default=False, + action="store_true", + help="If specified, will attempt to fix the " + "source code instead of recommending fixes, " + "defaults to %(default)s") + parser.add_argument("--quiet", default=False, + action="store_true", + help="If specified, only print errors") + arguments = parser.parse_args() + + exclude_globs = [] + if arguments.exclude_globs: + for line in open(arguments.exclude_globs): + exclude_globs.append(line.strip()) + + linted_filenames = [] + for path in lintutils.get_sources(arguments.source_dir, exclude_globs): + linted_filenames.append(path) + + if not arguments.quiet: + msg = 'Tidying {}' if arguments.fix else 'Checking {}' + print("\n".join(map(msg.format, linted_filenames))) + + cmd = [ + arguments.clang_tidy_binary, + '-p', + arguments.compile_commands + ] + if arguments.fix: + cmd.append('-fix') + results = lintutils.run_parallel( + [cmd + some for some in lintutils.chunk(linted_filenames, 16)]) + for returncode, stdout, stderr in results: + if returncode != 0: + sys.exit(returncode) + + else: + _check_all(cmd, linted_filenames) + diff --git a/cpp/build.sh b/cpp/build.sh index 86af2d42e0..ab6ba9b17e 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -89,6 +89,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then -DMILVUS_ENABLE_PROFILING=${PROFILING} \ -DBUILD_FAISS_WITH_MKL=${BUILD_FAISS_WITH_MKL} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ../" echo ${CMAKE_CMD} From 62b993295e8272bb1f34516e29fdf6fbbcd78779 Mon Sep 17 00:00:00 2001 From: test Date: Fri, 20 Sep 2019 09:00:15 +0000 Subject: [PATCH 066/217] MS-575 Add Clang-format & Clang-tidy & Cpplint Former-commit-id: 0e94a8add6753a158ce59912826a553a32fa8566 --- cpp/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/build.sh b/cpp/build.sh index ab6ba9b17e..b09e5b6313 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -89,7 +89,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then -DMILVUS_ENABLE_PROFILING=${PROFILING} \ -DBUILD_FAISS_WITH_MKL=${BUILD_FAISS_WITH_MKL} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ ../" echo ${CMAKE_CMD} From 82e37e11ba02640dbb6ae26cbd2dbb6bdf6370b0 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Fri, 20 Sep 2019 17:08:26 +0800 Subject: [PATCH 067/217] MS-574 uniform config name format, add config default value Former-commit-id: 2d11027d4b70632e22b3100fb2bde91c86972969 --- cpp/conf/server_config.template | 43 ++++++----- cpp/src/cache/CpuCacheMgr.cpp | 6 +- cpp/src/cache/GpuCacheMgr.cpp | 6 +- cpp/src/db/engine/ExecutionEngineImpl.cpp | 2 +- cpp/src/main.cpp | 2 +- cpp/src/metrics/Metrics.cpp | 3 +- cpp/src/metrics/PrometheusMetrics.cpp | 16 +++-- cpp/src/scheduler/SchedInst.cpp | 6 +- cpp/src/server/DBWrapper.cpp | 20 ++++-- cpp/src/server/Server.cpp | 11 +-- cpp/src/server/Server.h | 6 +- cpp/src/server/ServerConfig.cpp | 88 +++++++++++++---------- cpp/src/server/ServerConfig.h | 73 +++++++++++-------- cpp/src/server/grpc_impl/GrpcServer.cpp | 4 +- cpp/src/utils/SignalUtil.cpp | 4 +- cpp/src/wrapper/KnowhereResource.cpp | 7 +- cpp/unittest/server/rpc_test.cpp | 4 +- 17 files changed, 167 insertions(+), 134 deletions(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 8253788cb7..a96d2d81cf 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -1,42 +1,39 @@ server_config: - address: 0.0.0.0 # milvus server ip address (IPv4) - port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 - mode: single # milvus deployment type: single, cluster, read_only - time_zone: UTC+8 # Use the UTC-x or UTC+x to specify a time zone. eg. UTC+8 for China Standard Time + address: 0.0.0.0 # milvus server ip address (IPv4) + port: 19530 # port range: 1025 ~ 65534 + mode: single # deployment type: single, cluster, read_only + time_zone: UTC+8 db_config: - db_path: @MILVUS_DB_PATH@ # milvus data storage path - db_slave_path: # secondry data storage path, split by semicolon + db_path: @MILVUS_DB_PATH@ # milvus database path + db_slave_path: # secondary database path, split by semicolon # URI format: dialect://username:password@host:port/database # All parts except dialect are optional, but you MUST include the delimiters # Currently dialect supports mysql or sqlite db_backend_url: sqlite://:@:/ - archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB - archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day - insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB. - # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB - build_index_gpu: 0 # which gpu is used to build index, default: 0, range: 0 ~ gpu number - 1 + archive_disk_threshold: 0 # GB, file will be archived when disk usage exceed, 0 for no limit + archive_days_threshold: 0 # DAYS, older files will be archived, 0 for no limit + buffer_size: 4 # GB, maximum insert buffer size allowed + build_index_gpu: 0 # gpu id used for building index metric_config: - is_startup: off # if monitoring start: on, off - collector: prometheus # metrics collector: prometheus - prometheus_config: # following are prometheus configure - port: 8080 # the port prometheus use to fetch metrics - push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address - push_gateway_port: 9091 # push method configure: push gateway port + auto_bootup: off # whether enable monitoring when bootup + collector: prometheus # prometheus + prometheus_config: + port: 8080 # port prometheus used to fetch metrics cache_config: - cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory - cpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 - insert_cache_immediately: false # insert data will be load into cache immediately for hot query + cpu_mem_capacity: 16 # GB, CPU memory size used for cache + cpu_mem_threshold: 0.85 # percent of data kept when cache cleanup triggered + insert_immediately: false # whether load data into cache when insert engine_config: - use_blas_threshold: 20 + blas_threshold: 20 resource_config: mode: simple - resources: -# - cpu + pool: + - cpu - gpu0 diff --git a/cpp/src/cache/CpuCacheMgr.cpp b/cpp/src/cache/CpuCacheMgr.cpp index e47d768f34..ee30494192 100644 --- a/cpp/src/cache/CpuCacheMgr.cpp +++ b/cpp/src/cache/CpuCacheMgr.cpp @@ -30,11 +30,13 @@ namespace { CpuCacheMgr::CpuCacheMgr() { server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); - int64_t cap = config.GetInt64Value(server::CONFIG_CPU_CACHE_CAPACITY, 16); + int64_t cap = + config.GetInt64Value(server::CONFIG_CACHE_CPU_MEM_CAPACITY, std::stoi(server::CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT)); cap *= unit; cache_ = std::make_shared>(cap, 1UL<<32); - double free_percent = config.GetDoubleValue(server::CACHE_FREE_PERCENT, 0.85); + double free_percent = + config.GetDoubleValue(server::CONFIG_CACHE_CPU_MEM_THRESHOLD, std::stof(server::CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT)); if(free_percent > 0.0 && free_percent <= 1.0) { cache_->set_freemem_percent(free_percent); } else { diff --git a/cpp/src/cache/GpuCacheMgr.cpp b/cpp/src/cache/GpuCacheMgr.cpp index bd6f2adfe1..58c7db3fd6 100644 --- a/cpp/src/cache/GpuCacheMgr.cpp +++ b/cpp/src/cache/GpuCacheMgr.cpp @@ -35,11 +35,13 @@ namespace { GpuCacheMgr::GpuCacheMgr() { server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); - int64_t cap = config.GetInt64Value(server::CONFIG_GPU_CACHE_CAPACITY, 0); + int64_t cap = + config.GetInt64Value(server::CONFIG_CACHE_GPU_MEM_CAPACITY, std::stoi(server::CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT)); cap *= G_BYTE; cache_ = std::make_shared>(cap, 1UL<<32); - double free_percent = config.GetDoubleValue(server::GPU_CACHE_FREE_PERCENT, 0.85); + double free_percent = + config.GetDoubleValue(server::CONFIG_CACHE_GPU_MEM_THRESHOLD, std::stof(server::CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT)); if (free_percent > 0.0 && free_percent <= 1.0) { cache_->set_freemem_percent(free_percent); } else { diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 7b9bac2e3a..65483690d6 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -336,7 +336,7 @@ Status ExecutionEngineImpl::Init() { using namespace zilliz::milvus::server; ServerConfig &config = ServerConfig::GetInstance(); ConfigNode server_config = config.GetConfig(CONFIG_DB); - gpu_num_ = server_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, 0); + gpu_num_ = server_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); return Status::OK(); } diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index 5a61c5bfb4..dc17733a8b 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -110,7 +110,7 @@ main(int argc, char *argv[]) { signal(SIGUSR2, server::SignalUtil::HandleSignal); signal(SIGTERM, server::SignalUtil::HandleSignal); - server::Server &server = server::Server::Instance(); + server::Server &server = server::Server::GetInstance(); server.Init(start_daemonized, pid_filename, config_filename, log_config_file); server.Start(); diff --git a/cpp/src/metrics/Metrics.cpp b/cpp/src/metrics/Metrics.cpp index 10ddd7ee3c..ba51d7fd94 100644 --- a/cpp/src/metrics/Metrics.cpp +++ b/cpp/src/metrics/Metrics.cpp @@ -32,7 +32,8 @@ Metrics::GetInstance() { MetricsBase & Metrics::CreateMetricsCollector() { ConfigNode &config = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC); - std::string collector_type_str = config.GetValue(CONFIG_METRIC_COLLECTOR); + std::string collector_type_str = + config.GetValue(CONFIG_METRIC_COLLECTOR, CONFIG_METRIC_COLLECTOR_DEFAULT); if (collector_type_str == "prometheus") { return PrometheusMetrics::GetInstance(); diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index f4d70a21c3..cc8aca204f 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -26,15 +26,19 @@ namespace zilliz { namespace milvus { namespace server { - ErrorCode +ErrorCode PrometheusMetrics::Init() { try { - ConfigNode &configNode = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC); - startup_ = configNode.GetValue(CONFIG_METRIC_IS_STARTUP) == "on"; - if(!startup_) return SERVER_SUCCESS; + ConfigNode &metric_config = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC); + startup_ = + metric_config.GetBoolValue(CONFIG_METRIC_AUTO_BOOTUP, std::stoi(CONFIG_METRIC_AUTO_BOOTUP_DEFAULT)); + if (!startup_) return SERVER_SUCCESS; + // Following should be read from config file. - const std::string bind_address = configNode.GetChild(CONFIG_PROMETHEUS).GetValue(CONFIG_METRIC_PROMETHEUS_PORT); - const std::string uri = std::string("/metrics"); + ConfigNode &prometheus_config = metric_config.GetChild(CONFIG_METRIC_PROMETHEUS); + const std::string bind_address = + prometheus_config.GetValue(CONFIG_METRIC_PROMETHEUS_PORT, CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); + const std::string uri = std::string("/tmp/metrics"); const std::size_t num_threads = 2; // Init Exposer diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index 3b14158e58..0b1e7cd302 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -39,12 +39,12 @@ std::mutex JobMgrInst::mutex_; void load_simple_config() { server::ConfigNode &config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_RESOURCE); - auto mode = config.GetValue("mode", "simple"); + auto mode = config.GetValue(server::CONFIG_RESOURCE_MODE, server::CONFIG_RESOURCE_MODE_DEFAULT); - auto resources = config.GetSequence("resources"); + auto pool = config.GetSequence(server::CONFIG_RESOURCE_POOL); bool cpu = false; std::set gpu_ids; - for (auto &resource : resources) { + for (auto &resource : pool) { if (resource == "cpu") { cpu = true; break; diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 1409fff597..756438dfb3 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -38,7 +38,7 @@ Status DBWrapper::StartService() { //db config engine::DBOptions opt; ConfigNode& db_config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); - opt.meta.backend_uri = db_config.GetValue(CONFIG_DB_URL); + opt.meta.backend_uri = db_config.GetValue(CONFIG_DB_BACKEND_URL); std::string db_path = db_config.GetValue(CONFIG_DB_PATH); opt.meta.path = db_path + "/db"; @@ -47,10 +47,11 @@ Status DBWrapper::StartService() { // cache config ConfigNode& cache_config = ServerConfig::GetInstance().GetConfig(CONFIG_CACHE); - opt.insert_cache_immediately_ = cache_config.GetBoolValue(CONFIG_INSERT_CACHE_IMMEDIATELY, false); + opt.insert_cache_immediately_ = + cache_config.GetBoolValue(CONFIG_CACHE_INSERT_IMMEDIATELY, std::stoi(CONFIG_CACHE_INSERT_IMMEDIATELY_DEFAULT)); ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); - std::string mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); + std::string mode = serverConfig.GetValue(CONFIG_SERVER_MODE, CONFIG_SERVER_MODE_DEFAULT); if (mode == "single") { opt.mode = engine::DBOptions::MODE::SINGLE; } @@ -67,7 +68,8 @@ Status DBWrapper::StartService() { // engine config ConfigNode& engine_config = ServerConfig::GetInstance().GetConfig(CONFIG_ENGINE); - int32_t omp_thread = engine_config.GetInt32Value(CONFIG_OMP_THREAD_NUM, 0); + int32_t omp_thread = + engine_config.GetInt32Value(CONFIG_ENGINE_OMP_THREAD_NUM, std::stoi(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT)); if(omp_thread > 0) { omp_set_num_threads(omp_thread); SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; @@ -79,12 +81,16 @@ Status DBWrapper::StartService() { } } - faiss::distance_compute_blas_threshold = engine_config.GetInt32Value(CONFIG_DCBT, 20);//init faiss global variable + //init faiss global variable + faiss::distance_compute_blas_threshold = + engine_config.GetInt32Value(CONFIG_ENGINE_BLAS_THRESHOLD, std::stoi(CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT)); //set archive config engine::ArchiveConf::CriteriaT criterial; - int64_t disk = db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK, 0); - int64_t days = db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DAYS, 0); + int64_t disk = + db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, std::stoi(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT)); + int64_t days = + db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, std::stoi(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT)); if(disk > 0) { criterial[engine::ARCHIVE_CONF_DISK] = disk; } diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 2d0a4e5c99..7f85ed6509 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -43,18 +43,11 @@ namespace milvus { namespace server { Server & -Server::Instance() { +Server::GetInstance() { static Server server; return server; } -Server::Server() { - -} -Server::~Server() { - -} - void Server::Init(int64_t daemonized, const std::string &pid_filename, @@ -171,7 +164,7 @@ Server::Start() { ServerConfig &config = ServerConfig::GetInstance(); ConfigNode server_config = config.GetConfig(CONFIG_SERVER); - std::string time_zone = server_config.GetValue(CONFIG_TIME_ZONE, "UTC+8"); + std::string time_zone = server_config.GetValue(CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); if (time_zone.length() == 3) { time_zone = "CUT"; } else { diff --git a/cpp/src/server/Server.h b/cpp/src/server/Server.h index 30ea95a7bb..cb323434fe 100644 --- a/cpp/src/server/Server.h +++ b/cpp/src/server/Server.h @@ -29,7 +29,7 @@ namespace server { class Server { public: - static Server &Instance(); + static Server &GetInstance(); void Init(int64_t daemonized, const std::string &pid_filename, @@ -40,8 +40,8 @@ class Server { void Stop(); private: - Server(); - ~Server(); + Server() = default; + ~Server() = default; void Daemonize(); diff --git a/cpp/src/server/ServerConfig.cpp b/cpp/src/server/ServerConfig.cpp index 687490cf5e..0f51ecf6e2 100644 --- a/cpp/src/server/ServerConfig.cpp +++ b/cpp/src/server/ServerConfig.cpp @@ -109,50 +109,51 @@ ServerConfig::CheckServerConfig() { bool okay = true; ConfigNode server_config = GetConfig(CONFIG_SERVER); - std::string ip_address = server_config.GetValue(CONFIG_SERVER_ADDRESS, "127.0.0.1"); + std::string ip_address = server_config.GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); if (!ValidationUtil::ValidateIpAddress(ip_address).ok()) { std::cerr << "ERROR: invalid server IP address: " << ip_address << std::endl; okay = false; } - std::string port_str = server_config.GetValue(CONFIG_SERVER_PORT, "19530"); + std::string port_str = server_config.GetValue(CONFIG_SERVER_PORT, CONFIG_SERVER_PORT_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(port_str).ok()) { std::cerr << "ERROR: port " << port_str << " is not a number" << std::endl; okay = false; } else { int32_t port = std::stol(port_str); - if (port < 1025 | port > 65534) { - std::cerr << "ERROR: port " << port_str << " out of range [1025, 65534]" << std::endl; + if (!(port > 1024 && port < 65535)) { + std::cerr << "ERROR: port " << port_str << " out of range (1024, 65535)" << std::endl; okay = false; } } - std::string mode = server_config.GetValue(CONFIG_CLUSTER_MODE, "single"); + std::string mode = server_config.GetValue(CONFIG_SERVER_MODE, CONFIG_SERVER_MODE_DEFAULT); if (mode != "single" && mode != "cluster" && mode != "read_only") { std::cerr << "ERROR: mode " << mode << " is not one of ['single', 'cluster', 'read_only']" << std::endl; okay = false; } - std::string time_zone = server_config.GetValue(CONFIG_TIME_ZONE, "UTC+8"); + std::string time_zone = server_config.GetValue(CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); int flag = 0; - if(time_zone.length() < 3) + if (time_zone.length() <= 3) { flag = 1; - else if(time_zone.substr(0, 3) != "UTC") - flag = 1; - else if(time_zone.length() > 3){ - try { - stoi(time_zone.substr(3, std::string::npos)); - } - catch (std::invalid_argument &) { + } else { + if (time_zone.substr(0, 3) != "UTC") { flag = 1; + } else { + try { + stoi(time_zone.substr(3)); + } catch (...) { + flag = 1; + } } } - if(flag == 1){ - std::cerr << "ERROR: time_zone " << time_zone << " is not in a right format" << std::endl; + if (flag == 1) { + std::cerr << "ERROR: time_zone " << time_zone << " format wrong" << std::endl; okay = false; } - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Server config is illegal")); + return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Illegal server config")); } Status @@ -182,25 +183,25 @@ ServerConfig::CheckDBConfig() { okay = false; } - std::string db_backend_url = db_config.GetValue(CONFIG_DB_URL); + std::string db_backend_url = db_config.GetValue(CONFIG_DB_BACKEND_URL); if (!ValidationUtil::ValidateDbURI(db_backend_url).ok()) { std::cerr << "ERROR: invalid db_backend_url: " << db_backend_url << std::endl; okay = false; } - std::string archive_disk_threshold_str = db_config.GetValue(CONFIG_DB_INSERT_BUFFER_SIZE, "0"); + std::string archive_disk_threshold_str = db_config.GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(archive_disk_threshold_str).ok()) { std::cerr << "ERROR: archive_disk_threshold " << archive_disk_threshold_str << " is not a number" << std::endl; okay = false; } - std::string archive_days_threshold_str = db_config.GetValue(CONFIG_DB_INSERT_BUFFER_SIZE, "0"); + std::string archive_days_threshold_str = db_config.GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(archive_days_threshold_str).ok()) { std::cerr << "ERROR: archive_days_threshold " << archive_days_threshold_str << " is not a number" << std::endl; okay = false; } - std::string insert_buffer_size_str = db_config.GetValue(CONFIG_DB_INSERT_BUFFER_SIZE, "4"); + std::string insert_buffer_size_str = db_config.GetValue(CONFIG_DB_BUFFER_SIZE, CONFIG_DB_BUFFER_SIZE_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(insert_buffer_size_str).ok()) { std::cerr << "ERROR: insert_buffer_size " << insert_buffer_size_str << " is not a number" << std::endl; okay = false; @@ -216,7 +217,7 @@ ServerConfig::CheckDBConfig() { } } - std::string gpu_index_str = db_config.GetValue(CONFIG_DB_BUILD_INDEX_GPU, "0"); + std::string gpu_index_str = db_config.GetValue(CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(gpu_index_str).ok()) { std::cerr << "ERROR: gpu_index " << gpu_index_str << " is not a number" << std::endl; okay = false; @@ -245,13 +246,14 @@ ServerConfig::CheckMetricConfig() { bool okay = true; ConfigNode metric_config = GetConfig(CONFIG_METRIC); - std::string is_startup_str = metric_config.GetValue(CONFIG_METRIC_IS_STARTUP, "off"); + std::string is_startup_str = + metric_config.GetValue(CONFIG_METRIC_AUTO_BOOTUP, CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); if (!ValidationUtil::ValidateStringIsBool(is_startup_str).ok()) { std::cerr << "ERROR: invalid is_startup config: " << is_startup_str << std::endl; okay = false; } - std::string port_str = metric_config.GetChild(CONFIG_PROMETHEUS).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, "8080"); + std::string port_str = metric_config.GetChild(CONFIG_METRIC_PROMETHEUS).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, "8080"); if (!ValidationUtil::ValidateStringIsNumber(port_str).ok()) { std::cerr << "ERROR: port specified in prometheus_config " << port_str << " is not a number" << std::endl; okay = false; @@ -274,7 +276,8 @@ ServerConfig::CheckCacheConfig() { bool okay = true; ConfigNode cache_config = GetConfig(CONFIG_CACHE); - std::string cpu_cache_capacity_str = cache_config.GetValue(CONFIG_CPU_CACHE_CAPACITY, "16"); + std::string cpu_cache_capacity_str = + cache_config.GetValue(CONFIG_CACHE_CPU_MEM_CAPACITY, CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(cpu_cache_capacity_str).ok()) { std::cerr << "ERROR: cpu_cache_capacity " << cpu_cache_capacity_str << " is not a number" << std::endl; okay = false; @@ -292,15 +295,17 @@ ServerConfig::CheckCacheConfig() { std::cerr << "Warning: cpu_cache_capacity value is too aggressive" << std::endl; } - uint64_t insert_buffer_size = (uint64_t) GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_INSERT_BUFFER_SIZE, 4); - insert_buffer_size *= GB; - if (insert_buffer_size + cpu_cache_capacity >= total_mem) { + uint64_t buffer_size = + (uint64_t) GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_BUFFER_SIZE, std::stoi(CONFIG_DB_BUFFER_SIZE_DEFAULT)); + buffer_size *= GB; + if (buffer_size + cpu_cache_capacity >= total_mem) { std::cerr << "ERROR: sum of cpu_cache_capacity and insert_buffer_size exceed system memory" << std::endl; okay = false; } } - std::string cpu_cache_free_percent_str = cache_config.GetValue(CACHE_FREE_PERCENT, "0.85"); + std::string cpu_cache_free_percent_str = + cache_config.GetValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT); double cpu_cache_free_percent; if (!ValidationUtil::ValidateStringIsDouble(cpu_cache_free_percent_str, cpu_cache_free_percent).ok()) { std::cerr << "ERROR: cpu_cache_free_percent " << cpu_cache_free_percent_str << " is not a double" << std::endl; @@ -311,13 +316,15 @@ ServerConfig::CheckCacheConfig() { okay = false; } - std::string insert_cache_immediately_str = cache_config.GetValue(CONFIG_INSERT_CACHE_IMMEDIATELY, "false"); + std::string insert_cache_immediately_str = + cache_config.GetValue(CONFIG_CACHE_INSERT_IMMEDIATELY, CONFIG_CACHE_INSERT_IMMEDIATELY_DEFAULT); if (!ValidationUtil::ValidateStringIsBool(insert_cache_immediately_str).ok()) { std::cerr << "ERROR: invalid insert_cache_immediately config: " << insert_cache_immediately_str << std::endl; okay = false; } - std::string gpu_cache_capacity_str = cache_config.GetValue(CONFIG_GPU_CACHE_CAPACITY, "0"); + std::string gpu_cache_capacity_str = + cache_config.GetValue(CONFIG_CACHE_GPU_MEM_CAPACITY, CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(gpu_cache_capacity_str).ok()) { std::cerr << "ERROR: gpu_cache_capacity " << gpu_cache_capacity_str << " is not a number" << std::endl; okay = false; @@ -325,7 +332,7 @@ ServerConfig::CheckCacheConfig() { else { uint64_t gpu_cache_capacity = (uint64_t) std::stol(gpu_cache_capacity_str); gpu_cache_capacity *= GB; - int gpu_index = GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, 0); + int gpu_index = GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); size_t gpu_memory; if (!ValidationUtil::GetGpuMemory(gpu_index, gpu_memory).ok()) { std::cerr << "ERROR: could not get gpu memory for device " << gpu_index << std::endl; @@ -341,7 +348,8 @@ ServerConfig::CheckCacheConfig() { } } - std::string gpu_cache_free_percent_str = cache_config.GetValue(GPU_CACHE_FREE_PERCENT, "0.85"); + std::string gpu_cache_free_percent_str = + cache_config.GetValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT); double gpu_cache_free_percent; if (!ValidationUtil::ValidateStringIsDouble(gpu_cache_free_percent_str, gpu_cache_free_percent).ok()) { std::cerr << "ERROR: gpu_cache_free_percent " << gpu_cache_free_percent_str << " is not a double" << std::endl; @@ -365,13 +373,15 @@ ServerConfig::CheckEngineConfig() { bool okay = true; ConfigNode engine_config = GetConfig(CONFIG_ENGINE); - std::string use_blas_threshold_str = engine_config.GetValue(CONFIG_DCBT, "20"); + std::string use_blas_threshold_str = + engine_config.GetValue(CONFIG_ENGINE_BLAS_THRESHOLD, CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(use_blas_threshold_str).ok()) { std::cerr << "ERROR: use_blas_threshold " << use_blas_threshold_str << " is not a number" << std::endl; okay = false; } - std::string omp_thread_num_str = engine_config.GetValue(CONFIG_OMP_THREAD_NUM, "0"); + std::string omp_thread_num_str = + engine_config.GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(omp_thread_num_str).ok()) { std::cerr << "ERROR: omp_thread_num " << omp_thread_num_str << " is not a number" << std::endl; okay = false; @@ -393,20 +403,20 @@ ServerConfig::CheckResourceConfig() { /* resource_config: mode: simple - resources: + pool: - cpu - gpu0 - gpu100 */ bool okay = true; server::ConfigNode &config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_RESOURCE); - auto mode = config.GetValue("mode", "simple"); + auto mode = config.GetValue(CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); if (mode != "simple") { std::cerr << "ERROR: invalid resource config: mode is " << mode << std::endl; okay = false; } - auto resources = config.GetSequence("resources"); - if (resources.empty()) { + auto pool = config.GetSequence(CONFIG_RESOURCE_POOL); + if (pool.empty()) { std::cerr << "ERROR: invalid resource config: resources empty" << std::endl; okay = false; } diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index d0f3e30d6b..e515b74645 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -26,52 +26,69 @@ namespace zilliz { namespace milvus { namespace server { +/* server config */ static const char* CONFIG_SERVER = "server_config"; static const char* CONFIG_SERVER_ADDRESS = "address"; +static const char* CONFIG_SERVER_ADDRESS_DEFAULT = "127.0.0.1"; static const char* CONFIG_SERVER_PORT = "port"; -static const char* CONFIG_CLUSTER_MODE = "mode"; -static const char* CONFIG_TIME_ZONE = "time_zone"; +static const char* CONFIG_SERVER_PORT_DEFAULT = "19530"; +static const char* CONFIG_SERVER_MODE = "mode"; +static const char* CONFIG_SERVER_MODE_DEFAULT = "single"; +static const char* CONFIG_SERVER_TIME_ZONE = "time_zone"; +static const char* CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; +/* db config */ static const char* CONFIG_DB = "db_config"; -static const char* CONFIG_DB_URL = "db_backend_url"; static const char* CONFIG_DB_PATH = "db_path"; +static const char* CONFIG_DB_PATH_DEFAULT = "/tmp/milvus"; static const char* CONFIG_DB_SLAVE_PATH = "db_slave_path"; -static const char* CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold"; -static const char* CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold"; -static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; -static const char* CONFIG_DB_PARALLEL_REDUCE = "parallel_reduce"; +static const char* CONFIG_DB_SLAVE_PATH_DEFAULT = ""; +static const char* CONFIG_DB_BACKEND_URL = "db_backend_url"; +static const char* CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/"; +static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold"; +static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0"; +static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold"; +static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; +static const char* CONFIG_DB_BUFFER_SIZE = "buffer_size"; +static const char* CONFIG_DB_BUFFER_SIZE_DEFAULT = "4"; static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu"; +static const char* CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0"; -static const char* CONFIG_LOG = "log_config"; - +/* cache config */ static const char* CONFIG_CACHE = "cache_config"; -static const char* CONFIG_CPU_CACHE_CAPACITY = "cpu_cache_capacity"; -static const char* CONFIG_GPU_CACHE_CAPACITY = "gpu_cache_capacity"; -static const char* CACHE_FREE_PERCENT = "cpu_cache_free_percent"; -static const char* CONFIG_INSERT_CACHE_IMMEDIATELY = "insert_cache_immediately"; -static const char *GPU_CACHE_FREE_PERCENT = "gpu_cache_free_percent"; +static const char* CONFIG_CACHE_CPU_MEM_CAPACITY = "cpu_mem_capacity"; +static const char* CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT = "16"; +static const char* CONFIG_CACHE_GPU_MEM_CAPACITY = "gpu_mem_capacity"; +static const char* CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT = "0"; +static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD = "cpu_mem_threshold"; +static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT = "0.85"; +static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD = "gpu_mem_threshold"; +static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT = "0.85"; +static const char* CONFIG_CACHE_INSERT_IMMEDIATELY = "insert_immediately"; +static const char* CONFIG_CACHE_INSERT_IMMEDIATELY_DEFAULT = "0"; +/* metric config */ static const char* CONFIG_METRIC = "metric_config"; -static const char* CONFIG_METRIC_IS_STARTUP = "is_startup"; +static const char* CONFIG_METRIC_AUTO_BOOTUP = "auto_bootup"; +static const char* CONFIG_METRIC_AUTO_BOOTUP_DEFAULT = "0"; static const char* CONFIG_METRIC_COLLECTOR = "collector"; -static const char* CONFIG_PROMETHEUS = "prometheus_config"; +static const char* CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus"; +static const char* CONFIG_METRIC_PROMETHEUS = "prometheus_config"; static const char* CONFIG_METRIC_PROMETHEUS_PORT = "port"; +static const char* CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT = "8080"; +/* engine config */ static const char* CONFIG_ENGINE = "engine_config"; -static const char* CONFIG_DCBT = "use_blas_threshold"; -static const char* CONFIG_OMP_THREAD_NUM = "omp_thread_num"; +static const char* CONFIG_ENGINE_BLAS_THRESHOLD = "blas_threshold"; +static const char* CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT = "20"; +static const char* CONFIG_ENGINE_OMP_THREAD_NUM = "omp_thread_num"; +static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; +/* resource config */ static const char* CONFIG_RESOURCE = "resource_config"; -static const char* CONFIG_RESOURCES = "resources"; -static const char* CONFIG_RESOURCE_TYPE = "type"; -static const char* CONFIG_RESOURCE_DEVICE_ID = "device_id"; -static const char* CONFIG_RESOURCE_ENABLE_EXECUTOR = "enable_executor"; -static const char* CONFIG_RESOURCE_NUM = "gpu_resource_num"; -static const char* CONFIG_RESOURCE_PIN_MEMORY = "pinned_memory"; -static const char* CONFIG_RESOURCE_TEMP_MEMORY = "temp_memory"; -static const char* CONFIG_RESOURCE_CONNECTIONS = "connections"; -static const char* CONFIG_SPEED_CONNECTIONS = "speed"; -static const char* CONFIG_ENDPOINT_CONNECTIONS = "endpoint"; +static const char* CONFIG_RESOURCE_MODE = "mode"; +static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; +static const char* CONFIG_RESOURCE_POOL = "pool"; class ServerConfig { diff --git a/cpp/src/server/grpc_impl/GrpcServer.cpp b/cpp/src/server/grpc_impl/GrpcServer.cpp index 4ff7a8200f..5160e0c8ad 100644 --- a/cpp/src/server/grpc_impl/GrpcServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcServer.cpp @@ -76,8 +76,8 @@ GrpcServer::StartService() { ServerConfig &config = ServerConfig::GetInstance(); ConfigNode server_config = config.GetConfig(CONFIG_SERVER); ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); - std::string address = server_config.GetValue(CONFIG_SERVER_ADDRESS, "127.0.0.1"); - int32_t port = server_config.GetInt32Value(CONFIG_SERVER_PORT, 19530); + std::string address = server_config.GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); + int32_t port = server_config.GetInt32Value(CONFIG_SERVER_PORT, std::stoi(CONFIG_SERVER_PORT_DEFAULT)); std::string server_address(address + ":" + std::to_string(port)); diff --git a/cpp/src/utils/SignalUtil.cpp b/cpp/src/utils/SignalUtil.cpp index 6c68f702d2..92678cfe6b 100644 --- a/cpp/src/utils/SignalUtil.cpp +++ b/cpp/src/utils/SignalUtil.cpp @@ -34,7 +34,7 @@ void SignalUtil::HandleSignal(int signum) { case SIGUSR2: { SERVER_LOG_INFO << "Server received signal: " << signum; - server::Server &server = server::Server::Instance(); + server::Server &server = server::Server::GetInstance(); server.Stop(); exit(0); @@ -43,7 +43,7 @@ void SignalUtil::HandleSignal(int signum) { SERVER_LOG_INFO << "Server received critical signal: " << signum; SignalUtil::PrintStacktrace(); - server::Server &server = server::Server::Instance(); + server::Server &server = server::Server::GetInstance(); server.Stop(); exit(1); diff --git a/cpp/src/wrapper/KnowhereResource.cpp b/cpp/src/wrapper/KnowhereResource.cpp index 007858de37..e59b56b574 100644 --- a/cpp/src/wrapper/KnowhereResource.cpp +++ b/cpp/src/wrapper/KnowhereResource.cpp @@ -41,14 +41,15 @@ ErrorCode KnowhereResource::Initialize() { server::ServerConfig& root_config = server::ServerConfig::GetInstance(); server::ConfigNode& db_config = root_config.GetConfig(server::CONFIG_DB); - int32_t build_index_gpu = db_config.GetInt32Value(server::CONFIG_DB_BUILD_INDEX_GPU, 0); + int32_t build_index_gpu = + db_config.GetInt32Value(server::CONFIG_DB_BUILD_INDEX_GPU, std::stoi(server::CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); gpu_resources.insert(std::make_pair(build_index_gpu, GpuResourceSetting())); //get search gpu resource server::ConfigNode& res_config = root_config.GetConfig(server::CONFIG_RESOURCE); - auto resources = res_config.GetSequence("resources"); + auto pool = res_config.GetSequence(server::CONFIG_RESOURCE_POOL); std::set gpu_ids; - for (auto &resource : resources) { + for (auto &resource : pool) { if (resource.length() < 4 || resource.substr(0, 3) != "gpu") { // invalid continue; diff --git a/cpp/unittest/server/rpc_test.cpp b/cpp/unittest/server/rpc_test.cpp index 69b258aa75..5ad6005a36 100644 --- a/cpp/unittest/server/rpc_test.cpp +++ b/cpp/unittest/server/rpc_test.cpp @@ -67,7 +67,7 @@ class RpcHandlerTest : public testing::Test { engine::DBOptions opt; server::ConfigNode &db_config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_DB); - db_config.SetValue(server::CONFIG_DB_URL, "sqlite://:@:/"); + db_config.SetValue(server::CONFIG_DB_BACKEND_URL, "sqlite://:@:/"); db_config.SetValue(server::CONFIG_DB_PATH, "/tmp/milvus_test"); db_config.SetValue(server::CONFIG_DB_SLAVE_PATH, ""); db_config.SetValue(server::CONFIG_DB_ARCHIVE_DISK, ""); @@ -77,7 +77,7 @@ class RpcHandlerTest : public testing::Test { cache_config.SetValue(server::CONFIG_INSERT_CACHE_IMMEDIATELY, ""); server::ConfigNode &engine_config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_ENGINE); - engine_config.SetValue(server::CONFIG_OMP_THREAD_NUM, ""); + engine_config.SetValue(server::CONFIG_ENGINE_OMP_THREAD_NUM, ""); server::ConfigNode &serverConfig = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_SERVER); // serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "cluster"); From 5806f1dd5cea3e05fd15c048d703b27dedbaf3f2 Mon Sep 17 00:00:00 2001 From: test Date: Fri, 20 Sep 2019 09:14:20 +0000 Subject: [PATCH 068/217] MS-575 Add Clang-format & Clang-tidy & Cpplint Former-commit-id: 9e0fe09aaa5f30b7fcdafa66415b9b35921954c9 --- cpp/CMakeLists.txt | 14 + cpp/build-support/cpplint.py | 6476 ++++++++++++++++++++++++++++++ cpp/build-support/run_cpplint.py | 132 + 3 files changed, 6622 insertions(+) create mode 100755 cpp/build-support/cpplint.py create mode 100755 cpp/build-support/run_cpplint.py diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 26acbd2eee..d226a03e61 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -180,6 +180,20 @@ if(NOT LINT_EXCLUSIONS_FILE) set(LINT_EXCLUSIONS_FILE ${BUILD_SUPPORT_DIR}/lint_exclusions.txt) endif() +find_program(CPPLINT_BIN NAMES cpplint cpplint.py HINTS ${BUILD_SUPPORT_DIR}) +message(STATUS "Found cpplint executable at ${CPPLINT_BIN}") + +add_custom_target(lint + ${PYTHON_EXECUTABLE} + ${BUILD_SUPPORT_DIR}/run_cpplint.py + --cpplint_binary + ${CPPLINT_BIN} + --exclude_globs + ${LINT_EXCLUSIONS_FILE} + --source_dir + ${CMAKE_CURRENT_SOURCE_DIR}/src + ${MILVUS_LINT_QUIET}) + # # "make format" and "make check-format" targets # diff --git a/cpp/build-support/cpplint.py b/cpp/build-support/cpplint.py new file mode 100755 index 0000000000..8437d8cbaf --- /dev/null +++ b/cpp/build-support/cpplint.py @@ -0,0 +1,6476 @@ +#!/usr/bin/env python +# +# Copyright (c) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Does google-lint on c++ files. + +The goal of this script is to identify places in the code that *may* +be in non-compliance with google style. It does not attempt to fix +up these problems -- the point is to educate. It does also not +attempt to find all problems, or to ensure that everything it does +find is legitimately a problem. + +In particular, we can get very confused by /* and // inside strings! +We do a small hack, which is to ignore //'s with "'s after them on the +same line, but it is far from perfect (in either direction). +""" + +import codecs +import copy +import getopt +import glob +import itertools +import math # for log +import os +import re +import sre_compile +import string +import sys +import unicodedata +import xml.etree.ElementTree + +# if empty, use defaults +_header_extensions = set([]) + +# if empty, use defaults +_valid_extensions = set([]) + + +# Files with any of these extensions are considered to be +# header files (and will undergo different style checks). +# This set can be extended by using the --headers +# option (also supported in CPPLINT.cfg) +def GetHeaderExtensions(): + if not _header_extensions: + return set(['h', 'hpp', 'hxx', 'h++', 'cuh']) + return _header_extensions + +# The allowed extensions for file names +# This is set by --extensions flag +def GetAllExtensions(): + if not _valid_extensions: + return GetHeaderExtensions().union(set(['c', 'cc', 'cpp', 'cxx', 'c++', 'cu'])) + return _valid_extensions + +def GetNonHeaderExtensions(): + return GetAllExtensions().difference(GetHeaderExtensions()) + + +_USAGE = """ +Syntax: cpplint.py [--verbose=#] [--output=emacs|eclipse|vs7|junit] + [--filter=-x,+y,...] + [--counting=total|toplevel|detailed] [--repository=path] + [--root=subdir] [--linelength=digits] [--recursive] + [--exclude=path] + [--headers=ext1,ext2] + [--extensions=hpp,cpp,...] + [file] ... + + The style guidelines this tries to follow are those in + https://google.github.io/styleguide/cppguide.html + + Every problem is given a confidence score from 1-5, with 5 meaning we are + certain of the problem, and 1 meaning it could be a legitimate construct. + This will miss some errors, and is not a substitute for a code review. + + To suppress false-positive errors of a certain category, add a + 'NOLINT(category)' comment to the line. NOLINT or NOLINT(*) + suppresses errors of all categories on that line. + + The files passed in will be linted; at least one file must be provided. + Default linted extensions are %s. + Other file types will be ignored. + Change the extensions with the --extensions flag. + + Flags: + + output=emacs|eclipse|vs7|junit + By default, the output is formatted to ease emacs parsing. Output + compatible with eclipse (eclipse), Visual Studio (vs7), and JUnit + XML parsers such as those used in Jenkins and Bamboo may also be + used. Other formats are unsupported. + + verbose=# + Specify a number 0-5 to restrict errors to certain verbosity levels. + Errors with lower verbosity levels have lower confidence and are more + likely to be false positives. + + quiet + Supress output other than linting errors, such as information about + which files have been processed and excluded. + + filter=-x,+y,... + Specify a comma-separated list of category-filters to apply: only + error messages whose category names pass the filters will be printed. + (Category names are printed with the message and look like + "[whitespace/indent]".) Filters are evaluated left to right. + "-FOO" and "FOO" means "do not print categories that start with FOO". + "+FOO" means "do print categories that start with FOO". + + Examples: --filter=-whitespace,+whitespace/braces + --filter=whitespace,runtime/printf,+runtime/printf_format + --filter=-,+build/include_what_you_use + + To see a list of all the categories used in cpplint, pass no arg: + --filter= + + counting=total|toplevel|detailed + The total number of errors found is always printed. If + 'toplevel' is provided, then the count of errors in each of + the top-level categories like 'build' and 'whitespace' will + also be printed. If 'detailed' is provided, then a count + is provided for each category like 'build/class'. + + repository=path + The top level directory of the repository, used to derive the header + guard CPP variable. By default, this is determined by searching for a + path that contains .git, .hg, or .svn. When this flag is specified, the + given path is used instead. This option allows the header guard CPP + variable to remain consistent even if members of a team have different + repository root directories (such as when checking out a subdirectory + with SVN). In addition, users of non-mainstream version control systems + can use this flag to ensure readable header guard CPP variables. + + Examples: + Assuming that Alice checks out ProjectName and Bob checks out + ProjectName/trunk and trunk contains src/chrome/ui/browser.h, then + with no --repository flag, the header guard CPP variable will be: + + Alice => TRUNK_SRC_CHROME_BROWSER_UI_BROWSER_H_ + Bob => SRC_CHROME_BROWSER_UI_BROWSER_H_ + + If Alice uses the --repository=trunk flag and Bob omits the flag or + uses --repository=. then the header guard CPP variable will be: + + Alice => SRC_CHROME_BROWSER_UI_BROWSER_H_ + Bob => SRC_CHROME_BROWSER_UI_BROWSER_H_ + + root=subdir + The root directory used for deriving header guard CPP variables. This + directory is relative to the top level directory of the repository which + by default is determined by searching for a directory that contains .git, + .hg, or .svn but can also be controlled with the --repository flag. If + the specified directory does not exist, this flag is ignored. + + Examples: + Assuming that src is the top level directory of the repository, the + header guard CPP variables for src/chrome/browser/ui/browser.h are: + + No flag => CHROME_BROWSER_UI_BROWSER_H_ + --root=chrome => BROWSER_UI_BROWSER_H_ + --root=chrome/browser => UI_BROWSER_H_ + + linelength=digits + This is the allowed line length for the project. The default value is + 80 characters. + + Examples: + --linelength=120 + + recursive + Search for files to lint recursively. Each directory given in the list + of files to be linted is replaced by all files that descend from that + directory. Files with extensions not in the valid extensions list are + excluded. + + exclude=path + Exclude the given path from the list of files to be linted. Relative + paths are evaluated relative to the current directory and shell globbing + is performed. This flag can be provided multiple times to exclude + multiple files. + + Examples: + --exclude=one.cc + --exclude=src/*.cc + --exclude=src/*.cc --exclude=test/*.cc + + extensions=extension,extension,... + The allowed file extensions that cpplint will check + + Examples: + --extensions=%s + + headers=extension,extension,... + The allowed header extensions that cpplint will consider to be header files + (by default, only files with extensions %s + will be assumed to be headers) + + Examples: + --headers=%s + + cpplint.py supports per-directory configurations specified in CPPLINT.cfg + files. CPPLINT.cfg file can contain a number of key=value pairs. + Currently the following options are supported: + + set noparent + filter=+filter1,-filter2,... + exclude_files=regex + linelength=80 + root=subdir + + "set noparent" option prevents cpplint from traversing directory tree + upwards looking for more .cfg files in parent directories. This option + is usually placed in the top-level project directory. + + The "filter" option is similar in function to --filter flag. It specifies + message filters in addition to the |_DEFAULT_FILTERS| and those specified + through --filter command-line flag. + + "exclude_files" allows to specify a regular expression to be matched against + a file name. If the expression matches, the file is skipped and not run + through the linter. + + "linelength" specifies the allowed line length for the project. + + The "root" option is similar in function to the --root flag (see example + above). + + CPPLINT.cfg has an effect on files in the same directory and all + subdirectories, unless overridden by a nested configuration file. + + Example file: + filter=-build/include_order,+build/include_alpha + exclude_files=.*\\.cc + + The above example disables build/include_order warning and enables + build/include_alpha as well as excludes all .cc from being + processed by linter, in the current directory (where the .cfg + file is located) and all subdirectories. +""" % (list(GetAllExtensions()), + ','.join(list(GetAllExtensions())), + GetHeaderExtensions(), + ','.join(GetHeaderExtensions())) + +# We categorize each error message we print. Here are the categories. +# We want an explicit list so we can list them all in cpplint --filter=. +# If you add a new error message with a new category, add it to the list +# here! cpplint_unittest.py should tell you if you forget to do this. +_ERROR_CATEGORIES = [ + 'build/class', + 'build/c++11', + 'build/c++14', + 'build/c++tr1', + 'build/deprecated', + 'build/endif_comment', + 'build/explicit_make_pair', + 'build/forward_decl', + 'build/header_guard', + 'build/include', + 'build/include_subdir', + 'build/include_alpha', + 'build/include_order', + 'build/include_what_you_use', + 'build/namespaces_literals', + 'build/namespaces', + 'build/printf_format', + 'build/storage_class', + 'legal/copyright', + 'readability/alt_tokens', + 'readability/braces', + 'readability/casting', + 'readability/check', + 'readability/constructors', + 'readability/fn_size', + 'readability/inheritance', + 'readability/multiline_comment', + 'readability/multiline_string', + 'readability/namespace', + 'readability/nolint', + 'readability/nul', + 'readability/strings', + 'readability/todo', + 'readability/utf8', + 'runtime/arrays', + 'runtime/casting', + 'runtime/explicit', + 'runtime/int', + 'runtime/init', + 'runtime/invalid_increment', + 'runtime/member_string_references', + 'runtime/memset', + 'runtime/indentation_namespace', + 'runtime/operator', + 'runtime/printf', + 'runtime/printf_format', + 'runtime/references', + 'runtime/string', + 'runtime/threadsafe_fn', + 'runtime/vlog', + 'whitespace/blank_line', + 'whitespace/braces', + 'whitespace/comma', + 'whitespace/comments', + 'whitespace/empty_conditional_body', + 'whitespace/empty_if_body', + 'whitespace/empty_loop_body', + 'whitespace/end_of_line', + 'whitespace/ending_newline', + 'whitespace/forcolon', + 'whitespace/indent', + 'whitespace/line_length', + 'whitespace/newline', + 'whitespace/operators', + 'whitespace/parens', + 'whitespace/semicolon', + 'whitespace/tab', + 'whitespace/todo', + ] + +# These error categories are no longer enforced by cpplint, but for backwards- +# compatibility they may still appear in NOLINT comments. +_LEGACY_ERROR_CATEGORIES = [ + 'readability/streams', + 'readability/function', + ] + +# The default state of the category filter. This is overridden by the --filter= +# flag. By default all errors are on, so only add here categories that should be +# off by default (i.e., categories that must be enabled by the --filter= flags). +# All entries here should start with a '-' or '+', as in the --filter= flag. +_DEFAULT_FILTERS = ['-build/include_alpha'] + +# The default list of categories suppressed for C (not C++) files. +_DEFAULT_C_SUPPRESSED_CATEGORIES = [ + 'readability/casting', + ] + +# The default list of categories suppressed for Linux Kernel files. +_DEFAULT_KERNEL_SUPPRESSED_CATEGORIES = [ + 'whitespace/tab', + ] + +# We used to check for high-bit characters, but after much discussion we +# decided those were OK, as long as they were in UTF-8 and didn't represent +# hard-coded international strings, which belong in a separate i18n file. + +# C++ headers +_CPP_HEADERS = frozenset([ + # Legacy + 'algobase.h', + 'algo.h', + 'alloc.h', + 'builtinbuf.h', + 'bvector.h', + 'complex.h', + 'defalloc.h', + 'deque.h', + 'editbuf.h', + 'fstream.h', + 'function.h', + 'hash_map', + 'hash_map.h', + 'hash_set', + 'hash_set.h', + 'hashtable.h', + 'heap.h', + 'indstream.h', + 'iomanip.h', + 'iostream.h', + 'istream.h', + 'iterator.h', + 'list.h', + 'map.h', + 'multimap.h', + 'multiset.h', + 'ostream.h', + 'pair.h', + 'parsestream.h', + 'pfstream.h', + 'procbuf.h', + 'pthread_alloc', + 'pthread_alloc.h', + 'rope', + 'rope.h', + 'ropeimpl.h', + 'set.h', + 'slist', + 'slist.h', + 'stack.h', + 'stdiostream.h', + 'stl_alloc.h', + 'stl_relops.h', + 'streambuf.h', + 'stream.h', + 'strfile.h', + 'strstream.h', + 'tempbuf.h', + 'tree.h', + 'type_traits.h', + 'vector.h', + # 17.6.1.2 C++ library headers + 'algorithm', + 'array', + 'atomic', + 'bitset', + 'chrono', + 'codecvt', + 'complex', + 'condition_variable', + 'deque', + 'exception', + 'forward_list', + 'fstream', + 'functional', + 'future', + 'initializer_list', + 'iomanip', + 'ios', + 'iosfwd', + 'iostream', + 'istream', + 'iterator', + 'limits', + 'list', + 'locale', + 'map', + 'memory', + 'mutex', + 'new', + 'numeric', + 'ostream', + 'queue', + 'random', + 'ratio', + 'regex', + 'scoped_allocator', + 'set', + 'sstream', + 'stack', + 'stdexcept', + 'streambuf', + 'string', + 'strstream', + 'system_error', + 'thread', + 'tuple', + 'typeindex', + 'typeinfo', + 'type_traits', + 'unordered_map', + 'unordered_set', + 'utility', + 'valarray', + 'vector', + # 17.6.1.2 C++ headers for C library facilities + 'cassert', + 'ccomplex', + 'cctype', + 'cerrno', + 'cfenv', + 'cfloat', + 'cinttypes', + 'ciso646', + 'climits', + 'clocale', + 'cmath', + 'csetjmp', + 'csignal', + 'cstdalign', + 'cstdarg', + 'cstdbool', + 'cstddef', + 'cstdint', + 'cstdio', + 'cstdlib', + 'cstring', + 'ctgmath', + 'ctime', + 'cuchar', + 'cwchar', + 'cwctype', + ]) + +# Type names +_TYPES = re.compile( + r'^(?:' + # [dcl.type.simple] + r'(char(16_t|32_t)?)|wchar_t|' + r'bool|short|int|long|signed|unsigned|float|double|' + # [support.types] + r'(ptrdiff_t|size_t|max_align_t|nullptr_t)|' + # [cstdint.syn] + r'(u?int(_fast|_least)?(8|16|32|64)_t)|' + r'(u?int(max|ptr)_t)|' + r')$') + + +# These headers are excluded from [build/include] and [build/include_order] +# checks: +# - Anything not following google file name conventions (containing an +# uppercase character, such as Python.h or nsStringAPI.h, for example). +# - Lua headers. +_THIRD_PARTY_HEADERS_PATTERN = re.compile( + r'^(?:[^/]*[A-Z][^/]*\.h|lua\.h|lauxlib\.h|lualib\.h)$') + +# Pattern for matching FileInfo.BaseName() against test file name +_test_suffixes = ['_test', '_regtest', '_unittest'] +_TEST_FILE_SUFFIX = '(' + '|'.join(_test_suffixes) + r')$' + +# Pattern that matches only complete whitespace, possibly across multiple lines. +_EMPTY_CONDITIONAL_BODY_PATTERN = re.compile(r'^\s*$', re.DOTALL) + +# Assertion macros. These are defined in base/logging.h and +# testing/base/public/gunit.h. +_CHECK_MACROS = [ + 'DCHECK', 'CHECK', + 'EXPECT_TRUE', 'ASSERT_TRUE', + 'EXPECT_FALSE', 'ASSERT_FALSE', + ] + +# Replacement macros for CHECK/DCHECK/EXPECT_TRUE/EXPECT_FALSE +_CHECK_REPLACEMENT = dict([(macro_var, {}) for macro_var in _CHECK_MACROS]) + +for op, replacement in [('==', 'EQ'), ('!=', 'NE'), + ('>=', 'GE'), ('>', 'GT'), + ('<=', 'LE'), ('<', 'LT')]: + _CHECK_REPLACEMENT['DCHECK'][op] = 'DCHECK_%s' % replacement + _CHECK_REPLACEMENT['CHECK'][op] = 'CHECK_%s' % replacement + _CHECK_REPLACEMENT['EXPECT_TRUE'][op] = 'EXPECT_%s' % replacement + _CHECK_REPLACEMENT['ASSERT_TRUE'][op] = 'ASSERT_%s' % replacement + +for op, inv_replacement in [('==', 'NE'), ('!=', 'EQ'), + ('>=', 'LT'), ('>', 'LE'), + ('<=', 'GT'), ('<', 'GE')]: + _CHECK_REPLACEMENT['EXPECT_FALSE'][op] = 'EXPECT_%s' % inv_replacement + _CHECK_REPLACEMENT['ASSERT_FALSE'][op] = 'ASSERT_%s' % inv_replacement + +# Alternative tokens and their replacements. For full list, see section 2.5 +# Alternative tokens [lex.digraph] in the C++ standard. +# +# Digraphs (such as '%:') are not included here since it's a mess to +# match those on a word boundary. +_ALT_TOKEN_REPLACEMENT = { + 'and': '&&', + 'bitor': '|', + 'or': '||', + 'xor': '^', + 'compl': '~', + 'bitand': '&', + 'and_eq': '&=', + 'or_eq': '|=', + 'xor_eq': '^=', + 'not': '!', + 'not_eq': '!=' + } + +# Compile regular expression that matches all the above keywords. The "[ =()]" +# bit is meant to avoid matching these keywords outside of boolean expressions. +# +# False positives include C-style multi-line comments and multi-line strings +# but those have always been troublesome for cpplint. +_ALT_TOKEN_REPLACEMENT_PATTERN = re.compile( + r'[ =()](' + ('|'.join(_ALT_TOKEN_REPLACEMENT.keys())) + r')(?=[ (]|$)') + + +# These constants define types of headers for use with +# _IncludeState.CheckNextIncludeOrder(). +_C_SYS_HEADER = 1 +_CPP_SYS_HEADER = 2 +_LIKELY_MY_HEADER = 3 +_POSSIBLE_MY_HEADER = 4 +_OTHER_HEADER = 5 + +# These constants define the current inline assembly state +_NO_ASM = 0 # Outside of inline assembly block +_INSIDE_ASM = 1 # Inside inline assembly block +_END_ASM = 2 # Last line of inline assembly block +_BLOCK_ASM = 3 # The whole block is an inline assembly block + +# Match start of assembly blocks +_MATCH_ASM = re.compile(r'^\s*(?:asm|_asm|__asm|__asm__)' + r'(?:\s+(volatile|__volatile__))?' + r'\s*[{(]') + +# Match strings that indicate we're working on a C (not C++) file. +_SEARCH_C_FILE = re.compile(r'\b(?:LINT_C_FILE|' + r'vim?:\s*.*(\s*|:)filetype=c(\s*|:|$))') + +# Match string that indicates we're working on a Linux Kernel file. +_SEARCH_KERNEL_FILE = re.compile(r'\b(?:LINT_KERNEL_FILE)') + +_regexp_compile_cache = {} + +# {str, set(int)}: a map from error categories to sets of linenumbers +# on which those errors are expected and should be suppressed. +_error_suppressions = {} + +# The root directory used for deriving header guard CPP variable. +# This is set by --root flag. +_root = None + +# The top level repository directory. If set, _root is calculated relative to +# this directory instead of the directory containing version control artifacts. +# This is set by the --repository flag. +_repository = None + +# Files to exclude from linting. This is set by the --exclude flag. +_excludes = None + +# Whether to supress PrintInfo messages +_quiet = False + +# The allowed line length of files. +# This is set by --linelength flag. +_line_length = 80 + +try: + xrange(1, 0) +except NameError: + # -- pylint: disable=redefined-builtin + xrange = range + +try: + unicode +except NameError: + # -- pylint: disable=redefined-builtin + basestring = unicode = str + +try: + long(2) +except NameError: + # -- pylint: disable=redefined-builtin + long = int + +if sys.version_info < (3,): + # -- pylint: disable=no-member + # BINARY_TYPE = str + itervalues = dict.itervalues + iteritems = dict.iteritems +else: + # BINARY_TYPE = bytes + itervalues = dict.values + iteritems = dict.items + +def unicode_escape_decode(x): + if sys.version_info < (3,): + return codecs.unicode_escape_decode(x)[0] + else: + return x + +# {str, bool}: a map from error categories to booleans which indicate if the +# category should be suppressed for every line. +_global_error_suppressions = {} + + + + +def ParseNolintSuppressions(filename, raw_line, linenum, error): + """Updates the global list of line error-suppressions. + + Parses any NOLINT comments on the current line, updating the global + error_suppressions store. Reports an error if the NOLINT comment + was malformed. + + Args: + filename: str, the name of the input file. + raw_line: str, the line of input text, with comments. + linenum: int, the number of the current line. + error: function, an error handler. + """ + matched = Search(r'\bNOLINT(NEXTLINE)?\b(\([^)]+\))?', raw_line) + if matched: + if matched.group(1): + suppressed_line = linenum + 1 + else: + suppressed_line = linenum + category = matched.group(2) + if category in (None, '(*)'): # => "suppress all" + _error_suppressions.setdefault(None, set()).add(suppressed_line) + else: + if category.startswith('(') and category.endswith(')'): + category = category[1:-1] + if category in _ERROR_CATEGORIES: + _error_suppressions.setdefault(category, set()).add(suppressed_line) + elif category not in _LEGACY_ERROR_CATEGORIES: + error(filename, linenum, 'readability/nolint', 5, + 'Unknown NOLINT error category: %s' % category) + + +def ProcessGlobalSuppresions(lines): + """Updates the list of global error suppressions. + + Parses any lint directives in the file that have global effect. + + Args: + lines: An array of strings, each representing a line of the file, with the + last element being empty if the file is terminated with a newline. + """ + for line in lines: + if _SEARCH_C_FILE.search(line): + for category in _DEFAULT_C_SUPPRESSED_CATEGORIES: + _global_error_suppressions[category] = True + if _SEARCH_KERNEL_FILE.search(line): + for category in _DEFAULT_KERNEL_SUPPRESSED_CATEGORIES: + _global_error_suppressions[category] = True + + +def ResetNolintSuppressions(): + """Resets the set of NOLINT suppressions to empty.""" + _error_suppressions.clear() + _global_error_suppressions.clear() + + +def IsErrorSuppressedByNolint(category, linenum): + """Returns true if the specified error category is suppressed on this line. + + Consults the global error_suppressions map populated by + ParseNolintSuppressions/ProcessGlobalSuppresions/ResetNolintSuppressions. + + Args: + category: str, the category of the error. + linenum: int, the current line number. + Returns: + bool, True iff the error should be suppressed due to a NOLINT comment or + global suppression. + """ + return (_global_error_suppressions.get(category, False) or + linenum in _error_suppressions.get(category, set()) or + linenum in _error_suppressions.get(None, set())) + + +def Match(pattern, s): + """Matches the string with the pattern, caching the compiled regexp.""" + # The regexp compilation caching is inlined in both Match and Search for + # performance reasons; factoring it out into a separate function turns out + # to be noticeably expensive. + if pattern not in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].match(s) + + +def ReplaceAll(pattern, rep, s): + """Replaces instances of pattern in a string with a replacement. + + The compiled regex is kept in a cache shared by Match and Search. + + Args: + pattern: regex pattern + rep: replacement text + s: search string + + Returns: + string with replacements made (or original string if no replacements) + """ + if pattern not in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].sub(rep, s) + + +def Search(pattern, s): + """Searches the string for the pattern, caching the compiled regexp.""" + if pattern not in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].search(s) + + +def _IsSourceExtension(s): + """File extension (excluding dot) matches a source file extension.""" + return s in GetNonHeaderExtensions() + + +class _IncludeState(object): + """Tracks line numbers for includes, and the order in which includes appear. + + include_list contains list of lists of (header, line number) pairs. + It's a lists of lists rather than just one flat list to make it + easier to update across preprocessor boundaries. + + Call CheckNextIncludeOrder() once for each header in the file, passing + in the type constants defined above. Calls in an illegal order will + raise an _IncludeError with an appropriate error message. + + """ + # self._section will move monotonically through this set. If it ever + # needs to move backwards, CheckNextIncludeOrder will raise an error. + _INITIAL_SECTION = 0 + _MY_H_SECTION = 1 + _C_SECTION = 2 + _CPP_SECTION = 3 + _OTHER_H_SECTION = 4 + + _TYPE_NAMES = { + _C_SYS_HEADER: 'C system header', + _CPP_SYS_HEADER: 'C++ system header', + _LIKELY_MY_HEADER: 'header this file implements', + _POSSIBLE_MY_HEADER: 'header this file may implement', + _OTHER_HEADER: 'other header', + } + _SECTION_NAMES = { + _INITIAL_SECTION: "... nothing. (This can't be an error.)", + _MY_H_SECTION: 'a header this file implements', + _C_SECTION: 'C system header', + _CPP_SECTION: 'C++ system header', + _OTHER_H_SECTION: 'other header', + } + + def __init__(self): + self.include_list = [[]] + self._section = None + self._last_header = None + self.ResetSection('') + + def FindHeader(self, header): + """Check if a header has already been included. + + Args: + header: header to check. + Returns: + Line number of previous occurrence, or -1 if the header has not + been seen before. + """ + for section_list in self.include_list: + for f in section_list: + if f[0] == header: + return f[1] + return -1 + + def ResetSection(self, directive): + """Reset section checking for preprocessor directive. + + Args: + directive: preprocessor directive (e.g. "if", "else"). + """ + # The name of the current section. + self._section = self._INITIAL_SECTION + # The path of last found header. + self._last_header = '' + + # Update list of includes. Note that we never pop from the + # include list. + if directive in ('if', 'ifdef', 'ifndef'): + self.include_list.append([]) + elif directive in ('else', 'elif'): + self.include_list[-1] = [] + + def SetLastHeader(self, header_path): + self._last_header = header_path + + def CanonicalizeAlphabeticalOrder(self, header_path): + """Returns a path canonicalized for alphabetical comparison. + + - replaces "-" with "_" so they both cmp the same. + - removes '-inl' since we don't require them to be after the main header. + - lowercase everything, just in case. + + Args: + header_path: Path to be canonicalized. + + Returns: + Canonicalized path. + """ + return header_path.replace('-inl.h', '.h').replace('-', '_').lower() + + def IsInAlphabeticalOrder(self, clean_lines, linenum, header_path): + """Check if a header is in alphabetical order with the previous header. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + header_path: Canonicalized header to be checked. + + Returns: + Returns true if the header is in alphabetical order. + """ + # If previous section is different from current section, _last_header will + # be reset to empty string, so it's always less than current header. + # + # If previous line was a blank line, assume that the headers are + # intentionally sorted the way they are. + if (self._last_header > header_path and + Match(r'^\s*#\s*include\b', clean_lines.elided[linenum - 1])): + return False + return True + + def CheckNextIncludeOrder(self, header_type): + """Returns a non-empty error message if the next header is out of order. + + This function also updates the internal state to be ready to check + the next include. + + Args: + header_type: One of the _XXX_HEADER constants defined above. + + Returns: + The empty string if the header is in the right order, or an + error message describing what's wrong. + + """ + error_message = ('Found %s after %s' % + (self._TYPE_NAMES[header_type], + self._SECTION_NAMES[self._section])) + + last_section = self._section + + if header_type == _C_SYS_HEADER: + if self._section <= self._C_SECTION: + self._section = self._C_SECTION + else: + self._last_header = '' + return error_message + elif header_type == _CPP_SYS_HEADER: + if self._section <= self._CPP_SECTION: + self._section = self._CPP_SECTION + else: + self._last_header = '' + return error_message + elif header_type == _LIKELY_MY_HEADER: + if self._section <= self._MY_H_SECTION: + self._section = self._MY_H_SECTION + else: + self._section = self._OTHER_H_SECTION + elif header_type == _POSSIBLE_MY_HEADER: + if self._section <= self._MY_H_SECTION: + self._section = self._MY_H_SECTION + else: + # This will always be the fallback because we're not sure + # enough that the header is associated with this file. + self._section = self._OTHER_H_SECTION + else: + assert header_type == _OTHER_HEADER + self._section = self._OTHER_H_SECTION + + if last_section != self._section: + self._last_header = '' + + return '' + + +class _CppLintState(object): + """Maintains module-wide state..""" + + def __init__(self): + self.verbose_level = 1 # global setting. + self.error_count = 0 # global count of reported errors + # filters to apply when emitting error messages + self.filters = _DEFAULT_FILTERS[:] + # backup of filter list. Used to restore the state after each file. + self._filters_backup = self.filters[:] + self.counting = 'total' # In what way are we counting errors? + self.errors_by_category = {} # string to int dict storing error counts + + # output format: + # "emacs" - format that emacs can parse (default) + # "eclipse" - format that eclipse can parse + # "vs7" - format that Microsoft Visual Studio 7 can parse + # "junit" - format that Jenkins, Bamboo, etc can parse + self.output_format = 'emacs' + + # For JUnit output, save errors and failures until the end so that they + # can be written into the XML + self._junit_errors = [] + self._junit_failures = [] + + def SetOutputFormat(self, output_format): + """Sets the output format for errors.""" + self.output_format = output_format + + def SetVerboseLevel(self, level): + """Sets the module's verbosity, and returns the previous setting.""" + last_verbose_level = self.verbose_level + self.verbose_level = level + return last_verbose_level + + def SetCountingStyle(self, counting_style): + """Sets the module's counting options.""" + self.counting = counting_style + + def SetFilters(self, filters): + """Sets the error-message filters. + + These filters are applied when deciding whether to emit a given + error message. + + Args: + filters: A string of comma-separated filters (eg "+whitespace/indent"). + Each filter should start with + or -; else we die. + + Raises: + ValueError: The comma-separated filters did not all start with '+' or '-'. + E.g. "-,+whitespace,-whitespace/indent,whitespace/badfilter" + """ + # Default filters always have less priority than the flag ones. + self.filters = _DEFAULT_FILTERS[:] + self.AddFilters(filters) + + def AddFilters(self, filters): + """ Adds more filters to the existing list of error-message filters. """ + for filt in filters.split(','): + clean_filt = filt.strip() + if clean_filt: + self.filters.append(clean_filt) + for filt in self.filters: + if not (filt.startswith('+') or filt.startswith('-')): + raise ValueError('Every filter in --filters must start with + or -' + ' (%s does not)' % filt) + + def BackupFilters(self): + """ Saves the current filter list to backup storage.""" + self._filters_backup = self.filters[:] + + def RestoreFilters(self): + """ Restores filters previously backed up.""" + self.filters = self._filters_backup[:] + + def ResetErrorCounts(self): + """Sets the module's error statistic back to zero.""" + self.error_count = 0 + self.errors_by_category = {} + + def IncrementErrorCount(self, category): + """Bumps the module's error statistic.""" + self.error_count += 1 + if self.counting in ('toplevel', 'detailed'): + if self.counting != 'detailed': + category = category.split('/')[0] + if category not in self.errors_by_category: + self.errors_by_category[category] = 0 + self.errors_by_category[category] += 1 + + def PrintErrorCounts(self): + """Print a summary of errors by category, and the total.""" + for category, count in sorted(iteritems(self.errors_by_category)): + self.PrintInfo('Category \'%s\' errors found: %d\n' % + (category, count)) + if self.error_count > 0: + self.PrintInfo('Total errors found: %d\n' % self.error_count) + + def PrintInfo(self, message): + if not _quiet and self.output_format != 'junit': + sys.stderr.write(message) + + def PrintError(self, message): + if self.output_format == 'junit': + self._junit_errors.append(message) + else: + sys.stderr.write(message) + + def AddJUnitFailure(self, filename, linenum, message, category, confidence): + self._junit_failures.append((filename, linenum, message, category, + confidence)) + + def FormatJUnitXML(self): + num_errors = len(self._junit_errors) + num_failures = len(self._junit_failures) + + testsuite = xml.etree.ElementTree.Element('testsuite') + testsuite.attrib['name'] = 'cpplint' + testsuite.attrib['errors'] = str(num_errors) + testsuite.attrib['failures'] = str(num_failures) + + if num_errors == 0 and num_failures == 0: + testsuite.attrib['tests'] = str(1) + xml.etree.ElementTree.SubElement(testsuite, 'testcase', name='passed') + + else: + testsuite.attrib['tests'] = str(num_errors + num_failures) + if num_errors > 0: + testcase = xml.etree.ElementTree.SubElement(testsuite, 'testcase') + testcase.attrib['name'] = 'errors' + error = xml.etree.ElementTree.SubElement(testcase, 'error') + error.text = '\n'.join(self._junit_errors) + if num_failures > 0: + # Group failures by file + failed_file_order = [] + failures_by_file = {} + for failure in self._junit_failures: + failed_file = failure[0] + if failed_file not in failed_file_order: + failed_file_order.append(failed_file) + failures_by_file[failed_file] = [] + failures_by_file[failed_file].append(failure) + # Create a testcase for each file + for failed_file in failed_file_order: + failures = failures_by_file[failed_file] + testcase = xml.etree.ElementTree.SubElement(testsuite, 'testcase') + testcase.attrib['name'] = failed_file + failure = xml.etree.ElementTree.SubElement(testcase, 'failure') + template = '{0}: {1} [{2}] [{3}]' + texts = [template.format(f[1], f[2], f[3], f[4]) for f in failures] + failure.text = '\n'.join(texts) + + xml_decl = '\n' + return xml_decl + xml.etree.ElementTree.tostring(testsuite, 'utf-8').decode('utf-8') + + +_cpplint_state = _CppLintState() + + +def _OutputFormat(): + """Gets the module's output format.""" + return _cpplint_state.output_format + + +def _SetOutputFormat(output_format): + """Sets the module's output format.""" + _cpplint_state.SetOutputFormat(output_format) + + +def _VerboseLevel(): + """Returns the module's verbosity setting.""" + return _cpplint_state.verbose_level + + +def _SetVerboseLevel(level): + """Sets the module's verbosity, and returns the previous setting.""" + return _cpplint_state.SetVerboseLevel(level) + + +def _SetCountingStyle(level): + """Sets the module's counting options.""" + _cpplint_state.SetCountingStyle(level) + + +def _Filters(): + """Returns the module's list of output filters, as a list.""" + return _cpplint_state.filters + + +def _SetFilters(filters): + """Sets the module's error-message filters. + + These filters are applied when deciding whether to emit a given + error message. + + Args: + filters: A string of comma-separated filters (eg "whitespace/indent"). + Each filter should start with + or -; else we die. + """ + _cpplint_state.SetFilters(filters) + +def _AddFilters(filters): + """Adds more filter overrides. + + Unlike _SetFilters, this function does not reset the current list of filters + available. + + Args: + filters: A string of comma-separated filters (eg "whitespace/indent"). + Each filter should start with + or -; else we die. + """ + _cpplint_state.AddFilters(filters) + +def _BackupFilters(): + """ Saves the current filter list to backup storage.""" + _cpplint_state.BackupFilters() + +def _RestoreFilters(): + """ Restores filters previously backed up.""" + _cpplint_state.RestoreFilters() + +class _FunctionState(object): + """Tracks current function name and the number of lines in its body.""" + + _NORMAL_TRIGGER = 250 # for --v=0, 500 for --v=1, etc. + _TEST_TRIGGER = 400 # about 50% more than _NORMAL_TRIGGER. + + def __init__(self): + self.in_a_function = False + self.lines_in_function = 0 + self.current_function = '' + + def Begin(self, function_name): + """Start analyzing function body. + + Args: + function_name: The name of the function being tracked. + """ + self.in_a_function = True + self.lines_in_function = 0 + self.current_function = function_name + + def Count(self): + """Count line in current function body.""" + if self.in_a_function: + self.lines_in_function += 1 + + def Check(self, error, filename, linenum): + """Report if too many lines in function body. + + Args: + error: The function to call with any errors found. + filename: The name of the current file. + linenum: The number of the line to check. + """ + if not self.in_a_function: + return + + if Match(r'T(EST|est)', self.current_function): + base_trigger = self._TEST_TRIGGER + else: + base_trigger = self._NORMAL_TRIGGER + trigger = base_trigger * 2**_VerboseLevel() + + if self.lines_in_function > trigger: + error_level = int(math.log(self.lines_in_function / base_trigger, 2)) + # 50 => 0, 100 => 1, 200 => 2, 400 => 3, 800 => 4, 1600 => 5, ... + if error_level > 5: + error_level = 5 + error(filename, linenum, 'readability/fn_size', error_level, + 'Small and focused functions are preferred:' + ' %s has %d non-comment lines' + ' (error triggered by exceeding %d lines).' % ( + self.current_function, self.lines_in_function, trigger)) + + def End(self): + """Stop analyzing function body.""" + self.in_a_function = False + + +class _IncludeError(Exception): + """Indicates a problem with the include order in a file.""" + pass + + +class FileInfo(object): + """Provides utility functions for filenames. + + FileInfo provides easy access to the components of a file's path + relative to the project root. + """ + + def __init__(self, filename): + self._filename = filename + + def FullName(self): + """Make Windows paths like Unix.""" + return os.path.abspath(self._filename).replace('\\', '/') + + def RepositoryName(self): + r"""FullName after removing the local path to the repository. + + If we have a real absolute path name here we can try to do something smart: + detecting the root of the checkout and truncating /path/to/checkout from + the name so that we get header guards that don't include things like + "C:\Documents and Settings\..." or "/home/username/..." in them and thus + people on different computers who have checked the source out to different + locations won't see bogus errors. + """ + fullname = self.FullName() + + if os.path.exists(fullname): + project_dir = os.path.dirname(fullname) + + # If the user specified a repository path, it exists, and the file is + # contained in it, use the specified repository path + if _repository: + repo = FileInfo(_repository).FullName() + root_dir = project_dir + while os.path.exists(root_dir): + # allow case insensitive compare on Windows + if os.path.normcase(root_dir) == os.path.normcase(repo): + return os.path.relpath(fullname, root_dir).replace('\\', '/') + one_up_dir = os.path.dirname(root_dir) + if one_up_dir == root_dir: + break + root_dir = one_up_dir + + if os.path.exists(os.path.join(project_dir, ".svn")): + # If there's a .svn file in the current directory, we recursively look + # up the directory tree for the top of the SVN checkout + root_dir = project_dir + one_up_dir = os.path.dirname(root_dir) + while os.path.exists(os.path.join(one_up_dir, ".svn")): + root_dir = os.path.dirname(root_dir) + one_up_dir = os.path.dirname(one_up_dir) + + prefix = os.path.commonprefix([root_dir, project_dir]) + return fullname[len(prefix) + 1:] + + # Not SVN <= 1.6? Try to find a git, hg, or svn top level directory by + # searching up from the current path. + root_dir = current_dir = os.path.dirname(fullname) + while current_dir != os.path.dirname(current_dir): + if (os.path.exists(os.path.join(current_dir, ".git")) or + os.path.exists(os.path.join(current_dir, ".hg")) or + os.path.exists(os.path.join(current_dir, ".svn"))): + root_dir = current_dir + current_dir = os.path.dirname(current_dir) + + if (os.path.exists(os.path.join(root_dir, ".git")) or + os.path.exists(os.path.join(root_dir, ".hg")) or + os.path.exists(os.path.join(root_dir, ".svn"))): + prefix = os.path.commonprefix([root_dir, project_dir]) + return fullname[len(prefix) + 1:] + + # Don't know what to do; header guard warnings may be wrong... + return fullname + + def Split(self): + """Splits the file into the directory, basename, and extension. + + For 'chrome/browser/browser.cc', Split() would + return ('chrome/browser', 'browser', '.cc') + + Returns: + A tuple of (directory, basename, extension). + """ + + googlename = self.RepositoryName() + project, rest = os.path.split(googlename) + return (project,) + os.path.splitext(rest) + + def BaseName(self): + """File base name - text after the final slash, before the final period.""" + return self.Split()[1] + + def Extension(self): + """File extension - text following the final period, includes that period.""" + return self.Split()[2] + + def NoExtension(self): + """File has no source file extension.""" + return '/'.join(self.Split()[0:2]) + + def IsSource(self): + """File has a source file extension.""" + return _IsSourceExtension(self.Extension()[1:]) + + +def _ShouldPrintError(category, confidence, linenum): + """If confidence >= verbose, category passes filter and is not suppressed.""" + + # There are three ways we might decide not to print an error message: + # a "NOLINT(category)" comment appears in the source, + # the verbosity level isn't high enough, or the filters filter it out. + if IsErrorSuppressedByNolint(category, linenum): + return False + + if confidence < _cpplint_state.verbose_level: + return False + + is_filtered = False + for one_filter in _Filters(): + if one_filter.startswith('-'): + if category.startswith(one_filter[1:]): + is_filtered = True + elif one_filter.startswith('+'): + if category.startswith(one_filter[1:]): + is_filtered = False + else: + assert False # should have been checked for in SetFilter. + if is_filtered: + return False + + return True + + +def Error(filename, linenum, category, confidence, message): + """Logs the fact we've found a lint error. + + We log where the error was found, and also our confidence in the error, + that is, how certain we are this is a legitimate style regression, and + not a misidentification or a use that's sometimes justified. + + False positives can be suppressed by the use of + "cpplint(category)" comments on the offending line. These are + parsed into _error_suppressions. + + Args: + filename: The name of the file containing the error. + linenum: The number of the line containing the error. + category: A string used to describe the "category" this bug + falls under: "whitespace", say, or "runtime". Categories + may have a hierarchy separated by slashes: "whitespace/indent". + confidence: A number from 1-5 representing a confidence score for + the error, with 5 meaning that we are certain of the problem, + and 1 meaning that it could be a legitimate construct. + message: The error message. + """ + if _ShouldPrintError(category, confidence, linenum): + _cpplint_state.IncrementErrorCount(category) + if _cpplint_state.output_format == 'vs7': + _cpplint_state.PrintError('%s(%s): warning: %s [%s] [%d]\n' % ( + filename, linenum, message, category, confidence)) + elif _cpplint_state.output_format == 'eclipse': + sys.stderr.write('%s:%s: warning: %s [%s] [%d]\n' % ( + filename, linenum, message, category, confidence)) + elif _cpplint_state.output_format == 'junit': + _cpplint_state.AddJUnitFailure(filename, linenum, message, category, + confidence) + else: + final_message = '%s:%s: %s [%s] [%d]\n' % ( + filename, linenum, message, category, confidence) + sys.stderr.write(final_message) + +# Matches standard C++ escape sequences per 2.13.2.3 of the C++ standard. +_RE_PATTERN_CLEANSE_LINE_ESCAPES = re.compile( + r'\\([abfnrtv?"\\\']|\d+|x[0-9a-fA-F]+)') +# Match a single C style comment on the same line. +_RE_PATTERN_C_COMMENTS = r'/\*(?:[^*]|\*(?!/))*\*/' +# Matches multi-line C style comments. +# This RE is a little bit more complicated than one might expect, because we +# have to take care of space removals tools so we can handle comments inside +# statements better. +# The current rule is: We only clear spaces from both sides when we're at the +# end of the line. Otherwise, we try to remove spaces from the right side, +# if this doesn't work we try on left side but only if there's a non-character +# on the right. +_RE_PATTERN_CLEANSE_LINE_C_COMMENTS = re.compile( + r'(\s*' + _RE_PATTERN_C_COMMENTS + r'\s*$|' + + _RE_PATTERN_C_COMMENTS + r'\s+|' + + r'\s+' + _RE_PATTERN_C_COMMENTS + r'(?=\W)|' + + _RE_PATTERN_C_COMMENTS + r')') + + +def IsCppString(line): + """Does line terminate so, that the next symbol is in string constant. + + This function does not consider single-line nor multi-line comments. + + Args: + line: is a partial line of code starting from the 0..n. + + Returns: + True, if next character appended to 'line' is inside a + string constant. + """ + + line = line.replace(r'\\', 'XX') # after this, \\" does not match to \" + return ((line.count('"') - line.count(r'\"') - line.count("'\"'")) & 1) == 1 + + +def CleanseRawStrings(raw_lines): + """Removes C++11 raw strings from lines. + + Before: + static const char kData[] = R"( + multi-line string + )"; + + After: + static const char kData[] = "" + (replaced by blank line) + ""; + + Args: + raw_lines: list of raw lines. + + Returns: + list of lines with C++11 raw strings replaced by empty strings. + """ + + delimiter = None + lines_without_raw_strings = [] + for line in raw_lines: + if delimiter: + # Inside a raw string, look for the end + end = line.find(delimiter) + if end >= 0: + # Found the end of the string, match leading space for this + # line and resume copying the original lines, and also insert + # a "" on the last line. + leading_space = Match(r'^(\s*)\S', line) + line = leading_space.group(1) + '""' + line[end + len(delimiter):] + delimiter = None + else: + # Haven't found the end yet, append a blank line. + line = '""' + + # Look for beginning of a raw string, and replace them with + # empty strings. This is done in a loop to handle multiple raw + # strings on the same line. + while delimiter is None: + # Look for beginning of a raw string. + # See 2.14.15 [lex.string] for syntax. + # + # Once we have matched a raw string, we check the prefix of the + # line to make sure that the line is not part of a single line + # comment. It's done this way because we remove raw strings + # before removing comments as opposed to removing comments + # before removing raw strings. This is because there are some + # cpplint checks that requires the comments to be preserved, but + # we don't want to check comments that are inside raw strings. + matched = Match(r'^(.*?)\b(?:R|u8R|uR|UR|LR)"([^\s\\()]*)\((.*)$', line) + if (matched and + not Match(r'^([^\'"]|\'(\\.|[^\'])*\'|"(\\.|[^"])*")*//', + matched.group(1))): + delimiter = ')' + matched.group(2) + '"' + + end = matched.group(3).find(delimiter) + if end >= 0: + # Raw string ended on same line + line = (matched.group(1) + '""' + + matched.group(3)[end + len(delimiter):]) + delimiter = None + else: + # Start of a multi-line raw string + line = matched.group(1) + '""' + else: + break + + lines_without_raw_strings.append(line) + + # TODO(unknown): if delimiter is not None here, we might want to + # emit a warning for unterminated string. + return lines_without_raw_strings + + +def FindNextMultiLineCommentStart(lines, lineix): + """Find the beginning marker for a multiline comment.""" + while lineix < len(lines): + if lines[lineix].strip().startswith('/*'): + # Only return this marker if the comment goes beyond this line + if lines[lineix].strip().find('*/', 2) < 0: + return lineix + lineix += 1 + return len(lines) + + +def FindNextMultiLineCommentEnd(lines, lineix): + """We are inside a comment, find the end marker.""" + while lineix < len(lines): + if lines[lineix].strip().endswith('*/'): + return lineix + lineix += 1 + return len(lines) + + +def RemoveMultiLineCommentsFromRange(lines, begin, end): + """Clears a range of lines for multi-line comments.""" + # Having // dummy comments makes the lines non-empty, so we will not get + # unnecessary blank line warnings later in the code. + for i in range(begin, end): + lines[i] = '/**/' + + +def RemoveMultiLineComments(filename, lines, error): + """Removes multiline (c-style) comments from lines.""" + lineix = 0 + while lineix < len(lines): + lineix_begin = FindNextMultiLineCommentStart(lines, lineix) + if lineix_begin >= len(lines): + return + lineix_end = FindNextMultiLineCommentEnd(lines, lineix_begin) + if lineix_end >= len(lines): + error(filename, lineix_begin + 1, 'readability/multiline_comment', 5, + 'Could not find end of multi-line comment') + return + RemoveMultiLineCommentsFromRange(lines, lineix_begin, lineix_end + 1) + lineix = lineix_end + 1 + + +def CleanseComments(line): + """Removes //-comments and single-line C-style /* */ comments. + + Args: + line: A line of C++ source. + + Returns: + The line with single-line comments removed. + """ + commentpos = line.find('//') + if commentpos != -1 and not IsCppString(line[:commentpos]): + line = line[:commentpos].rstrip() + # get rid of /* ... */ + return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('', line) + + +class CleansedLines(object): + """Holds 4 copies of all lines with different preprocessing applied to them. + + 1) elided member contains lines without strings and comments. + 2) lines member contains lines without comments. + 3) raw_lines member contains all the lines without processing. + 4) lines_without_raw_strings member is same as raw_lines, but with C++11 raw + strings removed. + All these members are of , and of the same length. + """ + + def __init__(self, lines): + self.elided = [] + self.lines = [] + self.raw_lines = lines + self.num_lines = len(lines) + self.lines_without_raw_strings = CleanseRawStrings(lines) + for linenum in range(len(self.lines_without_raw_strings)): + self.lines.append(CleanseComments( + self.lines_without_raw_strings[linenum])) + elided = self._CollapseStrings(self.lines_without_raw_strings[linenum]) + self.elided.append(CleanseComments(elided)) + + def NumLines(self): + """Returns the number of lines represented.""" + return self.num_lines + + @staticmethod + def _CollapseStrings(elided): + """Collapses strings and chars on a line to simple "" or '' blocks. + + We nix strings first so we're not fooled by text like '"http://"' + + Args: + elided: The line being processed. + + Returns: + The line with collapsed strings. + """ + if _RE_PATTERN_INCLUDE.match(elided): + return elided + + # Remove escaped characters first to make quote/single quote collapsing + # basic. Things that look like escaped characters shouldn't occur + # outside of strings and chars. + elided = _RE_PATTERN_CLEANSE_LINE_ESCAPES.sub('', elided) + + # Replace quoted strings and digit separators. Both single quotes + # and double quotes are processed in the same loop, otherwise + # nested quotes wouldn't work. + collapsed = '' + while True: + # Find the first quote character + match = Match(r'^([^\'"]*)([\'"])(.*)$', elided) + if not match: + collapsed += elided + break + head, quote, tail = match.groups() + + if quote == '"': + # Collapse double quoted strings + second_quote = tail.find('"') + if second_quote >= 0: + collapsed += head + '""' + elided = tail[second_quote + 1:] + else: + # Unmatched double quote, don't bother processing the rest + # of the line since this is probably a multiline string. + collapsed += elided + break + else: + # Found single quote, check nearby text to eliminate digit separators. + # + # There is no special handling for floating point here, because + # the integer/fractional/exponent parts would all be parsed + # correctly as long as there are digits on both sides of the + # separator. So we are fine as long as we don't see something + # like "0.'3" (gcc 4.9.0 will not allow this literal). + if Search(r'\b(?:0[bBxX]?|[1-9])[0-9a-fA-F]*$', head): + match_literal = Match(r'^((?:\'?[0-9a-zA-Z_])*)(.*)$', "'" + tail) + collapsed += head + match_literal.group(1).replace("'", '') + elided = match_literal.group(2) + else: + second_quote = tail.find('\'') + if second_quote >= 0: + collapsed += head + "''" + elided = tail[second_quote + 1:] + else: + # Unmatched single quote + collapsed += elided + break + + return collapsed + + +def FindEndOfExpressionInLine(line, startpos, stack): + """Find the position just after the end of current parenthesized expression. + + Args: + line: a CleansedLines line. + startpos: start searching at this position. + stack: nesting stack at startpos. + + Returns: + On finding matching end: (index just after matching end, None) + On finding an unclosed expression: (-1, None) + Otherwise: (-1, new stack at end of this line) + """ + for i in xrange(startpos, len(line)): + char = line[i] + if char in '([{': + # Found start of parenthesized expression, push to expression stack + stack.append(char) + elif char == '<': + # Found potential start of template argument list + if i > 0 and line[i - 1] == '<': + # Left shift operator + if stack and stack[-1] == '<': + stack.pop() + if not stack: + return (-1, None) + elif i > 0 and Search(r'\boperator\s*$', line[0:i]): + # operator<, don't add to stack + continue + else: + # Tentative start of template argument list + stack.append('<') + elif char in ')]}': + # Found end of parenthesized expression. + # + # If we are currently expecting a matching '>', the pending '<' + # must have been an operator. Remove them from expression stack. + while stack and stack[-1] == '<': + stack.pop() + if not stack: + return (-1, None) + if ((stack[-1] == '(' and char == ')') or + (stack[-1] == '[' and char == ']') or + (stack[-1] == '{' and char == '}')): + stack.pop() + if not stack: + return (i + 1, None) + else: + # Mismatched parentheses + return (-1, None) + elif char == '>': + # Found potential end of template argument list. + + # Ignore "->" and operator functions + if (i > 0 and + (line[i - 1] == '-' or Search(r'\boperator\s*$', line[0:i - 1]))): + continue + + # Pop the stack if there is a matching '<'. Otherwise, ignore + # this '>' since it must be an operator. + if stack: + if stack[-1] == '<': + stack.pop() + if not stack: + return (i + 1, None) + elif char == ';': + # Found something that look like end of statements. If we are currently + # expecting a '>', the matching '<' must have been an operator, since + # template argument list should not contain statements. + while stack and stack[-1] == '<': + stack.pop() + if not stack: + return (-1, None) + + # Did not find end of expression or unbalanced parentheses on this line + return (-1, stack) + + +def CloseExpression(clean_lines, linenum, pos): + """If input points to ( or { or [ or <, finds the position that closes it. + + If lines[linenum][pos] points to a '(' or '{' or '[' or '<', finds the + linenum/pos that correspond to the closing of the expression. + + TODO(unknown): cpplint spends a fair bit of time matching parentheses. + Ideally we would want to index all opening and closing parentheses once + and have CloseExpression be just a simple lookup, but due to preprocessor + tricks, this is not so easy. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + pos: A position on the line. + + Returns: + A tuple (line, linenum, pos) pointer *past* the closing brace, or + (line, len(lines), -1) if we never find a close. Note we ignore + strings and comments when matching; and the line we return is the + 'cleansed' line at linenum. + """ + + line = clean_lines.elided[linenum] + if (line[pos] not in '({[<') or Match(r'<[<=]', line[pos:]): + return (line, clean_lines.NumLines(), -1) + + # Check first line + (end_pos, stack) = FindEndOfExpressionInLine(line, pos, []) + if end_pos > -1: + return (line, linenum, end_pos) + + # Continue scanning forward + while stack and linenum < clean_lines.NumLines() - 1: + linenum += 1 + line = clean_lines.elided[linenum] + (end_pos, stack) = FindEndOfExpressionInLine(line, 0, stack) + if end_pos > -1: + return (line, linenum, end_pos) + + # Did not find end of expression before end of file, give up + return (line, clean_lines.NumLines(), -1) + + +def FindStartOfExpressionInLine(line, endpos, stack): + """Find position at the matching start of current expression. + + This is almost the reverse of FindEndOfExpressionInLine, but note + that the input position and returned position differs by 1. + + Args: + line: a CleansedLines line. + endpos: start searching at this position. + stack: nesting stack at endpos. + + Returns: + On finding matching start: (index at matching start, None) + On finding an unclosed expression: (-1, None) + Otherwise: (-1, new stack at beginning of this line) + """ + i = endpos + while i >= 0: + char = line[i] + if char in ')]}': + # Found end of expression, push to expression stack + stack.append(char) + elif char == '>': + # Found potential end of template argument list. + # + # Ignore it if it's a "->" or ">=" or "operator>" + if (i > 0 and + (line[i - 1] == '-' or + Match(r'\s>=\s', line[i - 1:]) or + Search(r'\boperator\s*$', line[0:i]))): + i -= 1 + else: + stack.append('>') + elif char == '<': + # Found potential start of template argument list + if i > 0 and line[i - 1] == '<': + # Left shift operator + i -= 1 + else: + # If there is a matching '>', we can pop the expression stack. + # Otherwise, ignore this '<' since it must be an operator. + if stack and stack[-1] == '>': + stack.pop() + if not stack: + return (i, None) + elif char in '([{': + # Found start of expression. + # + # If there are any unmatched '>' on the stack, they must be + # operators. Remove those. + while stack and stack[-1] == '>': + stack.pop() + if not stack: + return (-1, None) + if ((char == '(' and stack[-1] == ')') or + (char == '[' and stack[-1] == ']') or + (char == '{' and stack[-1] == '}')): + stack.pop() + if not stack: + return (i, None) + else: + # Mismatched parentheses + return (-1, None) + elif char == ';': + # Found something that look like end of statements. If we are currently + # expecting a '<', the matching '>' must have been an operator, since + # template argument list should not contain statements. + while stack and stack[-1] == '>': + stack.pop() + if not stack: + return (-1, None) + + i -= 1 + + return (-1, stack) + + +def ReverseCloseExpression(clean_lines, linenum, pos): + """If input points to ) or } or ] or >, finds the position that opens it. + + If lines[linenum][pos] points to a ')' or '}' or ']' or '>', finds the + linenum/pos that correspond to the opening of the expression. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + pos: A position on the line. + + Returns: + A tuple (line, linenum, pos) pointer *at* the opening brace, or + (line, 0, -1) if we never find the matching opening brace. Note + we ignore strings and comments when matching; and the line we + return is the 'cleansed' line at linenum. + """ + line = clean_lines.elided[linenum] + if line[pos] not in ')}]>': + return (line, 0, -1) + + # Check last line + (start_pos, stack) = FindStartOfExpressionInLine(line, pos, []) + if start_pos > -1: + return (line, linenum, start_pos) + + # Continue scanning backward + while stack and linenum > 0: + linenum -= 1 + line = clean_lines.elided[linenum] + (start_pos, stack) = FindStartOfExpressionInLine(line, len(line) - 1, stack) + if start_pos > -1: + return (line, linenum, start_pos) + + # Did not find start of expression before beginning of file, give up + return (line, 0, -1) + + +def CheckForCopyright(filename, lines, error): + """Logs an error if no Copyright message appears at the top of the file.""" + + # We'll say it should occur by line 10. Don't forget there's a + # dummy line at the front. + for line in range(1, min(len(lines), 11)): + if re.search(r'Copyright', lines[line], re.I): break + else: # means no copyright line was found + error(filename, 0, 'legal/copyright', 5, + 'No copyright message found. ' + 'You should have a line: "Copyright [year] "') + + +def GetIndentLevel(line): + """Return the number of leading spaces in line. + + Args: + line: A string to check. + + Returns: + An integer count of leading spaces, possibly zero. + """ + indent = Match(r'^( *)\S', line) + if indent: + return len(indent.group(1)) + else: + return 0 + + +def GetHeaderGuardCPPVariable(filename): + """Returns the CPP variable that should be used as a header guard. + + Args: + filename: The name of a C++ header file. + + Returns: + The CPP variable that should be used as a header guard in the + named file. + + """ + + # Restores original filename in case that cpplint is invoked from Emacs's + # flymake. + filename = re.sub(r'_flymake\.h$', '.h', filename) + filename = re.sub(r'/\.flymake/([^/]*)$', r'/\1', filename) + # Replace 'c++' with 'cpp'. + filename = filename.replace('C++', 'cpp').replace('c++', 'cpp') + + fileinfo = FileInfo(filename) + file_path_from_root = fileinfo.RepositoryName() + if _root: + suffix = os.sep + # On Windows using directory separator will leave us with + # "bogus escape error" unless we properly escape regex. + if suffix == '\\': + suffix += '\\' + file_path_from_root = re.sub('^' + _root + suffix, '', file_path_from_root) + return re.sub(r'[^a-zA-Z0-9]', '_', file_path_from_root).upper() + '_' + + +def CheckForHeaderGuard(filename, clean_lines, error): + """Checks that the file contains a header guard. + + Logs an error if no #ifndef header guard is present. For other + headers, checks that the full pathname is used. + + Args: + filename: The name of the C++ header file. + clean_lines: A CleansedLines instance containing the file. + error: The function to call with any errors found. + """ + + # Don't check for header guards if there are error suppression + # comments somewhere in this file. + # + # Because this is silencing a warning for a nonexistent line, we + # only support the very specific NOLINT(build/header_guard) syntax, + # and not the general NOLINT or NOLINT(*) syntax. + raw_lines = clean_lines.lines_without_raw_strings + for i in raw_lines: + if Search(r'//\s*NOLINT\(build/header_guard\)', i): + return + + # Allow pragma once instead of header guards + for i in raw_lines: + if Search(r'^\s*#pragma\s+once', i): + return + + cppvar = GetHeaderGuardCPPVariable(filename) + + ifndef = '' + ifndef_linenum = 0 + define = '' + endif = '' + endif_linenum = 0 + for linenum, line in enumerate(raw_lines): + linesplit = line.split() + if len(linesplit) >= 2: + # find the first occurrence of #ifndef and #define, save arg + if not ifndef and linesplit[0] == '#ifndef': + # set ifndef to the header guard presented on the #ifndef line. + ifndef = linesplit[1] + ifndef_linenum = linenum + if not define and linesplit[0] == '#define': + define = linesplit[1] + # find the last occurrence of #endif, save entire line + if line.startswith('#endif'): + endif = line + endif_linenum = linenum + + if not ifndef or not define or ifndef != define: + error(filename, 0, 'build/header_guard', 5, + 'No #ifndef header guard found, suggested CPP variable is: %s' % + cppvar) + return + + # The guard should be PATH_FILE_H_, but we also allow PATH_FILE_H__ + # for backward compatibility. + if ifndef != cppvar: + error_level = 0 + if ifndef != cppvar + '_': + error_level = 5 + + ParseNolintSuppressions(filename, raw_lines[ifndef_linenum], ifndef_linenum, + error) + error(filename, ifndef_linenum, 'build/header_guard', error_level, + '#ifndef header guard has wrong style, please use: %s' % cppvar) + + # Check for "//" comments on endif line. + ParseNolintSuppressions(filename, raw_lines[endif_linenum], endif_linenum, + error) + match = Match(r'#endif\s*//\s*' + cppvar + r'(_)?\b', endif) + if match: + if match.group(1) == '_': + # Issue low severity warning for deprecated double trailing underscore + error(filename, endif_linenum, 'build/header_guard', 0, + '#endif line should be "#endif // %s"' % cppvar) + return + + # Didn't find the corresponding "//" comment. If this file does not + # contain any "//" comments at all, it could be that the compiler + # only wants "/**/" comments, look for those instead. + no_single_line_comments = True + for i in xrange(1, len(raw_lines) - 1): + line = raw_lines[i] + if Match(r'^(?:(?:\'(?:\.|[^\'])*\')|(?:"(?:\.|[^"])*")|[^\'"])*//', line): + no_single_line_comments = False + break + + if no_single_line_comments: + match = Match(r'#endif\s*/\*\s*' + cppvar + r'(_)?\s*\*/', endif) + if match: + if match.group(1) == '_': + # Low severity warning for double trailing underscore + error(filename, endif_linenum, 'build/header_guard', 0, + '#endif line should be "#endif /* %s */"' % cppvar) + return + + # Didn't find anything + error(filename, endif_linenum, 'build/header_guard', 5, + '#endif line should be "#endif // %s"' % cppvar) + + +def CheckHeaderFileIncluded(filename, include_state, error): + """Logs an error if a source file does not include its header.""" + + # Do not check test files + fileinfo = FileInfo(filename) + if Search(_TEST_FILE_SUFFIX, fileinfo.BaseName()): + return + + for ext in GetHeaderExtensions(): + basefilename = filename[0:len(filename) - len(fileinfo.Extension())] + headerfile = basefilename + '.' + ext + if not os.path.exists(headerfile): + continue + headername = FileInfo(headerfile).RepositoryName() + first_include = None + for section_list in include_state.include_list: + for f in section_list: + if headername in f[0] or f[0] in headername: + return + if not first_include: + first_include = f[1] + + error(filename, first_include, 'build/include', 5, + '%s should include its header file %s' % (fileinfo.RepositoryName(), + headername)) + + +def CheckForBadCharacters(filename, lines, error): + """Logs an error for each line containing bad characters. + + Two kinds of bad characters: + + 1. Unicode replacement characters: These indicate that either the file + contained invalid UTF-8 (likely) or Unicode replacement characters (which + it shouldn't). Note that it's possible for this to throw off line + numbering if the invalid UTF-8 occurred adjacent to a newline. + + 2. NUL bytes. These are problematic for some tools. + + Args: + filename: The name of the current file. + lines: An array of strings, each representing a line of the file. + error: The function to call with any errors found. + """ + for linenum, line in enumerate(lines): + if unicode_escape_decode('\ufffd') in line: + error(filename, linenum, 'readability/utf8', 5, + 'Line contains invalid UTF-8 (or Unicode replacement character).') + if '\0' in line: + error(filename, linenum, 'readability/nul', 5, 'Line contains NUL byte.') + + +def CheckForNewlineAtEOF(filename, lines, error): + """Logs an error if there is no newline char at the end of the file. + + Args: + filename: The name of the current file. + lines: An array of strings, each representing a line of the file. + error: The function to call with any errors found. + """ + + # The array lines() was created by adding two newlines to the + # original file (go figure), then splitting on \n. + # To verify that the file ends in \n, we just have to make sure the + # last-but-two element of lines() exists and is empty. + if len(lines) < 3 or lines[-2]: + error(filename, len(lines) - 2, 'whitespace/ending_newline', 5, + 'Could not find a newline character at the end of the file.') + + +def CheckForMultilineCommentsAndStrings(filename, clean_lines, linenum, error): + """Logs an error if we see /* ... */ or "..." that extend past one line. + + /* ... */ comments are legit inside macros, for one line. + Otherwise, we prefer // comments, so it's ok to warn about the + other. Likewise, it's ok for strings to extend across multiple + lines, as long as a line continuation character (backslash) + terminates each line. Although not currently prohibited by the C++ + style guide, it's ugly and unnecessary. We don't do well with either + in this lint program, so we warn about both. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Remove all \\ (escaped backslashes) from the line. They are OK, and the + # second (escaped) slash may trigger later \" detection erroneously. + line = line.replace('\\\\', '') + + if line.count('/*') > line.count('*/'): + error(filename, linenum, 'readability/multiline_comment', 5, + 'Complex multi-line /*...*/-style comment found. ' + 'Lint may give bogus warnings. ' + 'Consider replacing these with //-style comments, ' + 'with #if 0...#endif, ' + 'or with more clearly structured multi-line comments.') + + if (line.count('"') - line.count('\\"')) % 2: + error(filename, linenum, 'readability/multiline_string', 5, + 'Multi-line string ("...") found. This lint script doesn\'t ' + 'do well with such strings, and may give bogus warnings. ' + 'Use C++11 raw strings or concatenation instead.') + + +# (non-threadsafe name, thread-safe alternative, validation pattern) +# +# The validation pattern is used to eliminate false positives such as: +# _rand(); // false positive due to substring match. +# ->rand(); // some member function rand(). +# ACMRandom rand(seed); // some variable named rand. +# ISAACRandom rand(); // another variable named rand. +# +# Basically we require the return value of these functions to be used +# in some expression context on the same line by matching on some +# operator before the function name. This eliminates constructors and +# member function calls. +_UNSAFE_FUNC_PREFIX = r'(?:[-+*/=%^&|(<]\s*|>\s+)' +_THREADING_LIST = ( + ('asctime(', 'asctime_r(', _UNSAFE_FUNC_PREFIX + r'asctime\([^)]+\)'), + ('ctime(', 'ctime_r(', _UNSAFE_FUNC_PREFIX + r'ctime\([^)]+\)'), + ('getgrgid(', 'getgrgid_r(', _UNSAFE_FUNC_PREFIX + r'getgrgid\([^)]+\)'), + ('getgrnam(', 'getgrnam_r(', _UNSAFE_FUNC_PREFIX + r'getgrnam\([^)]+\)'), + ('getlogin(', 'getlogin_r(', _UNSAFE_FUNC_PREFIX + r'getlogin\(\)'), + ('getpwnam(', 'getpwnam_r(', _UNSAFE_FUNC_PREFIX + r'getpwnam\([^)]+\)'), + ('getpwuid(', 'getpwuid_r(', _UNSAFE_FUNC_PREFIX + r'getpwuid\([^)]+\)'), + ('gmtime(', 'gmtime_r(', _UNSAFE_FUNC_PREFIX + r'gmtime\([^)]+\)'), + ('localtime(', 'localtime_r(', _UNSAFE_FUNC_PREFIX + r'localtime\([^)]+\)'), + ('rand(', 'rand_r(', _UNSAFE_FUNC_PREFIX + r'rand\(\)'), + ('strtok(', 'strtok_r(', + _UNSAFE_FUNC_PREFIX + r'strtok\([^)]+\)'), + ('ttyname(', 'ttyname_r(', _UNSAFE_FUNC_PREFIX + r'ttyname\([^)]+\)'), + ) + + +def CheckPosixThreading(filename, clean_lines, linenum, error): + """Checks for calls to thread-unsafe functions. + + Much code has been originally written without consideration of + multi-threading. Also, engineers are relying on their old experience; + they have learned posix before threading extensions were added. These + tests guide the engineers to use thread-safe functions (when using + posix directly). + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + for single_thread_func, multithread_safe_func, pattern in _THREADING_LIST: + # Additional pattern matching check to confirm that this is the + # function we are looking for + if Search(pattern, line): + error(filename, linenum, 'runtime/threadsafe_fn', 2, + 'Consider using ' + multithread_safe_func + + '...) instead of ' + single_thread_func + + '...) for improved thread safety.') + + +def CheckVlogArguments(filename, clean_lines, linenum, error): + """Checks that VLOG() is only used for defining a logging level. + + For example, VLOG(2) is correct. VLOG(INFO), VLOG(WARNING), VLOG(ERROR), and + VLOG(FATAL) are not. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + if Search(r'\bVLOG\((INFO|ERROR|WARNING|DFATAL|FATAL)\)', line): + error(filename, linenum, 'runtime/vlog', 5, + 'VLOG() should be used with numeric verbosity level. ' + 'Use LOG() if you want symbolic severity levels.') + +# Matches invalid increment: *count++, which moves pointer instead of +# incrementing a value. +_RE_PATTERN_INVALID_INCREMENT = re.compile( + r'^\s*\*\w+(\+\+|--);') + + +def CheckInvalidIncrement(filename, clean_lines, linenum, error): + """Checks for invalid increment *count++. + + For example following function: + void increment_counter(int* count) { + *count++; + } + is invalid, because it effectively does count++, moving pointer, and should + be replaced with ++*count, (*count)++ or *count += 1. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + if _RE_PATTERN_INVALID_INCREMENT.match(line): + error(filename, linenum, 'runtime/invalid_increment', 5, + 'Changing pointer instead of value (or unused value of operator*).') + + +def IsMacroDefinition(clean_lines, linenum): + if Search(r'^#define', clean_lines[linenum]): + return True + + if linenum > 0 and Search(r'\\$', clean_lines[linenum - 1]): + return True + + return False + + +def IsForwardClassDeclaration(clean_lines, linenum): + return Match(r'^\s*(\btemplate\b)*.*class\s+\w+;\s*$', clean_lines[linenum]) + + +class _BlockInfo(object): + """Stores information about a generic block of code.""" + + def __init__(self, linenum, seen_open_brace): + self.starting_linenum = linenum + self.seen_open_brace = seen_open_brace + self.open_parentheses = 0 + self.inline_asm = _NO_ASM + self.check_namespace_indentation = False + + def CheckBegin(self, filename, clean_lines, linenum, error): + """Run checks that applies to text up to the opening brace. + + This is mostly for checking the text after the class identifier + and the "{", usually where the base class is specified. For other + blocks, there isn't much to check, so we always pass. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + pass + + def CheckEnd(self, filename, clean_lines, linenum, error): + """Run checks that applies to text after the closing brace. + + This is mostly used for checking end of namespace comments. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + pass + + def IsBlockInfo(self): + """Returns true if this block is a _BlockInfo. + + This is convenient for verifying that an object is an instance of + a _BlockInfo, but not an instance of any of the derived classes. + + Returns: + True for this class, False for derived classes. + """ + return self.__class__ == _BlockInfo + + +class _ExternCInfo(_BlockInfo): + """Stores information about an 'extern "C"' block.""" + + def __init__(self, linenum): + _BlockInfo.__init__(self, linenum, True) + + +class _ClassInfo(_BlockInfo): + """Stores information about a class.""" + + def __init__(self, name, class_or_struct, clean_lines, linenum): + _BlockInfo.__init__(self, linenum, False) + self.name = name + self.is_derived = False + self.check_namespace_indentation = True + if class_or_struct == 'struct': + self.access = 'public' + self.is_struct = True + else: + self.access = 'private' + self.is_struct = False + + # Remember initial indentation level for this class. Using raw_lines here + # instead of elided to account for leading comments. + self.class_indent = GetIndentLevel(clean_lines.raw_lines[linenum]) + + # Try to find the end of the class. This will be confused by things like: + # class A { + # } *x = { ... + # + # But it's still good enough for CheckSectionSpacing. + self.last_line = 0 + depth = 0 + for i in range(linenum, clean_lines.NumLines()): + line = clean_lines.elided[i] + depth += line.count('{') - line.count('}') + if not depth: + self.last_line = i + break + + def CheckBegin(self, filename, clean_lines, linenum, error): + # Look for a bare ':' + if Search('(^|[^:]):($|[^:])', clean_lines.elided[linenum]): + self.is_derived = True + + def CheckEnd(self, filename, clean_lines, linenum, error): + # If there is a DISALLOW macro, it should appear near the end of + # the class. + seen_last_thing_in_class = False + for i in xrange(linenum - 1, self.starting_linenum, -1): + match = Search( + r'\b(DISALLOW_COPY_AND_ASSIGN|DISALLOW_IMPLICIT_CONSTRUCTORS)\(' + + self.name + r'\)', + clean_lines.elided[i]) + if match: + if seen_last_thing_in_class: + error(filename, i, 'readability/constructors', 3, + match.group(1) + ' should be the last thing in the class') + break + + if not Match(r'^\s*$', clean_lines.elided[i]): + seen_last_thing_in_class = True + + # Check that closing brace is aligned with beginning of the class. + # Only do this if the closing brace is indented by only whitespaces. + # This means we will not check single-line class definitions. + indent = Match(r'^( *)\}', clean_lines.elided[linenum]) + if indent and len(indent.group(1)) != self.class_indent: + if self.is_struct: + parent = 'struct ' + self.name + else: + parent = 'class ' + self.name + error(filename, linenum, 'whitespace/indent', 3, + 'Closing brace should be aligned with beginning of %s' % parent) + + +class _NamespaceInfo(_BlockInfo): + """Stores information about a namespace.""" + + def __init__(self, name, linenum): + _BlockInfo.__init__(self, linenum, False) + self.name = name or '' + self.check_namespace_indentation = True + + def CheckEnd(self, filename, clean_lines, linenum, error): + """Check end of namespace comments.""" + line = clean_lines.raw_lines[linenum] + + # Check how many lines is enclosed in this namespace. Don't issue + # warning for missing namespace comments if there aren't enough + # lines. However, do apply checks if there is already an end of + # namespace comment and it's incorrect. + # + # TODO(unknown): We always want to check end of namespace comments + # if a namespace is large, but sometimes we also want to apply the + # check if a short namespace contained nontrivial things (something + # other than forward declarations). There is currently no logic on + # deciding what these nontrivial things are, so this check is + # triggered by namespace size only, which works most of the time. + if (linenum - self.starting_linenum < 10 + and not Match(r'^\s*};*\s*(//|/\*).*\bnamespace\b', line)): + return + + # Look for matching comment at end of namespace. + # + # Note that we accept C style "/* */" comments for terminating + # namespaces, so that code that terminate namespaces inside + # preprocessor macros can be cpplint clean. + # + # We also accept stuff like "// end of namespace ." with the + # period at the end. + # + # Besides these, we don't accept anything else, otherwise we might + # get false negatives when existing comment is a substring of the + # expected namespace. + if self.name: + # Named namespace + if not Match((r'^\s*};*\s*(//|/\*).*\bnamespace\s+' + + re.escape(self.name) + r'[\*/\.\\\s]*$'), + line): + error(filename, linenum, 'readability/namespace', 5, + 'Namespace should be terminated with "// namespace %s"' % + self.name) + else: + # Anonymous namespace + if not Match(r'^\s*};*\s*(//|/\*).*\bnamespace[\*/\.\\\s]*$', line): + # If "// namespace anonymous" or "// anonymous namespace (more text)", + # mention "// anonymous namespace" as an acceptable form + if Match(r'^\s*}.*\b(namespace anonymous|anonymous namespace)\b', line): + error(filename, linenum, 'readability/namespace', 5, + 'Anonymous namespace should be terminated with "// namespace"' + ' or "// anonymous namespace"') + else: + error(filename, linenum, 'readability/namespace', 5, + 'Anonymous namespace should be terminated with "// namespace"') + + +class _PreprocessorInfo(object): + """Stores checkpoints of nesting stacks when #if/#else is seen.""" + + def __init__(self, stack_before_if): + # The entire nesting stack before #if + self.stack_before_if = stack_before_if + + # The entire nesting stack up to #else + self.stack_before_else = [] + + # Whether we have already seen #else or #elif + self.seen_else = False + + +class NestingState(object): + """Holds states related to parsing braces.""" + + def __init__(self): + # Stack for tracking all braces. An object is pushed whenever we + # see a "{", and popped when we see a "}". Only 3 types of + # objects are possible: + # - _ClassInfo: a class or struct. + # - _NamespaceInfo: a namespace. + # - _BlockInfo: some other type of block. + self.stack = [] + + # Top of the previous stack before each Update(). + # + # Because the nesting_stack is updated at the end of each line, we + # had to do some convoluted checks to find out what is the current + # scope at the beginning of the line. This check is simplified by + # saving the previous top of nesting stack. + # + # We could save the full stack, but we only need the top. Copying + # the full nesting stack would slow down cpplint by ~10%. + self.previous_stack_top = [] + + # Stack of _PreprocessorInfo objects. + self.pp_stack = [] + + def SeenOpenBrace(self): + """Check if we have seen the opening brace for the innermost block. + + Returns: + True if we have seen the opening brace, False if the innermost + block is still expecting an opening brace. + """ + return (not self.stack) or self.stack[-1].seen_open_brace + + def InNamespaceBody(self): + """Check if we are currently one level inside a namespace body. + + Returns: + True if top of the stack is a namespace block, False otherwise. + """ + return self.stack and isinstance(self.stack[-1], _NamespaceInfo) + + def InExternC(self): + """Check if we are currently one level inside an 'extern "C"' block. + + Returns: + True if top of the stack is an extern block, False otherwise. + """ + return self.stack and isinstance(self.stack[-1], _ExternCInfo) + + def InClassDeclaration(self): + """Check if we are currently one level inside a class or struct declaration. + + Returns: + True if top of the stack is a class/struct, False otherwise. + """ + return self.stack and isinstance(self.stack[-1], _ClassInfo) + + def InAsmBlock(self): + """Check if we are currently one level inside an inline ASM block. + + Returns: + True if the top of the stack is a block containing inline ASM. + """ + return self.stack and self.stack[-1].inline_asm != _NO_ASM + + def InTemplateArgumentList(self, clean_lines, linenum, pos): + """Check if current position is inside template argument list. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + pos: position just after the suspected template argument. + Returns: + True if (linenum, pos) is inside template arguments. + """ + while linenum < clean_lines.NumLines(): + # Find the earliest character that might indicate a template argument + line = clean_lines.elided[linenum] + match = Match(r'^[^{};=\[\]\.<>]*(.)', line[pos:]) + if not match: + linenum += 1 + pos = 0 + continue + token = match.group(1) + pos += len(match.group(0)) + + # These things do not look like template argument list: + # class Suspect { + # class Suspect x; } + if token in ('{', '}', ';'): return False + + # These things look like template argument list: + # template + # template + # template + # template + if token in ('>', '=', '[', ']', '.'): return True + + # Check if token is an unmatched '<'. + # If not, move on to the next character. + if token != '<': + pos += 1 + if pos >= len(line): + linenum += 1 + pos = 0 + continue + + # We can't be sure if we just find a single '<', and need to + # find the matching '>'. + (_, end_line, end_pos) = CloseExpression(clean_lines, linenum, pos - 1) + if end_pos < 0: + # Not sure if template argument list or syntax error in file + return False + linenum = end_line + pos = end_pos + return False + + def UpdatePreprocessor(self, line): + """Update preprocessor stack. + + We need to handle preprocessors due to classes like this: + #ifdef SWIG + struct ResultDetailsPageElementExtensionPoint { + #else + struct ResultDetailsPageElementExtensionPoint : public Extension { + #endif + + We make the following assumptions (good enough for most files): + - Preprocessor condition evaluates to true from #if up to first + #else/#elif/#endif. + + - Preprocessor condition evaluates to false from #else/#elif up + to #endif. We still perform lint checks on these lines, but + these do not affect nesting stack. + + Args: + line: current line to check. + """ + if Match(r'^\s*#\s*(if|ifdef|ifndef)\b', line): + # Beginning of #if block, save the nesting stack here. The saved + # stack will allow us to restore the parsing state in the #else case. + self.pp_stack.append(_PreprocessorInfo(copy.deepcopy(self.stack))) + elif Match(r'^\s*#\s*(else|elif)\b', line): + # Beginning of #else block + if self.pp_stack: + if not self.pp_stack[-1].seen_else: + # This is the first #else or #elif block. Remember the + # whole nesting stack up to this point. This is what we + # keep after the #endif. + self.pp_stack[-1].seen_else = True + self.pp_stack[-1].stack_before_else = copy.deepcopy(self.stack) + + # Restore the stack to how it was before the #if + self.stack = copy.deepcopy(self.pp_stack[-1].stack_before_if) + else: + # TODO(unknown): unexpected #else, issue warning? + pass + elif Match(r'^\s*#\s*endif\b', line): + # End of #if or #else blocks. + if self.pp_stack: + # If we saw an #else, we will need to restore the nesting + # stack to its former state before the #else, otherwise we + # will just continue from where we left off. + if self.pp_stack[-1].seen_else: + # Here we can just use a shallow copy since we are the last + # reference to it. + self.stack = self.pp_stack[-1].stack_before_else + # Drop the corresponding #if + self.pp_stack.pop() + else: + # TODO(unknown): unexpected #endif, issue warning? + pass + + # TODO(unknown): Update() is too long, but we will refactor later. + def Update(self, filename, clean_lines, linenum, error): + """Update nesting state with current line. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Remember top of the previous nesting stack. + # + # The stack is always pushed/popped and not modified in place, so + # we can just do a shallow copy instead of copy.deepcopy. Using + # deepcopy would slow down cpplint by ~28%. + if self.stack: + self.previous_stack_top = self.stack[-1] + else: + self.previous_stack_top = None + + # Update pp_stack + self.UpdatePreprocessor(line) + + # Count parentheses. This is to avoid adding struct arguments to + # the nesting stack. + if self.stack: + inner_block = self.stack[-1] + depth_change = line.count('(') - line.count(')') + inner_block.open_parentheses += depth_change + + # Also check if we are starting or ending an inline assembly block. + if inner_block.inline_asm in (_NO_ASM, _END_ASM): + if (depth_change != 0 and + inner_block.open_parentheses == 1 and + _MATCH_ASM.match(line)): + # Enter assembly block + inner_block.inline_asm = _INSIDE_ASM + else: + # Not entering assembly block. If previous line was _END_ASM, + # we will now shift to _NO_ASM state. + inner_block.inline_asm = _NO_ASM + elif (inner_block.inline_asm == _INSIDE_ASM and + inner_block.open_parentheses == 0): + # Exit assembly block + inner_block.inline_asm = _END_ASM + + # Consume namespace declaration at the beginning of the line. Do + # this in a loop so that we catch same line declarations like this: + # namespace proto2 { namespace bridge { class MessageSet; } } + while True: + # Match start of namespace. The "\b\s*" below catches namespace + # declarations even if it weren't followed by a whitespace, this + # is so that we don't confuse our namespace checker. The + # missing spaces will be flagged by CheckSpacing. + namespace_decl_match = Match(r'^\s*namespace\b\s*([:\w]+)?(.*)$', line) + if not namespace_decl_match: + break + + new_namespace = _NamespaceInfo(namespace_decl_match.group(1), linenum) + self.stack.append(new_namespace) + + line = namespace_decl_match.group(2) + if line.find('{') != -1: + new_namespace.seen_open_brace = True + line = line[line.find('{') + 1:] + + # Look for a class declaration in whatever is left of the line + # after parsing namespaces. The regexp accounts for decorated classes + # such as in: + # class LOCKABLE API Object { + # }; + class_decl_match = Match( + r'^(\s*(?:template\s*<[\w\s<>,:=]*>\s*)?' + r'(class|struct)\s+(?:[A-Z_]+\s+)*(\w+(?:::\w+)*))' + r'(.*)$', line) + if (class_decl_match and + (not self.stack or self.stack[-1].open_parentheses == 0)): + # We do not want to accept classes that are actually template arguments: + # template , + # template class Ignore3> + # void Function() {}; + # + # To avoid template argument cases, we scan forward and look for + # an unmatched '>'. If we see one, assume we are inside a + # template argument list. + end_declaration = len(class_decl_match.group(1)) + if not self.InTemplateArgumentList(clean_lines, linenum, end_declaration): + self.stack.append(_ClassInfo( + class_decl_match.group(3), class_decl_match.group(2), + clean_lines, linenum)) + line = class_decl_match.group(4) + + # If we have not yet seen the opening brace for the innermost block, + # run checks here. + if not self.SeenOpenBrace(): + self.stack[-1].CheckBegin(filename, clean_lines, linenum, error) + + # Update access control if we are inside a class/struct + if self.stack and isinstance(self.stack[-1], _ClassInfo): + classinfo = self.stack[-1] + access_match = Match( + r'^(.*)\b(public|private|protected|signals)(\s+(?:slots\s*)?)?' + r':(?:[^:]|$)', + line) + if access_match: + classinfo.access = access_match.group(2) + + # Check that access keywords are indented +1 space. Skip this + # check if the keywords are not preceded by whitespaces. + indent = access_match.group(1) + if (len(indent) != classinfo.class_indent + 1 and + Match(r'^\s*$', indent)): + if classinfo.is_struct: + parent = 'struct ' + classinfo.name + else: + parent = 'class ' + classinfo.name + slots = '' + if access_match.group(3): + slots = access_match.group(3) + error(filename, linenum, 'whitespace/indent', 3, + '%s%s: should be indented +1 space inside %s' % ( + access_match.group(2), slots, parent)) + + # Consume braces or semicolons from what's left of the line + while True: + # Match first brace, semicolon, or closed parenthesis. + matched = Match(r'^[^{;)}]*([{;)}])(.*)$', line) + if not matched: + break + + token = matched.group(1) + if token == '{': + # If namespace or class hasn't seen a opening brace yet, mark + # namespace/class head as complete. Push a new block onto the + # stack otherwise. + if not self.SeenOpenBrace(): + self.stack[-1].seen_open_brace = True + elif Match(r'^extern\s*"[^"]*"\s*\{', line): + self.stack.append(_ExternCInfo(linenum)) + else: + self.stack.append(_BlockInfo(linenum, True)) + if _MATCH_ASM.match(line): + self.stack[-1].inline_asm = _BLOCK_ASM + + elif token == ';' or token == ')': + # If we haven't seen an opening brace yet, but we already saw + # a semicolon, this is probably a forward declaration. Pop + # the stack for these. + # + # Similarly, if we haven't seen an opening brace yet, but we + # already saw a closing parenthesis, then these are probably + # function arguments with extra "class" or "struct" keywords. + # Also pop these stack for these. + if not self.SeenOpenBrace(): + self.stack.pop() + else: # token == '}' + # Perform end of block checks and pop the stack. + if self.stack: + self.stack[-1].CheckEnd(filename, clean_lines, linenum, error) + self.stack.pop() + line = matched.group(2) + + def InnermostClass(self): + """Get class info on the top of the stack. + + Returns: + A _ClassInfo object if we are inside a class, or None otherwise. + """ + for i in range(len(self.stack), 0, -1): + classinfo = self.stack[i - 1] + if isinstance(classinfo, _ClassInfo): + return classinfo + return None + + def CheckCompletedBlocks(self, filename, error): + """Checks that all classes and namespaces have been completely parsed. + + Call this when all lines in a file have been processed. + Args: + filename: The name of the current file. + error: The function to call with any errors found. + """ + # Note: This test can result in false positives if #ifdef constructs + # get in the way of brace matching. See the testBuildClass test in + # cpplint_unittest.py for an example of this. + for obj in self.stack: + if isinstance(obj, _ClassInfo): + error(filename, obj.starting_linenum, 'build/class', 5, + 'Failed to find complete declaration of class %s' % + obj.name) + elif isinstance(obj, _NamespaceInfo): + error(filename, obj.starting_linenum, 'build/namespaces', 5, + 'Failed to find complete declaration of namespace %s' % + obj.name) + + +def CheckForNonStandardConstructs(filename, clean_lines, linenum, + nesting_state, error): + r"""Logs an error if we see certain non-ANSI constructs ignored by gcc-2. + + Complain about several constructs which gcc-2 accepts, but which are + not standard C++. Warning about these in lint is one way to ease the + transition to new compilers. + - put storage class first (e.g. "static const" instead of "const static"). + - "%lld" instead of %qd" in printf-type functions. + - "%1$d" is non-standard in printf-type functions. + - "\%" is an undefined character escape sequence. + - text after #endif is not allowed. + - invalid inner-style forward declaration. + - >? and ?= and )\?=?\s*(\w+|[+-]?\d+)(\.\d*)?', + line): + error(filename, linenum, 'build/deprecated', 3, + '>? and ))?' + # r'\s*const\s*' + type_name + '\s*&\s*\w+\s*;' + error(filename, linenum, 'runtime/member_string_references', 2, + 'const string& members are dangerous. It is much better to use ' + 'alternatives, such as pointers or simple constants.') + + # Everything else in this function operates on class declarations. + # Return early if the top of the nesting stack is not a class, or if + # the class head is not completed yet. + classinfo = nesting_state.InnermostClass() + if not classinfo or not classinfo.seen_open_brace: + return + + # The class may have been declared with namespace or classname qualifiers. + # The constructor and destructor will not have those qualifiers. + base_classname = classinfo.name.split('::')[-1] + + # Look for single-argument constructors that aren't marked explicit. + # Technically a valid construct, but against style. + explicit_constructor_match = Match( + r'\s+(?:inline\s+)?(explicit\s+)?(?:inline\s+)?%s\s*' + r'\(((?:[^()]|\([^()]*\))*)\)' + % re.escape(base_classname), + line) + + if explicit_constructor_match: + is_marked_explicit = explicit_constructor_match.group(1) + + if not explicit_constructor_match.group(2): + constructor_args = [] + else: + constructor_args = explicit_constructor_match.group(2).split(',') + + # collapse arguments so that commas in template parameter lists and function + # argument parameter lists don't split arguments in two + i = 0 + while i < len(constructor_args): + constructor_arg = constructor_args[i] + while (constructor_arg.count('<') > constructor_arg.count('>') or + constructor_arg.count('(') > constructor_arg.count(')')): + constructor_arg += ',' + constructor_args[i + 1] + del constructor_args[i + 1] + constructor_args[i] = constructor_arg + i += 1 + + variadic_args = [arg for arg in constructor_args if '&&...' in arg] + defaulted_args = [arg for arg in constructor_args if '=' in arg] + noarg_constructor = (not constructor_args or # empty arg list + # 'void' arg specifier + (len(constructor_args) == 1 and + constructor_args[0].strip() == 'void')) + onearg_constructor = ((len(constructor_args) == 1 and # exactly one arg + not noarg_constructor) or + # all but at most one arg defaulted + (len(constructor_args) >= 1 and + not noarg_constructor and + len(defaulted_args) >= len(constructor_args) - 1) or + # variadic arguments with zero or one argument + (len(constructor_args) <= 2 and + len(variadic_args) >= 1)) + initializer_list_constructor = bool( + onearg_constructor and + Search(r'\bstd\s*::\s*initializer_list\b', constructor_args[0])) + copy_constructor = bool( + onearg_constructor and + Match(r'(const\s+)?%s(\s*<[^>]*>)?(\s+const)?\s*(?:<\w+>\s*)?&' + % re.escape(base_classname), constructor_args[0].strip())) + + if (not is_marked_explicit and + onearg_constructor and + not initializer_list_constructor and + not copy_constructor): + if defaulted_args or variadic_args: + error(filename, linenum, 'runtime/explicit', 5, + 'Constructors callable with one argument ' + 'should be marked explicit.') + else: + error(filename, linenum, 'runtime/explicit', 5, + 'Single-parameter constructors should be marked explicit.') + elif is_marked_explicit and not onearg_constructor: + if noarg_constructor: + error(filename, linenum, 'runtime/explicit', 5, + 'Zero-parameter constructors should not be marked explicit.') + + +def CheckSpacingForFunctionCall(filename, clean_lines, linenum, error): + """Checks for the correctness of various spacing around function calls. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Since function calls often occur inside if/for/while/switch + # expressions - which have their own, more liberal conventions - we + # first see if we should be looking inside such an expression for a + # function call, to which we can apply more strict standards. + fncall = line # if there's no control flow construct, look at whole line + for pattern in (r'\bif\s*\((.*)\)\s*{', + r'\bfor\s*\((.*)\)\s*{', + r'\bwhile\s*\((.*)\)\s*[{;]', + r'\bswitch\s*\((.*)\)\s*{'): + match = Search(pattern, line) + if match: + fncall = match.group(1) # look inside the parens for function calls + break + + # Except in if/for/while/switch, there should never be space + # immediately inside parens (eg "f( 3, 4 )"). We make an exception + # for nested parens ( (a+b) + c ). Likewise, there should never be + # a space before a ( when it's a function argument. I assume it's a + # function argument when the char before the whitespace is legal in + # a function name (alnum + _) and we're not starting a macro. Also ignore + # pointers and references to arrays and functions coz they're too tricky: + # we use a very simple way to recognize these: + # " (something)(maybe-something)" or + # " (something)(maybe-something," or + # " (something)[something]" + # Note that we assume the contents of [] to be short enough that + # they'll never need to wrap. + if ( # Ignore control structures. + not Search(r'\b(if|for|while|switch|return|new|delete|catch|sizeof)\b', + fncall) and + # Ignore pointers/references to functions. + not Search(r' \([^)]+\)\([^)]*(\)|,$)', fncall) and + # Ignore pointers/references to arrays. + not Search(r' \([^)]+\)\[[^\]]+\]', fncall)): + if Search(r'\w\s*\(\s(?!\s*\\$)', fncall): # a ( used for a fn call + error(filename, linenum, 'whitespace/parens', 4, + 'Extra space after ( in function call') + elif Search(r'\(\s+(?!(\s*\\)|\()', fncall): + error(filename, linenum, 'whitespace/parens', 2, + 'Extra space after (') + if (Search(r'\w\s+\(', fncall) and + not Search(r'_{0,2}asm_{0,2}\s+_{0,2}volatile_{0,2}\s+\(', fncall) and + not Search(r'#\s*define|typedef|using\s+\w+\s*=', fncall) and + not Search(r'\w\s+\((\w+::)*\*\w+\)\(', fncall) and + not Search(r'\bcase\s+\(', fncall)): + # TODO(unknown): Space after an operator function seem to be a common + # error, silence those for now by restricting them to highest verbosity. + if Search(r'\boperator_*\b', line): + error(filename, linenum, 'whitespace/parens', 0, + 'Extra space before ( in function call') + else: + error(filename, linenum, 'whitespace/parens', 4, + 'Extra space before ( in function call') + # If the ) is followed only by a newline or a { + newline, assume it's + # part of a control statement (if/while/etc), and don't complain + if Search(r'[^)]\s+\)\s*[^{\s]', fncall): + # If the closing parenthesis is preceded by only whitespaces, + # try to give a more descriptive error message. + if Search(r'^\s+\)', fncall): + error(filename, linenum, 'whitespace/parens', 2, + 'Closing ) should be moved to the previous line') + else: + error(filename, linenum, 'whitespace/parens', 2, + 'Extra space before )') + + +def IsBlankLine(line): + """Returns true if the given line is blank. + + We consider a line to be blank if the line is empty or consists of + only white spaces. + + Args: + line: A line of a string. + + Returns: + True, if the given line is blank. + """ + return not line or line.isspace() + + +def CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line, + error): + is_namespace_indent_item = ( + len(nesting_state.stack) > 1 and + nesting_state.stack[-1].check_namespace_indentation and + isinstance(nesting_state.previous_stack_top, _NamespaceInfo) and + nesting_state.previous_stack_top == nesting_state.stack[-2]) + + if ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, + clean_lines.elided, line): + CheckItemIndentationInNamespace(filename, clean_lines.elided, + line, error) + + +def CheckForFunctionLengths(filename, clean_lines, linenum, + function_state, error): + """Reports for long function bodies. + + For an overview why this is done, see: + https://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Write_Short_Functions + + Uses a simplistic algorithm assuming other style guidelines + (especially spacing) are followed. + Only checks unindented functions, so class members are unchecked. + Trivial bodies are unchecked, so constructors with huge initializer lists + may be missed. + Blank/comment lines are not counted so as to avoid encouraging the removal + of vertical space and comments just to get through a lint check. + NOLINT *on the last line of a function* disables this check. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + function_state: Current function name and lines in body so far. + error: The function to call with any errors found. + """ + lines = clean_lines.lines + line = lines[linenum] + joined_line = '' + + starting_func = False + regexp = r'(\w(\w|::|\*|\&|\s)*)\(' # decls * & space::name( ... + match_result = Match(regexp, line) + if match_result: + # If the name is all caps and underscores, figure it's a macro and + # ignore it, unless it's TEST or TEST_F. + function_name = match_result.group(1).split()[-1] + if function_name == 'TEST' or function_name == 'TEST_F' or ( + not Match(r'[A-Z_]+$', function_name)): + starting_func = True + + if starting_func: + body_found = False + for start_linenum in range(linenum, clean_lines.NumLines()): + start_line = lines[start_linenum] + joined_line += ' ' + start_line.lstrip() + if Search(r'(;|})', start_line): # Declarations and trivial functions + body_found = True + break # ... ignore + elif Search(r'{', start_line): + body_found = True + function = Search(r'((\w|:)*)\(', line).group(1) + if Match(r'TEST', function): # Handle TEST... macros + parameter_regexp = Search(r'(\(.*\))', joined_line) + if parameter_regexp: # Ignore bad syntax + function += parameter_regexp.group(1) + else: + function += '()' + function_state.Begin(function) + break + if not body_found: + # No body for the function (or evidence of a non-function) was found. + error(filename, linenum, 'readability/fn_size', 5, + 'Lint failed to find start of function body.') + elif Match(r'^\}\s*$', line): # function end + function_state.Check(error, filename, linenum) + function_state.End() + elif not Match(r'^\s*$', line): + function_state.Count() # Count non-blank/non-comment lines. + + +_RE_PATTERN_TODO = re.compile(r'^//(\s*)TODO(\(.+?\))?:?(\s|$)?') + + +def CheckComment(line, filename, linenum, next_line_start, error): + """Checks for common mistakes in comments. + + Args: + line: The line in question. + filename: The name of the current file. + linenum: The number of the line to check. + next_line_start: The first non-whitespace column of the next line. + error: The function to call with any errors found. + """ + commentpos = line.find('//') + if commentpos != -1: + # Check if the // may be in quotes. If so, ignore it + if re.sub(r'\\.', '', line[0:commentpos]).count('"') % 2 == 0: + # Allow one space for new scopes, two spaces otherwise: + if (not (Match(r'^.*{ *//', line) and next_line_start == commentpos) and + ((commentpos >= 1 and + line[commentpos-1] not in string.whitespace) or + (commentpos >= 2 and + line[commentpos-2] not in string.whitespace))): + error(filename, linenum, 'whitespace/comments', 2, + 'At least two spaces is best between code and comments') + + # Checks for common mistakes in TODO comments. + comment = line[commentpos:] + match = _RE_PATTERN_TODO.match(comment) + if match: + # One whitespace is correct; zero whitespace is handled elsewhere. + leading_whitespace = match.group(1) + if len(leading_whitespace) > 1: + error(filename, linenum, 'whitespace/todo', 2, + 'Too many spaces before TODO') + + username = match.group(2) + if not username: + error(filename, linenum, 'readability/todo', 2, + 'Missing username in TODO; it should look like ' + '"// TODO(my_username): Stuff."') + + middle_whitespace = match.group(3) + # Comparisons made explicit for correctness -- pylint: disable=g-explicit-bool-comparison + if middle_whitespace != ' ' and middle_whitespace != '': + error(filename, linenum, 'whitespace/todo', 2, + 'TODO(my_username) should be followed by a space') + + # If the comment contains an alphanumeric character, there + # should be a space somewhere between it and the // unless + # it's a /// or //! Doxygen comment. + if (Match(r'//[^ ]*\w', comment) and + not Match(r'(///|//\!)(\s+|$)', comment)): + error(filename, linenum, 'whitespace/comments', 4, + 'Should have a space between // and comment') + + +def CheckAccess(filename, clean_lines, linenum, nesting_state, error): + """Checks for improper use of DISALLOW* macros. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + nesting_state: A NestingState instance which maintains information about + the current stack of nested blocks being parsed. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] # get rid of comments and strings + + matched = Match((r'\s*(DISALLOW_COPY_AND_ASSIGN|' + r'DISALLOW_IMPLICIT_CONSTRUCTORS)'), line) + if not matched: + return + if nesting_state.stack and isinstance(nesting_state.stack[-1], _ClassInfo): + if nesting_state.stack[-1].access != 'private': + error(filename, linenum, 'readability/constructors', 3, + '%s must be in the private: section' % matched.group(1)) + + else: + # Found DISALLOW* macro outside a class declaration, or perhaps it + # was used inside a function when it should have been part of the + # class declaration. We could issue a warning here, but it + # probably resulted in a compiler error already. + pass + + +def CheckSpacing(filename, clean_lines, linenum, nesting_state, error): + """Checks for the correctness of various spacing issues in the code. + + Things we check for: spaces around operators, spaces after + if/for/while/switch, no spaces around parens in function calls, two + spaces between code and comment, don't start a block with a blank + line, don't end a function with a blank line, don't add a blank line + after public/protected/private, don't have too many blank lines in a row. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + nesting_state: A NestingState instance which maintains information about + the current stack of nested blocks being parsed. + error: The function to call with any errors found. + """ + + # Don't use "elided" lines here, otherwise we can't check commented lines. + # Don't want to use "raw" either, because we don't want to check inside C++11 + # raw strings, + raw = clean_lines.lines_without_raw_strings + line = raw[linenum] + + # Before nixing comments, check if the line is blank for no good + # reason. This includes the first line after a block is opened, and + # blank lines at the end of a function (ie, right before a line like '}' + # + # Skip all the blank line checks if we are immediately inside a + # namespace body. In other words, don't issue blank line warnings + # for this block: + # namespace { + # + # } + # + # A warning about missing end of namespace comments will be issued instead. + # + # Also skip blank line checks for 'extern "C"' blocks, which are formatted + # like namespaces. + if (IsBlankLine(line) and + not nesting_state.InNamespaceBody() and + not nesting_state.InExternC()): + elided = clean_lines.elided + prev_line = elided[linenum - 1] + prevbrace = prev_line.rfind('{') + # TODO(unknown): Don't complain if line before blank line, and line after, + # both start with alnums and are indented the same amount. + # This ignores whitespace at the start of a namespace block + # because those are not usually indented. + if prevbrace != -1 and prev_line[prevbrace:].find('}') == -1: + # OK, we have a blank line at the start of a code block. Before we + # complain, we check if it is an exception to the rule: The previous + # non-empty line has the parameters of a function header that are indented + # 4 spaces (because they did not fit in a 80 column line when placed on + # the same line as the function name). We also check for the case where + # the previous line is indented 6 spaces, which may happen when the + # initializers of a constructor do not fit into a 80 column line. + exception = False + if Match(r' {6}\w', prev_line): # Initializer list? + # We are looking for the opening column of initializer list, which + # should be indented 4 spaces to cause 6 space indentation afterwards. + search_position = linenum-2 + while (search_position >= 0 + and Match(r' {6}\w', elided[search_position])): + search_position -= 1 + exception = (search_position >= 0 + and elided[search_position][:5] == ' :') + else: + # Search for the function arguments or an initializer list. We use a + # simple heuristic here: If the line is indented 4 spaces; and we have a + # closing paren, without the opening paren, followed by an opening brace + # or colon (for initializer lists) we assume that it is the last line of + # a function header. If we have a colon indented 4 spaces, it is an + # initializer list. + exception = (Match(r' {4}\w[^\(]*\)\s*(const\s*)?(\{\s*$|:)', + prev_line) + or Match(r' {4}:', prev_line)) + + if not exception: + error(filename, linenum, 'whitespace/blank_line', 2, + 'Redundant blank line at the start of a code block ' + 'should be deleted.') + # Ignore blank lines at the end of a block in a long if-else + # chain, like this: + # if (condition1) { + # // Something followed by a blank line + # + # } else if (condition2) { + # // Something else + # } + if linenum + 1 < clean_lines.NumLines(): + next_line = raw[linenum + 1] + if (next_line + and Match(r'\s*}', next_line) + and next_line.find('} else ') == -1): + error(filename, linenum, 'whitespace/blank_line', 3, + 'Redundant blank line at the end of a code block ' + 'should be deleted.') + + matched = Match(r'\s*(public|protected|private):', prev_line) + if matched: + error(filename, linenum, 'whitespace/blank_line', 3, + 'Do not leave a blank line after "%s:"' % matched.group(1)) + + # Next, check comments + next_line_start = 0 + if linenum + 1 < clean_lines.NumLines(): + next_line = raw[linenum + 1] + next_line_start = len(next_line) - len(next_line.lstrip()) + CheckComment(line, filename, linenum, next_line_start, error) + + # get rid of comments and strings + line = clean_lines.elided[linenum] + + # You shouldn't have spaces before your brackets, except maybe after + # 'delete []' or 'return []() {};' + if Search(r'\w\s+\[', line) and not Search(r'(?:delete|return)\s+\[', line): + error(filename, linenum, 'whitespace/braces', 5, + 'Extra space before [') + + # In range-based for, we wanted spaces before and after the colon, but + # not around "::" tokens that might appear. + if (Search(r'for *\(.*[^:]:[^: ]', line) or + Search(r'for *\(.*[^: ]:[^:]', line)): + error(filename, linenum, 'whitespace/forcolon', 2, + 'Missing space around colon in range-based for loop') + + +def CheckOperatorSpacing(filename, clean_lines, linenum, error): + """Checks for horizontal spacing around operators. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Don't try to do spacing checks for operator methods. Do this by + # replacing the troublesome characters with something else, + # preserving column position for all other characters. + # + # The replacement is done repeatedly to avoid false positives from + # operators that call operators. + while True: + match = Match(r'^(.*\boperator\b)(\S+)(\s*\(.*)$', line) + if match: + line = match.group(1) + ('_' * len(match.group(2))) + match.group(3) + else: + break + + # We allow no-spaces around = within an if: "if ( (a=Foo()) == 0 )". + # Otherwise not. Note we only check for non-spaces on *both* sides; + # sometimes people put non-spaces on one side when aligning ='s among + # many lines (not that this is behavior that I approve of...) + if ((Search(r'[\w.]=', line) or + Search(r'=[\w.]', line)) + and not Search(r'\b(if|while|for) ', line) + # Operators taken from [lex.operators] in C++11 standard. + and not Search(r'(>=|<=|==|!=|&=|\^=|\|=|\+=|\*=|\/=|\%=)', line) + and not Search(r'operator=', line)): + error(filename, linenum, 'whitespace/operators', 4, + 'Missing spaces around =') + + # It's ok not to have spaces around binary operators like + - * /, but if + # there's too little whitespace, we get concerned. It's hard to tell, + # though, so we punt on this one for now. TODO. + + # You should always have whitespace around binary operators. + # + # Check <= and >= first to avoid false positives with < and >, then + # check non-include lines for spacing around < and >. + # + # If the operator is followed by a comma, assume it's be used in a + # macro context and don't do any checks. This avoids false + # positives. + # + # Note that && is not included here. This is because there are too + # many false positives due to RValue references. + match = Search(r'[^<>=!\s](==|!=|<=|>=|\|\|)[^<>=!\s,;\)]', line) + if match: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around %s' % match.group(1)) + elif not Match(r'#.*include', line): + # Look for < that is not surrounded by spaces. This is only + # triggered if both sides are missing spaces, even though + # technically should should flag if at least one side is missing a + # space. This is done to avoid some false positives with shifts. + match = Match(r'^(.*[^\s<])<[^\s=<,]', line) + if match: + (_, _, end_pos) = CloseExpression( + clean_lines, linenum, len(match.group(1))) + if end_pos <= -1: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around <') + + # Look for > that is not surrounded by spaces. Similar to the + # above, we only trigger if both sides are missing spaces to avoid + # false positives with shifts. + match = Match(r'^(.*[^-\s>])>[^\s=>,]', line) + if match: + (_, _, start_pos) = ReverseCloseExpression( + clean_lines, linenum, len(match.group(1))) + if start_pos <= -1: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around >') + + # We allow no-spaces around << when used like this: 10<<20, but + # not otherwise (particularly, not when used as streams) + # + # We also allow operators following an opening parenthesis, since + # those tend to be macros that deal with operators. + match = Search(r'(operator|[^\s(<])(?:L|UL|LL|ULL|l|ul|ll|ull)?<<([^\s,=<])', line) + if (match and not (match.group(1).isdigit() and match.group(2).isdigit()) and + not (match.group(1) == 'operator' and match.group(2) == ';')): + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around <<') + + # We allow no-spaces around >> for almost anything. This is because + # C++11 allows ">>" to close nested templates, which accounts for + # most cases when ">>" is not followed by a space. + # + # We still warn on ">>" followed by alpha character, because that is + # likely due to ">>" being used for right shifts, e.g.: + # value >> alpha + # + # When ">>" is used to close templates, the alphanumeric letter that + # follows would be part of an identifier, and there should still be + # a space separating the template type and the identifier. + # type> alpha + match = Search(r'>>[a-zA-Z_]', line) + if match: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around >>') + + # There shouldn't be space around unary operators + match = Search(r'(!\s|~\s|[\s]--[\s;]|[\s]\+\+[\s;])', line) + if match: + error(filename, linenum, 'whitespace/operators', 4, + 'Extra space for operator %s' % match.group(1)) + + +def CheckParenthesisSpacing(filename, clean_lines, linenum, error): + """Checks for horizontal spacing around parentheses. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # No spaces after an if, while, switch, or for + match = Search(r' (if\(|for\(|while\(|switch\()', line) + if match: + error(filename, linenum, 'whitespace/parens', 5, + 'Missing space before ( in %s' % match.group(1)) + + # For if/for/while/switch, the left and right parens should be + # consistent about how many spaces are inside the parens, and + # there should either be zero or one spaces inside the parens. + # We don't want: "if ( foo)" or "if ( foo )". + # Exception: "for ( ; foo; bar)" and "for (foo; bar; )" are allowed. + match = Search(r'\b(if|for|while|switch)\s*' + r'\(([ ]*)(.).*[^ ]+([ ]*)\)\s*{\s*$', + line) + if match: + if len(match.group(2)) != len(match.group(4)): + if not (match.group(3) == ';' and + len(match.group(2)) == 1 + len(match.group(4)) or + not match.group(2) and Search(r'\bfor\s*\(.*; \)', line)): + error(filename, linenum, 'whitespace/parens', 5, + 'Mismatching spaces inside () in %s' % match.group(1)) + if len(match.group(2)) not in [0, 1]: + error(filename, linenum, 'whitespace/parens', 5, + 'Should have zero or one spaces inside ( and ) in %s' % + match.group(1)) + + +def CheckCommaSpacing(filename, clean_lines, linenum, error): + """Checks for horizontal spacing near commas and semicolons. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + raw = clean_lines.lines_without_raw_strings + line = clean_lines.elided[linenum] + + # You should always have a space after a comma (either as fn arg or operator) + # + # This does not apply when the non-space character following the + # comma is another comma, since the only time when that happens is + # for empty macro arguments. + # + # We run this check in two passes: first pass on elided lines to + # verify that lines contain missing whitespaces, second pass on raw + # lines to confirm that those missing whitespaces are not due to + # elided comments. + if (Search(r',[^,\s]', ReplaceAll(r'\boperator\s*,\s*\(', 'F(', line)) and + Search(r',[^,\s]', raw[linenum])): + error(filename, linenum, 'whitespace/comma', 3, + 'Missing space after ,') + + # You should always have a space after a semicolon + # except for few corner cases + # TODO(unknown): clarify if 'if (1) { return 1;}' is requires one more + # space after ; + if Search(r';[^\s};\\)/]', line): + error(filename, linenum, 'whitespace/semicolon', 3, + 'Missing space after ;') + + +def _IsType(clean_lines, nesting_state, expr): + """Check if expression looks like a type name, returns true if so. + + Args: + clean_lines: A CleansedLines instance containing the file. + nesting_state: A NestingState instance which maintains information about + the current stack of nested blocks being parsed. + expr: The expression to check. + Returns: + True, if token looks like a type. + """ + # Keep only the last token in the expression + last_word = Match(r'^.*(\b\S+)$', expr) + if last_word: + token = last_word.group(1) + else: + token = expr + + # Match native types and stdint types + if _TYPES.match(token): + return True + + # Try a bit harder to match templated types. Walk up the nesting + # stack until we find something that resembles a typename + # declaration for what we are looking for. + typename_pattern = (r'\b(?:typename|class|struct)\s+' + re.escape(token) + + r'\b') + block_index = len(nesting_state.stack) - 1 + while block_index >= 0: + if isinstance(nesting_state.stack[block_index], _NamespaceInfo): + return False + + # Found where the opening brace is. We want to scan from this + # line up to the beginning of the function, minus a few lines. + # template + # class C + # : public ... { // start scanning here + last_line = nesting_state.stack[block_index].starting_linenum + + next_block_start = 0 + if block_index > 0: + next_block_start = nesting_state.stack[block_index - 1].starting_linenum + first_line = last_line + while first_line >= next_block_start: + if clean_lines.elided[first_line].find('template') >= 0: + break + first_line -= 1 + if first_line < next_block_start: + # Didn't find any "template" keyword before reaching the next block, + # there are probably no template things to check for this block + block_index -= 1 + continue + + # Look for typename in the specified range + for i in xrange(first_line, last_line + 1, 1): + if Search(typename_pattern, clean_lines.elided[i]): + return True + block_index -= 1 + + return False + + +def CheckBracesSpacing(filename, clean_lines, linenum, nesting_state, error): + """Checks for horizontal spacing near commas. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + nesting_state: A NestingState instance which maintains information about + the current stack of nested blocks being parsed. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Except after an opening paren, or after another opening brace (in case of + # an initializer list, for instance), you should have spaces before your + # braces when they are delimiting blocks, classes, namespaces etc. + # And since you should never have braces at the beginning of a line, + # this is an easy test. Except that braces used for initialization don't + # follow the same rule; we often don't want spaces before those. + match = Match(r'^(.*[^ ({>]){', line) + + if match: + # Try a bit harder to check for brace initialization. This + # happens in one of the following forms: + # Constructor() : initializer_list_{} { ... } + # Constructor{}.MemberFunction() + # Type variable{}; + # FunctionCall(type{}, ...); + # LastArgument(..., type{}); + # LOG(INFO) << type{} << " ..."; + # map_of_type[{...}] = ...; + # ternary = expr ? new type{} : nullptr; + # OuterTemplate{}> + # + # We check for the character following the closing brace, and + # silence the warning if it's one of those listed above, i.e. + # "{.;,)<>]:". + # + # To account for nested initializer list, we allow any number of + # closing braces up to "{;,)<". We can't simply silence the + # warning on first sight of closing brace, because that would + # cause false negatives for things that are not initializer lists. + # Silence this: But not this: + # Outer{ if (...) { + # Inner{...} if (...){ // Missing space before { + # }; } + # + # There is a false negative with this approach if people inserted + # spurious semicolons, e.g. "if (cond){};", but we will catch the + # spurious semicolon with a separate check. + leading_text = match.group(1) + (endline, endlinenum, endpos) = CloseExpression( + clean_lines, linenum, len(match.group(1))) + trailing_text = '' + if endpos > -1: + trailing_text = endline[endpos:] + for offset in xrange(endlinenum + 1, + min(endlinenum + 3, clean_lines.NumLines() - 1)): + trailing_text += clean_lines.elided[offset] + # We also suppress warnings for `uint64_t{expression}` etc., as the style + # guide recommends brace initialization for integral types to avoid + # overflow/truncation. + if (not Match(r'^[\s}]*[{.;,)<>\]:]', trailing_text) + and not _IsType(clean_lines, nesting_state, leading_text)): + error(filename, linenum, 'whitespace/braces', 5, + 'Missing space before {') + + # Make sure '} else {' has spaces. + if Search(r'}else', line): + error(filename, linenum, 'whitespace/braces', 5, + 'Missing space before else') + + # You shouldn't have a space before a semicolon at the end of the line. + # There's a special case for "for" since the style guide allows space before + # the semicolon there. + if Search(r':\s*;\s*$', line): + error(filename, linenum, 'whitespace/semicolon', 5, + 'Semicolon defining empty statement. Use {} instead.') + elif Search(r'^\s*;\s*$', line): + error(filename, linenum, 'whitespace/semicolon', 5, + 'Line contains only semicolon. If this should be an empty statement, ' + 'use {} instead.') + elif (Search(r'\s+;\s*$', line) and + not Search(r'\bfor\b', line)): + error(filename, linenum, 'whitespace/semicolon', 5, + 'Extra space before last semicolon. If this should be an empty ' + 'statement, use {} instead.') + + +def IsDecltype(clean_lines, linenum, column): + """Check if the token ending on (linenum, column) is decltype(). + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: the number of the line to check. + column: end column of the token to check. + Returns: + True if this token is decltype() expression, False otherwise. + """ + (text, _, start_col) = ReverseCloseExpression(clean_lines, linenum, column) + if start_col < 0: + return False + if Search(r'\bdecltype\s*$', text[0:start_col]): + return True + return False + +def CheckSectionSpacing(filename, clean_lines, class_info, linenum, error): + """Checks for additional blank line issues related to sections. + + Currently the only thing checked here is blank line before protected/private. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + class_info: A _ClassInfo objects. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + # Skip checks if the class is small, where small means 25 lines or less. + # 25 lines seems like a good cutoff since that's the usual height of + # terminals, and any class that can't fit in one screen can't really + # be considered "small". + # + # Also skip checks if we are on the first line. This accounts for + # classes that look like + # class Foo { public: ... }; + # + # If we didn't find the end of the class, last_line would be zero, + # and the check will be skipped by the first condition. + if (class_info.last_line - class_info.starting_linenum <= 24 or + linenum <= class_info.starting_linenum): + return + + matched = Match(r'\s*(public|protected|private):', clean_lines.lines[linenum]) + if matched: + # Issue warning if the line before public/protected/private was + # not a blank line, but don't do this if the previous line contains + # "class" or "struct". This can happen two ways: + # - We are at the beginning of the class. + # - We are forward-declaring an inner class that is semantically + # private, but needed to be public for implementation reasons. + # Also ignores cases where the previous line ends with a backslash as can be + # common when defining classes in C macros. + prev_line = clean_lines.lines[linenum - 1] + if (not IsBlankLine(prev_line) and + not Search(r'\b(class|struct)\b', prev_line) and + not Search(r'\\$', prev_line)): + # Try a bit harder to find the beginning of the class. This is to + # account for multi-line base-specifier lists, e.g.: + # class Derived + # : public Base { + end_class_head = class_info.starting_linenum + for i in range(class_info.starting_linenum, linenum): + if Search(r'\{\s*$', clean_lines.lines[i]): + end_class_head = i + break + if end_class_head < linenum - 1: + error(filename, linenum, 'whitespace/blank_line', 3, + '"%s:" should be preceded by a blank line' % matched.group(1)) + + +def GetPreviousNonBlankLine(clean_lines, linenum): + """Return the most recent non-blank line and its line number. + + Args: + clean_lines: A CleansedLines instance containing the file contents. + linenum: The number of the line to check. + + Returns: + A tuple with two elements. The first element is the contents of the last + non-blank line before the current line, or the empty string if this is the + first non-blank line. The second is the line number of that line, or -1 + if this is the first non-blank line. + """ + + prevlinenum = linenum - 1 + while prevlinenum >= 0: + prevline = clean_lines.elided[prevlinenum] + if not IsBlankLine(prevline): # if not a blank line... + return (prevline, prevlinenum) + prevlinenum -= 1 + return ('', -1) + + +def CheckBraces(filename, clean_lines, linenum, error): + """Looks for misplaced braces (e.g. at the end of line). + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + + line = clean_lines.elided[linenum] # get rid of comments and strings + + if Match(r'\s*{\s*$', line): + # We allow an open brace to start a line in the case where someone is using + # braces in a block to explicitly create a new scope, which is commonly used + # to control the lifetime of stack-allocated variables. Braces are also + # used for brace initializers inside function calls. We don't detect this + # perfectly: we just don't complain if the last non-whitespace character on + # the previous non-blank line is ',', ';', ':', '(', '{', or '}', or if the + # previous line starts a preprocessor block. We also allow a brace on the + # following line if it is part of an array initialization and would not fit + # within the 80 character limit of the preceding line. + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if (not Search(r'[,;:}{(]\s*$', prevline) and + not Match(r'\s*#', prevline) and + not (GetLineWidth(prevline) > _line_length - 2 and '[]' in prevline)): + error(filename, linenum, 'whitespace/braces', 4, + '{ should almost always be at the end of the previous line') + + # An else clause should be on the same line as the preceding closing brace. + if Match(r'\s*else\b\s*(?:if\b|\{|$)', line): + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if Match(r'\s*}\s*$', prevline): + error(filename, linenum, 'whitespace/newline', 4, + 'An else should appear on the same line as the preceding }') + + # If braces come on one side of an else, they should be on both. + # However, we have to worry about "else if" that spans multiple lines! + if Search(r'else if\s*\(', line): # could be multi-line if + brace_on_left = bool(Search(r'}\s*else if\s*\(', line)) + # find the ( after the if + pos = line.find('else if') + pos = line.find('(', pos) + if pos > 0: + (endline, _, endpos) = CloseExpression(clean_lines, linenum, pos) + brace_on_right = endline[endpos:].find('{') != -1 + if brace_on_left != brace_on_right: # must be brace after if + error(filename, linenum, 'readability/braces', 5, + 'If an else has a brace on one side, it should have it on both') + elif Search(r'}\s*else[^{]*$', line) or Match(r'[^}]*else\s*{', line): + error(filename, linenum, 'readability/braces', 5, + 'If an else has a brace on one side, it should have it on both') + + # Likewise, an else should never have the else clause on the same line + if Search(r'\belse [^\s{]', line) and not Search(r'\belse if\b', line): + error(filename, linenum, 'whitespace/newline', 4, + 'Else clause should never be on same line as else (use 2 lines)') + + # In the same way, a do/while should never be on one line + if Match(r'\s*do [^\s{]', line): + error(filename, linenum, 'whitespace/newline', 4, + 'do/while clauses should not be on a single line') + + # Check single-line if/else bodies. The style guide says 'curly braces are not + # required for single-line statements'. We additionally allow multi-line, + # single statements, but we reject anything with more than one semicolon in + # it. This means that the first semicolon after the if should be at the end of + # its line, and the line after that should have an indent level equal to or + # lower than the if. We also check for ambiguous if/else nesting without + # braces. + if_else_match = Search(r'\b(if\s*\(|else\b)', line) + if if_else_match and not Match(r'\s*#', line): + if_indent = GetIndentLevel(line) + endline, endlinenum, endpos = line, linenum, if_else_match.end() + if_match = Search(r'\bif\s*\(', line) + if if_match: + # This could be a multiline if condition, so find the end first. + pos = if_match.end() - 1 + (endline, endlinenum, endpos) = CloseExpression(clean_lines, linenum, pos) + # Check for an opening brace, either directly after the if or on the next + # line. If found, this isn't a single-statement conditional. + if (not Match(r'\s*{', endline[endpos:]) + and not (Match(r'\s*$', endline[endpos:]) + and endlinenum < (len(clean_lines.elided) - 1) + and Match(r'\s*{', clean_lines.elided[endlinenum + 1]))): + while (endlinenum < len(clean_lines.elided) + and ';' not in clean_lines.elided[endlinenum][endpos:]): + endlinenum += 1 + endpos = 0 + if endlinenum < len(clean_lines.elided): + endline = clean_lines.elided[endlinenum] + # We allow a mix of whitespace and closing braces (e.g. for one-liner + # methods) and a single \ after the semicolon (for macros) + endpos = endline.find(';') + if not Match(r';[\s}]*(\\?)$', endline[endpos:]): + # Semicolon isn't the last character, there's something trailing. + # Output a warning if the semicolon is not contained inside + # a lambda expression. + if not Match(r'^[^{};]*\[[^\[\]]*\][^{}]*\{[^{}]*\}\s*\)*[;,]\s*$', + endline): + error(filename, linenum, 'readability/braces', 4, + 'If/else bodies with multiple statements require braces') + elif endlinenum < len(clean_lines.elided) - 1: + # Make sure the next line is dedented + next_line = clean_lines.elided[endlinenum + 1] + next_indent = GetIndentLevel(next_line) + # With ambiguous nested if statements, this will error out on the + # if that *doesn't* match the else, regardless of whether it's the + # inner one or outer one. + if (if_match and Match(r'\s*else\b', next_line) + and next_indent != if_indent): + error(filename, linenum, 'readability/braces', 4, + 'Else clause should be indented at the same level as if. ' + 'Ambiguous nested if/else chains require braces.') + elif next_indent > if_indent: + error(filename, linenum, 'readability/braces', 4, + 'If/else bodies with multiple statements require braces') + + +def CheckTrailingSemicolon(filename, clean_lines, linenum, error): + """Looks for redundant trailing semicolon. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + + line = clean_lines.elided[linenum] + + # Block bodies should not be followed by a semicolon. Due to C++11 + # brace initialization, there are more places where semicolons are + # required than not, so we use a whitelist approach to check these + # rather than a blacklist. These are the places where "};" should + # be replaced by just "}": + # 1. Some flavor of block following closing parenthesis: + # for (;;) {}; + # while (...) {}; + # switch (...) {}; + # Function(...) {}; + # if (...) {}; + # if (...) else if (...) {}; + # + # 2. else block: + # if (...) else {}; + # + # 3. const member function: + # Function(...) const {}; + # + # 4. Block following some statement: + # x = 42; + # {}; + # + # 5. Block at the beginning of a function: + # Function(...) { + # {}; + # } + # + # Note that naively checking for the preceding "{" will also match + # braces inside multi-dimensional arrays, but this is fine since + # that expression will not contain semicolons. + # + # 6. Block following another block: + # while (true) {} + # {}; + # + # 7. End of namespaces: + # namespace {}; + # + # These semicolons seems far more common than other kinds of + # redundant semicolons, possibly due to people converting classes + # to namespaces. For now we do not warn for this case. + # + # Try matching case 1 first. + match = Match(r'^(.*\)\s*)\{', line) + if match: + # Matched closing parenthesis (case 1). Check the token before the + # matching opening parenthesis, and don't warn if it looks like a + # macro. This avoids these false positives: + # - macro that defines a base class + # - multi-line macro that defines a base class + # - macro that defines the whole class-head + # + # But we still issue warnings for macros that we know are safe to + # warn, specifically: + # - TEST, TEST_F, TEST_P, MATCHER, MATCHER_P + # - TYPED_TEST + # - INTERFACE_DEF + # - EXCLUSIVE_LOCKS_REQUIRED, SHARED_LOCKS_REQUIRED, LOCKS_EXCLUDED: + # + # We implement a whitelist of safe macros instead of a blacklist of + # unsafe macros, even though the latter appears less frequently in + # google code and would have been easier to implement. This is because + # the downside for getting the whitelist wrong means some extra + # semicolons, while the downside for getting the blacklist wrong + # would result in compile errors. + # + # In addition to macros, we also don't want to warn on + # - Compound literals + # - Lambdas + # - alignas specifier with anonymous structs + # - decltype + closing_brace_pos = match.group(1).rfind(')') + opening_parenthesis = ReverseCloseExpression( + clean_lines, linenum, closing_brace_pos) + if opening_parenthesis[2] > -1: + line_prefix = opening_parenthesis[0][0:opening_parenthesis[2]] + macro = Search(r'\b([A-Z_][A-Z0-9_]*)\s*$', line_prefix) + func = Match(r'^(.*\])\s*$', line_prefix) + if ((macro and + macro.group(1) not in ( + 'TEST', 'TEST_F', 'MATCHER', 'MATCHER_P', 'TYPED_TEST', + 'EXCLUSIVE_LOCKS_REQUIRED', 'SHARED_LOCKS_REQUIRED', + 'LOCKS_EXCLUDED', 'INTERFACE_DEF')) or + (func and not Search(r'\boperator\s*\[\s*\]', func.group(1))) or + Search(r'\b(?:struct|union)\s+alignas\s*$', line_prefix) or + Search(r'\bdecltype$', line_prefix) or + Search(r'\s+=\s*$', line_prefix)): + match = None + if (match and + opening_parenthesis[1] > 1 and + Search(r'\]\s*$', clean_lines.elided[opening_parenthesis[1] - 1])): + # Multi-line lambda-expression + match = None + + else: + # Try matching cases 2-3. + match = Match(r'^(.*(?:else|\)\s*const)\s*)\{', line) + if not match: + # Try matching cases 4-6. These are always matched on separate lines. + # + # Note that we can't simply concatenate the previous line to the + # current line and do a single match, otherwise we may output + # duplicate warnings for the blank line case: + # if (cond) { + # // blank line + # } + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if prevline and Search(r'[;{}]\s*$', prevline): + match = Match(r'^(\s*)\{', line) + + # Check matching closing brace + if match: + (endline, endlinenum, endpos) = CloseExpression( + clean_lines, linenum, len(match.group(1))) + if endpos > -1 and Match(r'^\s*;', endline[endpos:]): + # Current {} pair is eligible for semicolon check, and we have found + # the redundant semicolon, output warning here. + # + # Note: because we are scanning forward for opening braces, and + # outputting warnings for the matching closing brace, if there are + # nested blocks with trailing semicolons, we will get the error + # messages in reversed order. + + # We need to check the line forward for NOLINT + raw_lines = clean_lines.raw_lines + ParseNolintSuppressions(filename, raw_lines[endlinenum-1], endlinenum-1, + error) + ParseNolintSuppressions(filename, raw_lines[endlinenum], endlinenum, + error) + + error(filename, endlinenum, 'readability/braces', 4, + "You don't need a ; after a }") + + +def CheckEmptyBlockBody(filename, clean_lines, linenum, error): + """Look for empty loop/conditional body with only a single semicolon. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + + # Search for loop keywords at the beginning of the line. Because only + # whitespaces are allowed before the keywords, this will also ignore most + # do-while-loops, since those lines should start with closing brace. + # + # We also check "if" blocks here, since an empty conditional block + # is likely an error. + line = clean_lines.elided[linenum] + matched = Match(r'\s*(for|while|if)\s*\(', line) + if matched: + # Find the end of the conditional expression. + (end_line, end_linenum, end_pos) = CloseExpression( + clean_lines, linenum, line.find('(')) + + # Output warning if what follows the condition expression is a semicolon. + # No warning for all other cases, including whitespace or newline, since we + # have a separate check for semicolons preceded by whitespace. + if end_pos >= 0 and Match(r';', end_line[end_pos:]): + if matched.group(1) == 'if': + error(filename, end_linenum, 'whitespace/empty_conditional_body', 5, + 'Empty conditional bodies should use {}') + else: + error(filename, end_linenum, 'whitespace/empty_loop_body', 5, + 'Empty loop bodies should use {} or continue') + + # Check for if statements that have completely empty bodies (no comments) + # and no else clauses. + if end_pos >= 0 and matched.group(1) == 'if': + # Find the position of the opening { for the if statement. + # Return without logging an error if it has no brackets. + opening_linenum = end_linenum + opening_line_fragment = end_line[end_pos:] + # Loop until EOF or find anything that's not whitespace or opening {. + while not Search(r'^\s*\{', opening_line_fragment): + if Search(r'^(?!\s*$)', opening_line_fragment): + # Conditional has no brackets. + return + opening_linenum += 1 + if opening_linenum == len(clean_lines.elided): + # Couldn't find conditional's opening { or any code before EOF. + return + opening_line_fragment = clean_lines.elided[opening_linenum] + # Set opening_line (opening_line_fragment may not be entire opening line). + opening_line = clean_lines.elided[opening_linenum] + + # Find the position of the closing }. + opening_pos = opening_line_fragment.find('{') + if opening_linenum == end_linenum: + # We need to make opening_pos relative to the start of the entire line. + opening_pos += end_pos + (closing_line, closing_linenum, closing_pos) = CloseExpression( + clean_lines, opening_linenum, opening_pos) + if closing_pos < 0: + return + + # Now construct the body of the conditional. This consists of the portion + # of the opening line after the {, all lines until the closing line, + # and the portion of the closing line before the }. + if (clean_lines.raw_lines[opening_linenum] != + CleanseComments(clean_lines.raw_lines[opening_linenum])): + # Opening line ends with a comment, so conditional isn't empty. + return + if closing_linenum > opening_linenum: + # Opening line after the {. Ignore comments here since we checked above. + bodylist = list(opening_line[opening_pos+1:]) + # All lines until closing line, excluding closing line, with comments. + bodylist.extend(clean_lines.raw_lines[opening_linenum+1:closing_linenum]) + # Closing line before the }. Won't (and can't) have comments. + bodylist.append(clean_lines.elided[closing_linenum][:closing_pos-1]) + body = '\n'.join(bodylist) + else: + # If statement has brackets and fits on a single line. + body = opening_line[opening_pos+1:closing_pos-1] + + # Check if the body is empty + if not _EMPTY_CONDITIONAL_BODY_PATTERN.search(body): + return + # The body is empty. Now make sure there's not an else clause. + current_linenum = closing_linenum + current_line_fragment = closing_line[closing_pos:] + # Loop until EOF or find anything that's not whitespace or else clause. + while Search(r'^\s*$|^(?=\s*else)', current_line_fragment): + if Search(r'^(?=\s*else)', current_line_fragment): + # Found an else clause, so don't log an error. + return + current_linenum += 1 + if current_linenum == len(clean_lines.elided): + break + current_line_fragment = clean_lines.elided[current_linenum] + + # The body is empty and there's no else clause until EOF or other code. + error(filename, end_linenum, 'whitespace/empty_if_body', 4, + ('If statement had no body and no else clause')) + + +def FindCheckMacro(line): + """Find a replaceable CHECK-like macro. + + Args: + line: line to search on. + Returns: + (macro name, start position), or (None, -1) if no replaceable + macro is found. + """ + for macro in _CHECK_MACROS: + i = line.find(macro) + if i >= 0: + # Find opening parenthesis. Do a regular expression match here + # to make sure that we are matching the expected CHECK macro, as + # opposed to some other macro that happens to contain the CHECK + # substring. + matched = Match(r'^(.*\b' + macro + r'\s*)\(', line) + if not matched: + continue + return (macro, len(matched.group(1))) + return (None, -1) + + +def CheckCheck(filename, clean_lines, linenum, error): + """Checks the use of CHECK and EXPECT macros. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + + # Decide the set of replacement macros that should be suggested + lines = clean_lines.elided + (check_macro, start_pos) = FindCheckMacro(lines[linenum]) + if not check_macro: + return + + # Find end of the boolean expression by matching parentheses + (last_line, end_line, end_pos) = CloseExpression( + clean_lines, linenum, start_pos) + if end_pos < 0: + return + + # If the check macro is followed by something other than a + # semicolon, assume users will log their own custom error messages + # and don't suggest any replacements. + if not Match(r'\s*;', last_line[end_pos:]): + return + + if linenum == end_line: + expression = lines[linenum][start_pos + 1:end_pos - 1] + else: + expression = lines[linenum][start_pos + 1:] + for i in xrange(linenum + 1, end_line): + expression += lines[i] + expression += last_line[0:end_pos - 1] + + # Parse expression so that we can take parentheses into account. + # This avoids false positives for inputs like "CHECK((a < 4) == b)", + # which is not replaceable by CHECK_LE. + lhs = '' + rhs = '' + operator = None + while expression: + matched = Match(r'^\s*(<<|<<=|>>|>>=|->\*|->|&&|\|\||' + r'==|!=|>=|>|<=|<|\()(.*)$', expression) + if matched: + token = matched.group(1) + if token == '(': + # Parenthesized operand + expression = matched.group(2) + (end, _) = FindEndOfExpressionInLine(expression, 0, ['(']) + if end < 0: + return # Unmatched parenthesis + lhs += '(' + expression[0:end] + expression = expression[end:] + elif token in ('&&', '||'): + # Logical and/or operators. This means the expression + # contains more than one term, for example: + # CHECK(42 < a && a < b); + # + # These are not replaceable with CHECK_LE, so bail out early. + return + elif token in ('<<', '<<=', '>>', '>>=', '->*', '->'): + # Non-relational operator + lhs += token + expression = matched.group(2) + else: + # Relational operator + operator = token + rhs = matched.group(2) + break + else: + # Unparenthesized operand. Instead of appending to lhs one character + # at a time, we do another regular expression match to consume several + # characters at once if possible. Trivial benchmark shows that this + # is more efficient when the operands are longer than a single + # character, which is generally the case. + matched = Match(r'^([^-=!<>()&|]+)(.*)$', expression) + if not matched: + matched = Match(r'^(\s*\S)(.*)$', expression) + if not matched: + break + lhs += matched.group(1) + expression = matched.group(2) + + # Only apply checks if we got all parts of the boolean expression + if not (lhs and operator and rhs): + return + + # Check that rhs do not contain logical operators. We already know + # that lhs is fine since the loop above parses out && and ||. + if rhs.find('&&') > -1 or rhs.find('||') > -1: + return + + # At least one of the operands must be a constant literal. This is + # to avoid suggesting replacements for unprintable things like + # CHECK(variable != iterator) + # + # The following pattern matches decimal, hex integers, strings, and + # characters (in that order). + lhs = lhs.strip() + rhs = rhs.strip() + match_constant = r'^([-+]?(\d+|0[xX][0-9a-fA-F]+)[lLuU]{0,3}|".*"|\'.*\')$' + if Match(match_constant, lhs) or Match(match_constant, rhs): + # Note: since we know both lhs and rhs, we can provide a more + # descriptive error message like: + # Consider using CHECK_EQ(x, 42) instead of CHECK(x == 42) + # Instead of: + # Consider using CHECK_EQ instead of CHECK(a == b) + # + # We are still keeping the less descriptive message because if lhs + # or rhs gets long, the error message might become unreadable. + error(filename, linenum, 'readability/check', 2, + 'Consider using %s instead of %s(a %s b)' % ( + _CHECK_REPLACEMENT[check_macro][operator], + check_macro, operator)) + + +def CheckAltTokens(filename, clean_lines, linenum, error): + """Check alternative keywords being used in boolean expressions. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Avoid preprocessor lines + if Match(r'^\s*#', line): + return + + # Last ditch effort to avoid multi-line comments. This will not help + # if the comment started before the current line or ended after the + # current line, but it catches most of the false positives. At least, + # it provides a way to workaround this warning for people who use + # multi-line comments in preprocessor macros. + # + # TODO(unknown): remove this once cpplint has better support for + # multi-line comments. + if line.find('/*') >= 0 or line.find('*/') >= 0: + return + + for match in _ALT_TOKEN_REPLACEMENT_PATTERN.finditer(line): + error(filename, linenum, 'readability/alt_tokens', 2, + 'Use operator %s instead of %s' % ( + _ALT_TOKEN_REPLACEMENT[match.group(1)], match.group(1))) + + +def GetLineWidth(line): + """Determines the width of the line in column positions. + + Args: + line: A string, which may be a Unicode string. + + Returns: + The width of the line in column positions, accounting for Unicode + combining characters and wide characters. + """ + if isinstance(line, unicode): + width = 0 + for uc in unicodedata.normalize('NFC', line): + if unicodedata.east_asian_width(uc) in ('W', 'F'): + width += 2 + elif not unicodedata.combining(uc): + width += 1 + return width + else: + return len(line) + + +def CheckStyle(filename, clean_lines, linenum, file_extension, nesting_state, + error): + """Checks rules from the 'C++ style rules' section of cppguide.html. + + Most of these rules are hard to test (naming, comment style), but we + do what we can. In particular we check for 2-space indents, line lengths, + tab usage, spaces inside code, etc. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + file_extension: The extension (without the dot) of the filename. + nesting_state: A NestingState instance which maintains information about + the current stack of nested blocks being parsed. + error: The function to call with any errors found. + """ + + # Don't use "elided" lines here, otherwise we can't check commented lines. + # Don't want to use "raw" either, because we don't want to check inside C++11 + # raw strings, + raw_lines = clean_lines.lines_without_raw_strings + line = raw_lines[linenum] + prev = raw_lines[linenum - 1] if linenum > 0 else '' + + if line.find('\t') != -1: + error(filename, linenum, 'whitespace/tab', 1, + 'Tab found; better to use spaces') + + # One or three blank spaces at the beginning of the line is weird; it's + # hard to reconcile that with 2-space indents. + # NOTE: here are the conditions rob pike used for his tests. Mine aren't + # as sophisticated, but it may be worth becoming so: RLENGTH==initial_spaces + # if(RLENGTH > 20) complain = 0; + # if(match($0, " +(error|private|public|protected):")) complain = 0; + # if(match(prev, "&& *$")) complain = 0; + # if(match(prev, "\\|\\| *$")) complain = 0; + # if(match(prev, "[\",=><] *$")) complain = 0; + # if(match($0, " <<")) complain = 0; + # if(match(prev, " +for \\(")) complain = 0; + # if(prevodd && match(prevprev, " +for \\(")) complain = 0; + scope_or_label_pattern = r'\s*\w+\s*:\s*\\?$' + classinfo = nesting_state.InnermostClass() + initial_spaces = 0 + cleansed_line = clean_lines.elided[linenum] + while initial_spaces < len(line) and line[initial_spaces] == ' ': + initial_spaces += 1 + # There are certain situations we allow one space, notably for + # section labels, and also lines containing multi-line raw strings. + # We also don't check for lines that look like continuation lines + # (of lines ending in double quotes, commas, equals, or angle brackets) + # because the rules for how to indent those are non-trivial. + if (not Search(r'[",=><] *$', prev) and + (initial_spaces == 1 or initial_spaces == 3) and + not Match(scope_or_label_pattern, cleansed_line) and + not (clean_lines.raw_lines[linenum] != line and + Match(r'^\s*""', line))): + error(filename, linenum, 'whitespace/indent', 3, + 'Weird number of spaces at line-start. ' + 'Are you using a 2-space indent?') + + if line and line[-1].isspace(): + error(filename, linenum, 'whitespace/end_of_line', 4, + 'Line ends in whitespace. Consider deleting these extra spaces.') + + # Check if the line is a header guard. + is_header_guard = False + if file_extension in GetHeaderExtensions(): + cppvar = GetHeaderGuardCPPVariable(filename) + if (line.startswith('#ifndef %s' % cppvar) or + line.startswith('#define %s' % cppvar) or + line.startswith('#endif // %s' % cppvar)): + is_header_guard = True + # #include lines and header guards can be long, since there's no clean way to + # split them. + # + # URLs can be long too. It's possible to split these, but it makes them + # harder to cut&paste. + # + # The "$Id:...$" comment may also get very long without it being the + # developers fault. + # + # Doxygen documentation copying can get pretty long when using an overloaded + # function declaration + if (not line.startswith('#include') and not is_header_guard and + not Match(r'^\s*//.*http(s?)://\S*$', line) and + not Match(r'^\s*//\s*[^\s]*$', line) and + not Match(r'^// \$Id:.*#[0-9]+ \$$', line) and + not Match(r'^\s*/// [@\\](copydoc|copydetails|copybrief) .*$', line)): + line_width = GetLineWidth(line) + if line_width > _line_length: + error(filename, linenum, 'whitespace/line_length', 2, + 'Lines should be <= %i characters long' % _line_length) + + if (cleansed_line.count(';') > 1 and + # allow simple single line lambdas + not Match(r'^[^{};]*\[[^\[\]]*\][^{}]*\{[^{}\n\r]*\}', + line) and + # for loops are allowed two ;'s (and may run over two lines). + cleansed_line.find('for') == -1 and + (GetPreviousNonBlankLine(clean_lines, linenum)[0].find('for') == -1 or + GetPreviousNonBlankLine(clean_lines, linenum)[0].find(';') != -1) and + # It's ok to have many commands in a switch case that fits in 1 line + not ((cleansed_line.find('case ') != -1 or + cleansed_line.find('default:') != -1) and + cleansed_line.find('break;') != -1)): + error(filename, linenum, 'whitespace/newline', 0, + 'More than one command on the same line') + + # Some more style checks + CheckBraces(filename, clean_lines, linenum, error) + CheckTrailingSemicolon(filename, clean_lines, linenum, error) + CheckEmptyBlockBody(filename, clean_lines, linenum, error) + CheckAccess(filename, clean_lines, linenum, nesting_state, error) + CheckSpacing(filename, clean_lines, linenum, nesting_state, error) + CheckOperatorSpacing(filename, clean_lines, linenum, error) + CheckParenthesisSpacing(filename, clean_lines, linenum, error) + CheckCommaSpacing(filename, clean_lines, linenum, error) + CheckBracesSpacing(filename, clean_lines, linenum, nesting_state, error) + CheckSpacingForFunctionCall(filename, clean_lines, linenum, error) + CheckCheck(filename, clean_lines, linenum, error) + CheckAltTokens(filename, clean_lines, linenum, error) + classinfo = nesting_state.InnermostClass() + if classinfo: + CheckSectionSpacing(filename, clean_lines, classinfo, linenum, error) + + +_RE_PATTERN_INCLUDE = re.compile(r'^\s*#\s*include\s*([<"])([^>"]*)[>"].*$') +# Matches the first component of a filename delimited by -s and _s. That is: +# _RE_FIRST_COMPONENT.match('foo').group(0) == 'foo' +# _RE_FIRST_COMPONENT.match('foo.cc').group(0) == 'foo' +# _RE_FIRST_COMPONENT.match('foo-bar_baz.cc').group(0) == 'foo' +# _RE_FIRST_COMPONENT.match('foo_bar-baz.cc').group(0) == 'foo' +_RE_FIRST_COMPONENT = re.compile(r'^[^-_.]+') + + +def _DropCommonSuffixes(filename): + """Drops common suffixes like _test.cc or -inl.h from filename. + + For example: + >>> _DropCommonSuffixes('foo/foo-inl.h') + 'foo/foo' + >>> _DropCommonSuffixes('foo/bar/foo.cc') + 'foo/bar/foo' + >>> _DropCommonSuffixes('foo/foo_internal.h') + 'foo/foo' + >>> _DropCommonSuffixes('foo/foo_unusualinternal.h') + 'foo/foo_unusualinternal' + + Args: + filename: The input filename. + + Returns: + The filename with the common suffix removed. + """ + for suffix in itertools.chain( + ('%s.%s' % (test_suffix.lstrip('_'), ext) + for test_suffix, ext in itertools.product(_test_suffixes, GetNonHeaderExtensions())), + ('%s.%s' % (suffix, ext) + for suffix, ext in itertools.product(['inl', 'imp', 'internal'], GetHeaderExtensions()))): + if (filename.endswith(suffix) and len(filename) > len(suffix) and + filename[-len(suffix) - 1] in ('-', '_')): + return filename[:-len(suffix) - 1] + return os.path.splitext(filename)[0] + + +def _ClassifyInclude(fileinfo, include, is_system): + """Figures out what kind of header 'include' is. + + Args: + fileinfo: The current file cpplint is running over. A FileInfo instance. + include: The path to a #included file. + is_system: True if the #include used <> rather than "". + + Returns: + One of the _XXX_HEADER constants. + + For example: + >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'stdio.h', True) + _C_SYS_HEADER + >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'string', True) + _CPP_SYS_HEADER + >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/foo.h', False) + _LIKELY_MY_HEADER + >>> _ClassifyInclude(FileInfo('foo/foo_unknown_extension.cc'), + ... 'bar/foo_other_ext.h', False) + _POSSIBLE_MY_HEADER + >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/bar.h', False) + _OTHER_HEADER + """ + # This is a list of all standard c++ header files, except + # those already checked for above. + is_cpp_h = include in _CPP_HEADERS + + # Headers with C++ extensions shouldn't be considered C system headers + if is_system and os.path.splitext(include)[1] in ['.hpp', '.hxx', '.h++']: + is_system = False + + if is_system: + if is_cpp_h: + return _CPP_SYS_HEADER + else: + return _C_SYS_HEADER + + # If the target file and the include we're checking share a + # basename when we drop common extensions, and the include + # lives in . , then it's likely to be owned by the target file. + target_dir, target_base = ( + os.path.split(_DropCommonSuffixes(fileinfo.RepositoryName()))) + include_dir, include_base = os.path.split(_DropCommonSuffixes(include)) + target_dir_pub = os.path.normpath(target_dir + '/../public') + target_dir_pub = target_dir_pub.replace('\\', '/') + if target_base == include_base and ( + include_dir == target_dir or + include_dir == target_dir_pub): + return _LIKELY_MY_HEADER + + # If the target and include share some initial basename + # component, it's possible the target is implementing the + # include, so it's allowed to be first, but we'll never + # complain if it's not there. + target_first_component = _RE_FIRST_COMPONENT.match(target_base) + include_first_component = _RE_FIRST_COMPONENT.match(include_base) + if (target_first_component and include_first_component and + target_first_component.group(0) == + include_first_component.group(0)): + return _POSSIBLE_MY_HEADER + + return _OTHER_HEADER + + + +def CheckIncludeLine(filename, clean_lines, linenum, include_state, error): + """Check rules that are applicable to #include lines. + + Strings on #include lines are NOT removed from elided line, to make + certain tasks easier. However, to prevent false positives, checks + applicable to #include lines in CheckLanguage must be put here. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + include_state: An _IncludeState instance in which the headers are inserted. + error: The function to call with any errors found. + """ + fileinfo = FileInfo(filename) + line = clean_lines.lines[linenum] + + # "include" should use the new style "foo/bar.h" instead of just "bar.h" + # Only do this check if the included header follows google naming + # conventions. If not, assume that it's a 3rd party API that + # requires special include conventions. + # + # We also make an exception for Lua headers, which follow google + # naming convention but not the include convention. + match = Match(r'#include\s*"([^/]+\.h)"', line) + if match and not _THIRD_PARTY_HEADERS_PATTERN.match(match.group(1)): + error(filename, linenum, 'build/include_subdir', 4, + 'Include the directory when naming .h files') + + # we shouldn't include a file more than once. actually, there are a + # handful of instances where doing so is okay, but in general it's + # not. + match = _RE_PATTERN_INCLUDE.search(line) + if match: + include = match.group(2) + is_system = (match.group(1) == '<') + duplicate_line = include_state.FindHeader(include) + if duplicate_line >= 0: + error(filename, linenum, 'build/include', 4, + '"%s" already included at %s:%s' % + (include, filename, duplicate_line)) + return + + for extension in GetNonHeaderExtensions(): + if (include.endswith('.' + extension) and + os.path.dirname(fileinfo.RepositoryName()) != os.path.dirname(include)): + error(filename, linenum, 'build/include', 4, + 'Do not include .' + extension + ' files from other packages') + return + + if not _THIRD_PARTY_HEADERS_PATTERN.match(include): + include_state.include_list[-1].append((include, linenum)) + + # We want to ensure that headers appear in the right order: + # 1) for foo.cc, foo.h (preferred location) + # 2) c system files + # 3) cpp system files + # 4) for foo.cc, foo.h (deprecated location) + # 5) other google headers + # + # We classify each include statement as one of those 5 types + # using a number of techniques. The include_state object keeps + # track of the highest type seen, and complains if we see a + # lower type after that. + error_message = include_state.CheckNextIncludeOrder( + _ClassifyInclude(fileinfo, include, is_system)) + if error_message: + error(filename, linenum, 'build/include_order', 4, + '%s. Should be: %s.h, c system, c++ system, other.' % + (error_message, fileinfo.BaseName())) + canonical_include = include_state.CanonicalizeAlphabeticalOrder(include) + if not include_state.IsInAlphabeticalOrder( + clean_lines, linenum, canonical_include): + error(filename, linenum, 'build/include_alpha', 4, + 'Include "%s" not in alphabetical order' % include) + include_state.SetLastHeader(canonical_include) + + + +def _GetTextInside(text, start_pattern): + r"""Retrieves all the text between matching open and close parentheses. + + Given a string of lines and a regular expression string, retrieve all the text + following the expression and between opening punctuation symbols like + (, [, or {, and the matching close-punctuation symbol. This properly nested + occurrences of the punctuations, so for the text like + printf(a(), b(c())); + a call to _GetTextInside(text, r'printf\(') will return 'a(), b(c())'. + start_pattern must match string having an open punctuation symbol at the end. + + Args: + text: The lines to extract text. Its comments and strings must be elided. + It can be single line and can span multiple lines. + start_pattern: The regexp string indicating where to start extracting + the text. + Returns: + The extracted text. + None if either the opening string or ending punctuation could not be found. + """ + # TODO(unknown): Audit cpplint.py to see what places could be profitably + # rewritten to use _GetTextInside (and use inferior regexp matching today). + + # Give opening punctuations to get the matching close-punctuations. + matching_punctuation = {'(': ')', '{': '}', '[': ']'} + closing_punctuation = set(itervalues(matching_punctuation)) + + # Find the position to start extracting text. + match = re.search(start_pattern, text, re.M) + if not match: # start_pattern not found in text. + return None + start_position = match.end(0) + + assert start_position > 0, ( + 'start_pattern must ends with an opening punctuation.') + assert text[start_position - 1] in matching_punctuation, ( + 'start_pattern must ends with an opening punctuation.') + # Stack of closing punctuations we expect to have in text after position. + punctuation_stack = [matching_punctuation[text[start_position - 1]]] + position = start_position + while punctuation_stack and position < len(text): + if text[position] == punctuation_stack[-1]: + punctuation_stack.pop() + elif text[position] in closing_punctuation: + # A closing punctuation without matching opening punctuations. + return None + elif text[position] in matching_punctuation: + punctuation_stack.append(matching_punctuation[text[position]]) + position += 1 + if punctuation_stack: + # Opening punctuations left without matching close-punctuations. + return None + # punctuations match. + return text[start_position:position - 1] + + +# Patterns for matching call-by-reference parameters. +# +# Supports nested templates up to 2 levels deep using this messy pattern: +# < (?: < (?: < [^<>]* +# > +# | [^<>] )* +# > +# | [^<>] )* +# > +_RE_PATTERN_IDENT = r'[_a-zA-Z]\w*' # =~ [[:alpha:]][[:alnum:]]* +_RE_PATTERN_TYPE = ( + r'(?:const\s+)?(?:typename\s+|class\s+|struct\s+|union\s+|enum\s+)?' + r'(?:\w|' + r'\s*<(?:<(?:<[^<>]*>|[^<>])*>|[^<>])*>|' + r'::)+') +# A call-by-reference parameter ends with '& identifier'. +_RE_PATTERN_REF_PARAM = re.compile( + r'(' + _RE_PATTERN_TYPE + r'(?:\s*(?:\bconst\b|[*]))*\s*' + r'&\s*' + _RE_PATTERN_IDENT + r')\s*(?:=[^,()]+)?[,)]') +# A call-by-const-reference parameter either ends with 'const& identifier' +# or looks like 'const type& identifier' when 'type' is atomic. +_RE_PATTERN_CONST_REF_PARAM = ( + r'(?:.*\s*\bconst\s*&\s*' + _RE_PATTERN_IDENT + + r'|const\s+' + _RE_PATTERN_TYPE + r'\s*&\s*' + _RE_PATTERN_IDENT + r')') +# Stream types. +_RE_PATTERN_REF_STREAM_PARAM = ( + r'(?:.*stream\s*&\s*' + _RE_PATTERN_IDENT + r')') + + +def CheckLanguage(filename, clean_lines, linenum, file_extension, + include_state, nesting_state, error): + """Checks rules from the 'C++ language rules' section of cppguide.html. + + Some of these rules are hard to test (function overloading, using + uint32 inappropriately), but we do the best we can. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + file_extension: The extension (without the dot) of the filename. + include_state: An _IncludeState instance in which the headers are inserted. + nesting_state: A NestingState instance which maintains information about + the current stack of nested blocks being parsed. + error: The function to call with any errors found. + """ + # If the line is empty or consists of entirely a comment, no need to + # check it. + line = clean_lines.elided[linenum] + if not line: + return + + match = _RE_PATTERN_INCLUDE.search(line) + if match: + CheckIncludeLine(filename, clean_lines, linenum, include_state, error) + return + + # Reset include state across preprocessor directives. This is meant + # to silence warnings for conditional includes. + match = Match(r'^\s*#\s*(if|ifdef|ifndef|elif|else|endif)\b', line) + if match: + include_state.ResetSection(match.group(1)) + + + # Perform other checks now that we are sure that this is not an include line + CheckCasts(filename, clean_lines, linenum, error) + CheckGlobalStatic(filename, clean_lines, linenum, error) + CheckPrintf(filename, clean_lines, linenum, error) + + if file_extension in GetHeaderExtensions(): + # TODO(unknown): check that 1-arg constructors are explicit. + # How to tell it's a constructor? + # (handled in CheckForNonStandardConstructs for now) + # TODO(unknown): check that classes declare or disable copy/assign + # (level 1 error) + pass + + # Check if people are using the verboten C basic types. The only exception + # we regularly allow is "unsigned short port" for port. + if Search(r'\bshort port\b', line): + if not Search(r'\bunsigned short port\b', line): + error(filename, linenum, 'runtime/int', 4, + 'Use "unsigned short" for ports, not "short"') + else: + match = Search(r'\b(short|long(?! +double)|long long)\b', line) + if match: + error(filename, linenum, 'runtime/int', 4, + 'Use int16/int64/etc, rather than the C type %s' % match.group(1)) + + # Check if some verboten operator overloading is going on + # TODO(unknown): catch out-of-line unary operator&: + # class X {}; + # int operator&(const X& x) { return 42; } // unary operator& + # The trick is it's hard to tell apart from binary operator&: + # class Y { int operator&(const Y& x) { return 23; } }; // binary operator& + if Search(r'\boperator\s*&\s*\(\s*\)', line): + error(filename, linenum, 'runtime/operator', 4, + 'Unary operator& is dangerous. Do not use it.') + + # Check for suspicious usage of "if" like + # } if (a == b) { + if Search(r'\}\s*if\s*\(', line): + error(filename, linenum, 'readability/braces', 4, + 'Did you mean "else if"? If not, start a new line for "if".') + + # Check for potential format string bugs like printf(foo). + # We constrain the pattern not to pick things like DocidForPrintf(foo). + # Not perfect but it can catch printf(foo.c_str()) and printf(foo->c_str()) + # TODO(unknown): Catch the following case. Need to change the calling + # convention of the whole function to process multiple line to handle it. + # printf( + # boy_this_is_a_really_long_variable_that_cannot_fit_on_the_prev_line); + printf_args = _GetTextInside(line, r'(?i)\b(string)?printf\s*\(') + if printf_args: + match = Match(r'([\w.\->()]+)$', printf_args) + if match and match.group(1) != '__VA_ARGS__': + function_name = re.search(r'\b((?:string)?printf)\s*\(', + line, re.I).group(1) + error(filename, linenum, 'runtime/printf', 4, + 'Potential format string bug. Do %s("%%s", %s) instead.' + % (function_name, match.group(1))) + + # Check for potential memset bugs like memset(buf, sizeof(buf), 0). + match = Search(r'memset\s*\(([^,]*),\s*([^,]*),\s*0\s*\)', line) + if match and not Match(r"^''|-?[0-9]+|0x[0-9A-Fa-f]$", match.group(2)): + error(filename, linenum, 'runtime/memset', 4, + 'Did you mean "memset(%s, 0, %s)"?' + % (match.group(1), match.group(2))) + + if Search(r'\busing namespace\b', line): + if Search(r'\bliterals\b', line): + error(filename, linenum, 'build/namespaces_literals', 5, + 'Do not use namespace using-directives. ' + 'Use using-declarations instead.') + else: + error(filename, linenum, 'build/namespaces', 5, + 'Do not use namespace using-directives. ' + 'Use using-declarations instead.') + + # Detect variable-length arrays. + match = Match(r'\s*(.+::)?(\w+) [a-z]\w*\[(.+)];', line) + if (match and match.group(2) != 'return' and match.group(2) != 'delete' and + match.group(3).find(']') == -1): + # Split the size using space and arithmetic operators as delimiters. + # If any of the resulting tokens are not compile time constants then + # report the error. + tokens = re.split(r'\s|\+|\-|\*|\/|<<|>>]', match.group(3)) + is_const = True + skip_next = False + for tok in tokens: + if skip_next: + skip_next = False + continue + + if Search(r'sizeof\(.+\)', tok): continue + if Search(r'arraysize\(\w+\)', tok): continue + + tok = tok.lstrip('(') + tok = tok.rstrip(')') + if not tok: continue + if Match(r'\d+', tok): continue + if Match(r'0[xX][0-9a-fA-F]+', tok): continue + if Match(r'k[A-Z0-9]\w*', tok): continue + if Match(r'(.+::)?k[A-Z0-9]\w*', tok): continue + if Match(r'(.+::)?[A-Z][A-Z0-9_]*', tok): continue + # A catch all for tricky sizeof cases, including 'sizeof expression', + # 'sizeof(*type)', 'sizeof(const type)', 'sizeof(struct StructName)' + # requires skipping the next token because we split on ' ' and '*'. + if tok.startswith('sizeof'): + skip_next = True + continue + is_const = False + break + if not is_const: + error(filename, linenum, 'runtime/arrays', 1, + 'Do not use variable-length arrays. Use an appropriately named ' + "('k' followed by CamelCase) compile-time constant for the size.") + + # Check for use of unnamed namespaces in header files. Registration + # macros are typically OK, so we allow use of "namespace {" on lines + # that end with backslashes. + if (file_extension in GetHeaderExtensions() + and Search(r'\bnamespace\s*{', line) + and line[-1] != '\\'): + error(filename, linenum, 'build/namespaces', 4, + 'Do not use unnamed namespaces in header files. See ' + 'https://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Namespaces' + ' for more information.') + + +def CheckGlobalStatic(filename, clean_lines, linenum, error): + """Check for unsafe global or static objects. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Match two lines at a time to support multiline declarations + if linenum + 1 < clean_lines.NumLines() and not Search(r'[;({]', line): + line += clean_lines.elided[linenum + 1].strip() + + # Check for people declaring static/global STL strings at the top level. + # This is dangerous because the C++ language does not guarantee that + # globals with constructors are initialized before the first access, and + # also because globals can be destroyed when some threads are still running. + # TODO(unknown): Generalize this to also find static unique_ptr instances. + # TODO(unknown): File bugs for clang-tidy to find these. + match = Match( + r'((?:|static +)(?:|const +))(?::*std::)?string( +const)? +' + r'([a-zA-Z0-9_:]+)\b(.*)', + line) + + # Remove false positives: + # - String pointers (as opposed to values). + # string *pointer + # const string *pointer + # string const *pointer + # string *const pointer + # + # - Functions and template specializations. + # string Function(... + # string Class::Method(... + # + # - Operators. These are matched separately because operator names + # cross non-word boundaries, and trying to match both operators + # and functions at the same time would decrease accuracy of + # matching identifiers. + # string Class::operator*() + if (match and + not Search(r'\bstring\b(\s+const)?\s*[\*\&]\s*(const\s+)?\w', line) and + not Search(r'\boperator\W', line) and + not Match(r'\s*(<.*>)?(::[a-zA-Z0-9_]+)*\s*\(([^"]|$)', match.group(4))): + if Search(r'\bconst\b', line): + error(filename, linenum, 'runtime/string', 4, + 'For a static/global string constant, use a C style string ' + 'instead: "%schar%s %s[]".' % + (match.group(1), match.group(2) or '', match.group(3))) + else: + error(filename, linenum, 'runtime/string', 4, + 'Static/global string variables are not permitted.') + + if (Search(r'\b([A-Za-z0-9_]*_)\(\1\)', line) or + Search(r'\b([A-Za-z0-9_]*_)\(CHECK_NOTNULL\(\1\)\)', line)): + error(filename, linenum, 'runtime/init', 4, + 'You seem to be initializing a member variable with itself.') + + +def CheckPrintf(filename, clean_lines, linenum, error): + """Check for printf related issues. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # When snprintf is used, the second argument shouldn't be a literal. + match = Search(r'snprintf\s*\(([^,]*),\s*([0-9]*)\s*,', line) + if match and match.group(2) != '0': + # If 2nd arg is zero, snprintf is used to calculate size. + error(filename, linenum, 'runtime/printf', 3, + 'If you can, use sizeof(%s) instead of %s as the 2nd arg ' + 'to snprintf.' % (match.group(1), match.group(2))) + + # Check if some verboten C functions are being used. + if Search(r'\bsprintf\s*\(', line): + error(filename, linenum, 'runtime/printf', 5, + 'Never use sprintf. Use snprintf instead.') + match = Search(r'\b(strcpy|strcat)\s*\(', line) + if match: + error(filename, linenum, 'runtime/printf', 4, + 'Almost always, snprintf is better than %s' % match.group(1)) + + +def IsDerivedFunction(clean_lines, linenum): + """Check if current line contains an inherited function. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + Returns: + True if current line contains a function with "override" + virt-specifier. + """ + # Scan back a few lines for start of current function + for i in xrange(linenum, max(-1, linenum - 10), -1): + match = Match(r'^([^()]*\w+)\(', clean_lines.elided[i]) + if match: + # Look for "override" after the matching closing parenthesis + line, _, closing_paren = CloseExpression( + clean_lines, i, len(match.group(1))) + return (closing_paren >= 0 and + Search(r'\boverride\b', line[closing_paren:])) + return False + + +def IsOutOfLineMethodDefinition(clean_lines, linenum): + """Check if current line contains an out-of-line method definition. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + Returns: + True if current line contains an out-of-line method definition. + """ + # Scan back a few lines for start of current function + for i in xrange(linenum, max(-1, linenum - 10), -1): + if Match(r'^([^()]*\w+)\(', clean_lines.elided[i]): + return Match(r'^[^()]*\w+::\w+\(', clean_lines.elided[i]) is not None + return False + + +def IsInitializerList(clean_lines, linenum): + """Check if current line is inside constructor initializer list. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + Returns: + True if current line appears to be inside constructor initializer + list, False otherwise. + """ + for i in xrange(linenum, 1, -1): + line = clean_lines.elided[i] + if i == linenum: + remove_function_body = Match(r'^(.*)\{\s*$', line) + if remove_function_body: + line = remove_function_body.group(1) + + if Search(r'\s:\s*\w+[({]', line): + # A lone colon tend to indicate the start of a constructor + # initializer list. It could also be a ternary operator, which + # also tend to appear in constructor initializer lists as + # opposed to parameter lists. + return True + if Search(r'\}\s*,\s*$', line): + # A closing brace followed by a comma is probably the end of a + # brace-initialized member in constructor initializer list. + return True + if Search(r'[{};]\s*$', line): + # Found one of the following: + # - A closing brace or semicolon, probably the end of the previous + # function. + # - An opening brace, probably the start of current class or namespace. + # + # Current line is probably not inside an initializer list since + # we saw one of those things without seeing the starting colon. + return False + + # Got to the beginning of the file without seeing the start of + # constructor initializer list. + return False + + +def CheckForNonConstReference(filename, clean_lines, linenum, + nesting_state, error): + """Check for non-const references. + + Separate from CheckLanguage since it scans backwards from current + line, instead of scanning forward. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + nesting_state: A NestingState instance which maintains information about + the current stack of nested blocks being parsed. + error: The function to call with any errors found. + """ + # Do nothing if there is no '&' on current line. + line = clean_lines.elided[linenum] + if '&' not in line: + return + + # If a function is inherited, current function doesn't have much of + # a choice, so any non-const references should not be blamed on + # derived function. + if IsDerivedFunction(clean_lines, linenum): + return + + # Don't warn on out-of-line method definitions, as we would warn on the + # in-line declaration, if it isn't marked with 'override'. + if IsOutOfLineMethodDefinition(clean_lines, linenum): + return + + # Long type names may be broken across multiple lines, usually in one + # of these forms: + # LongType + # ::LongTypeContinued &identifier + # LongType:: + # LongTypeContinued &identifier + # LongType< + # ...>::LongTypeContinued &identifier + # + # If we detected a type split across two lines, join the previous + # line to current line so that we can match const references + # accordingly. + # + # Note that this only scans back one line, since scanning back + # arbitrary number of lines would be expensive. If you have a type + # that spans more than 2 lines, please use a typedef. + if linenum > 1: + previous = None + if Match(r'\s*::(?:[\w<>]|::)+\s*&\s*\S', line): + # previous_line\n + ::current_line + previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+[\w<>])\s*$', + clean_lines.elided[linenum - 1]) + elif Match(r'\s*[a-zA-Z_]([\w<>]|::)+\s*&\s*\S', line): + # previous_line::\n + current_line + previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+::)\s*$', + clean_lines.elided[linenum - 1]) + if previous: + line = previous.group(1) + line.lstrip() + else: + # Check for templated parameter that is split across multiple lines + endpos = line.rfind('>') + if endpos > -1: + (_, startline, startpos) = ReverseCloseExpression( + clean_lines, linenum, endpos) + if startpos > -1 and startline < linenum: + # Found the matching < on an earlier line, collect all + # pieces up to current line. + line = '' + for i in xrange(startline, linenum + 1): + line += clean_lines.elided[i].strip() + + # Check for non-const references in function parameters. A single '&' may + # found in the following places: + # inside expression: binary & for bitwise AND + # inside expression: unary & for taking the address of something + # inside declarators: reference parameter + # We will exclude the first two cases by checking that we are not inside a + # function body, including one that was just introduced by a trailing '{'. + # TODO(unknown): Doesn't account for 'catch(Exception& e)' [rare]. + if (nesting_state.previous_stack_top and + not (isinstance(nesting_state.previous_stack_top, _ClassInfo) or + isinstance(nesting_state.previous_stack_top, _NamespaceInfo))): + # Not at toplevel, not within a class, and not within a namespace + return + + # Avoid initializer lists. We only need to scan back from the + # current line for something that starts with ':'. + # + # We don't need to check the current line, since the '&' would + # appear inside the second set of parentheses on the current line as + # opposed to the first set. + if linenum > 0: + for i in xrange(linenum - 1, max(0, linenum - 10), -1): + previous_line = clean_lines.elided[i] + if not Search(r'[),]\s*$', previous_line): + break + if Match(r'^\s*:\s+\S', previous_line): + return + + # Avoid preprocessors + if Search(r'\\\s*$', line): + return + + # Avoid constructor initializer lists + if IsInitializerList(clean_lines, linenum): + return + + # We allow non-const references in a few standard places, like functions + # called "swap()" or iostream operators like "<<" or ">>". Do not check + # those function parameters. + # + # We also accept & in static_assert, which looks like a function but + # it's actually a declaration expression. + whitelisted_functions = (r'(?:[sS]wap(?:<\w:+>)?|' + r'operator\s*[<>][<>]|' + r'static_assert|COMPILE_ASSERT' + r')\s*\(') + if Search(whitelisted_functions, line): + return + elif not Search(r'\S+\([^)]*$', line): + # Don't see a whitelisted function on this line. Actually we + # didn't see any function name on this line, so this is likely a + # multi-line parameter list. Try a bit harder to catch this case. + for i in xrange(2): + if (linenum > i and + Search(whitelisted_functions, clean_lines.elided[linenum - i - 1])): + return + + decls = ReplaceAll(r'{[^}]*}', ' ', line) # exclude function body + for parameter in re.findall(_RE_PATTERN_REF_PARAM, decls): + if (not Match(_RE_PATTERN_CONST_REF_PARAM, parameter) and + not Match(_RE_PATTERN_REF_STREAM_PARAM, parameter)): + error(filename, linenum, 'runtime/references', 2, + 'Is this a non-const reference? ' + 'If so, make const or use a pointer: ' + + ReplaceAll(' *<', '<', parameter)) + + +def CheckCasts(filename, clean_lines, linenum, error): + """Various cast related checks. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Check to see if they're using an conversion function cast. + # I just try to capture the most common basic types, though there are more. + # Parameterless conversion functions, such as bool(), are allowed as they are + # probably a member operator declaration or default constructor. + match = Search( + r'(\bnew\s+(?:const\s+)?|\S<\s*(?:const\s+)?)?\b' + r'(int|float|double|bool|char|int32|uint32|int64|uint64)' + r'(\([^)].*)', line) + expecting_function = ExpectingFunctionArgs(clean_lines, linenum) + if match and not expecting_function: + matched_type = match.group(2) + + # matched_new_or_template is used to silence two false positives: + # - New operators + # - Template arguments with function types + # + # For template arguments, we match on types immediately following + # an opening bracket without any spaces. This is a fast way to + # silence the common case where the function type is the first + # template argument. False negative with less-than comparison is + # avoided because those operators are usually followed by a space. + # + # function // bracket + no space = false positive + # value < double(42) // bracket + space = true positive + matched_new_or_template = match.group(1) + + # Avoid arrays by looking for brackets that come after the closing + # parenthesis. + if Match(r'\([^()]+\)\s*\[', match.group(3)): + return + + # Other things to ignore: + # - Function pointers + # - Casts to pointer types + # - Placement new + # - Alias declarations + matched_funcptr = match.group(3) + if (matched_new_or_template is None and + not (matched_funcptr and + (Match(r'\((?:[^() ]+::\s*\*\s*)?[^() ]+\)\s*\(', + matched_funcptr) or + matched_funcptr.startswith('(*)'))) and + not Match(r'\s*using\s+\S+\s*=\s*' + matched_type, line) and + not Search(r'new\(\S+\)\s*' + matched_type, line)): + error(filename, linenum, 'readability/casting', 4, + 'Using deprecated casting style. ' + 'Use static_cast<%s>(...) instead' % + matched_type) + + if not expecting_function: + CheckCStyleCast(filename, clean_lines, linenum, 'static_cast', + r'\((int|float|double|bool|char|u?int(16|32|64))\)', error) + + # This doesn't catch all cases. Consider (const char * const)"hello". + # + # (char *) "foo" should always be a const_cast (reinterpret_cast won't + # compile). + if CheckCStyleCast(filename, clean_lines, linenum, 'const_cast', + r'\((char\s?\*+\s?)\)\s*"', error): + pass + else: + # Check pointer casts for other than string constants + CheckCStyleCast(filename, clean_lines, linenum, 'reinterpret_cast', + r'\((\w+\s?\*+\s?)\)', error) + + # In addition, we look for people taking the address of a cast. This + # is dangerous -- casts can assign to temporaries, so the pointer doesn't + # point where you think. + # + # Some non-identifier character is required before the '&' for the + # expression to be recognized as a cast. These are casts: + # expression = &static_cast(temporary()); + # function(&(int*)(temporary())); + # + # This is not a cast: + # reference_type&(int* function_param); + match = Search( + r'(?:[^\w]&\(([^)*][^)]*)\)[\w(])|' + r'(?:[^\w]&(static|dynamic|down|reinterpret)_cast\b)', line) + if match: + # Try a better error message when the & is bound to something + # dereferenced by the casted pointer, as opposed to the casted + # pointer itself. + parenthesis_error = False + match = Match(r'^(.*&(?:static|dynamic|down|reinterpret)_cast\b)<', line) + if match: + _, y1, x1 = CloseExpression(clean_lines, linenum, len(match.group(1))) + if x1 >= 0 and clean_lines.elided[y1][x1] == '(': + _, y2, x2 = CloseExpression(clean_lines, y1, x1) + if x2 >= 0: + extended_line = clean_lines.elided[y2][x2:] + if y2 < clean_lines.NumLines() - 1: + extended_line += clean_lines.elided[y2 + 1] + if Match(r'\s*(?:->|\[)', extended_line): + parenthesis_error = True + + if parenthesis_error: + error(filename, linenum, 'readability/casting', 4, + ('Are you taking an address of something dereferenced ' + 'from a cast? Wrapping the dereferenced expression in ' + 'parentheses will make the binding more obvious')) + else: + error(filename, linenum, 'runtime/casting', 4, + ('Are you taking an address of a cast? ' + 'This is dangerous: could be a temp var. ' + 'Take the address before doing the cast, rather than after')) + + +def CheckCStyleCast(filename, clean_lines, linenum, cast_type, pattern, error): + """Checks for a C-style cast by looking for the pattern. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + cast_type: The string for the C++ cast to recommend. This is either + reinterpret_cast, static_cast, or const_cast, depending. + pattern: The regular expression used to find C-style casts. + error: The function to call with any errors found. + + Returns: + True if an error was emitted. + False otherwise. + """ + line = clean_lines.elided[linenum] + match = Search(pattern, line) + if not match: + return False + + # Exclude lines with keywords that tend to look like casts + context = line[0:match.start(1) - 1] + if Match(r'.*\b(?:sizeof|alignof|alignas|[_A-Z][_A-Z0-9]*)\s*$', context): + return False + + # Try expanding current context to see if we one level of + # parentheses inside a macro. + if linenum > 0: + for i in xrange(linenum - 1, max(0, linenum - 5), -1): + context = clean_lines.elided[i] + context + if Match(r'.*\b[_A-Z][_A-Z0-9]*\s*\((?:\([^()]*\)|[^()])*$', context): + return False + + # operator++(int) and operator--(int) + if context.endswith(' operator++') or context.endswith(' operator--'): + return False + + # A single unnamed argument for a function tends to look like old style cast. + # If we see those, don't issue warnings for deprecated casts. + remainder = line[match.end(0):] + if Match(r'^\s*(?:;|const\b|throw\b|final\b|override\b|[=>{),]|->)', + remainder): + return False + + # At this point, all that should be left is actual casts. + error(filename, linenum, 'readability/casting', 4, + 'Using C-style cast. Use %s<%s>(...) instead' % + (cast_type, match.group(1))) + + return True + + +def ExpectingFunctionArgs(clean_lines, linenum): + """Checks whether where function type arguments are expected. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + + Returns: + True if the line at 'linenum' is inside something that expects arguments + of function types. + """ + line = clean_lines.elided[linenum] + return (Match(r'^\s*MOCK_(CONST_)?METHOD\d+(_T)?\(', line) or + (linenum >= 2 and + (Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\((?:\S+,)?\s*$', + clean_lines.elided[linenum - 1]) or + Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\(\s*$', + clean_lines.elided[linenum - 2]) or + Search(r'\bstd::m?function\s*\<\s*$', + clean_lines.elided[linenum - 1])))) + + +_HEADERS_CONTAINING_TEMPLATES = ( + ('', ('deque',)), + ('', ('unary_function', 'binary_function', + 'plus', 'minus', 'multiplies', 'divides', 'modulus', + 'negate', + 'equal_to', 'not_equal_to', 'greater', 'less', + 'greater_equal', 'less_equal', + 'logical_and', 'logical_or', 'logical_not', + 'unary_negate', 'not1', 'binary_negate', 'not2', + 'bind1st', 'bind2nd', + 'pointer_to_unary_function', + 'pointer_to_binary_function', + 'ptr_fun', + 'mem_fun_t', 'mem_fun', 'mem_fun1_t', 'mem_fun1_ref_t', + 'mem_fun_ref_t', + 'const_mem_fun_t', 'const_mem_fun1_t', + 'const_mem_fun_ref_t', 'const_mem_fun1_ref_t', + 'mem_fun_ref', + )), + ('', ('numeric_limits',)), + ('', ('list',)), + ('', ('map', 'multimap',)), + ('', ('allocator', 'make_shared', 'make_unique', 'shared_ptr', + 'unique_ptr', 'weak_ptr')), + ('', ('queue', 'priority_queue',)), + ('', ('set', 'multiset',)), + ('', ('stack',)), + ('', ('char_traits', 'basic_string',)), + ('', ('tuple',)), + ('', ('unordered_map', 'unordered_multimap')), + ('', ('unordered_set', 'unordered_multiset')), + ('', ('pair',)), + ('', ('vector',)), + + # gcc extensions. + # Note: std::hash is their hash, ::hash is our hash + ('', ('hash_map', 'hash_multimap',)), + ('', ('hash_set', 'hash_multiset',)), + ('', ('slist',)), + ) + +_HEADERS_MAYBE_TEMPLATES = ( + ('', ('copy', 'max', 'min', 'min_element', 'sort', + 'transform', + )), + ('', ('forward', 'make_pair', 'move', 'swap')), + ) + +_RE_PATTERN_STRING = re.compile(r'\bstring\b') + +_re_pattern_headers_maybe_templates = [] +for _header, _templates in _HEADERS_MAYBE_TEMPLATES: + for _template in _templates: + # Match max(..., ...), max(..., ...), but not foo->max, foo.max or + # type::max(). + _re_pattern_headers_maybe_templates.append( + (re.compile(r'[^>.]\b' + _template + r'(<.*?>)?\([^\)]'), + _template, + _header)) + +# Other scripts may reach in and modify this pattern. +_re_pattern_templates = [] +for _header, _templates in _HEADERS_CONTAINING_TEMPLATES: + for _template in _templates: + _re_pattern_templates.append( + (re.compile(r'(\<|\b)' + _template + r'\s*\<'), + _template + '<>', + _header)) + + +def FilesBelongToSameModule(filename_cc, filename_h): + """Check if these two filenames belong to the same module. + + The concept of a 'module' here is a as follows: + foo.h, foo-inl.h, foo.cc, foo_test.cc and foo_unittest.cc belong to the + same 'module' if they are in the same directory. + some/path/public/xyzzy and some/path/internal/xyzzy are also considered + to belong to the same module here. + + If the filename_cc contains a longer path than the filename_h, for example, + '/absolute/path/to/base/sysinfo.cc', and this file would include + 'base/sysinfo.h', this function also produces the prefix needed to open the + header. This is used by the caller of this function to more robustly open the + header file. We don't have access to the real include paths in this context, + so we need this guesswork here. + + Known bugs: tools/base/bar.cc and base/bar.h belong to the same module + according to this implementation. Because of this, this function gives + some false positives. This should be sufficiently rare in practice. + + Args: + filename_cc: is the path for the source (e.g. .cc) file + filename_h: is the path for the header path + + Returns: + Tuple with a bool and a string: + bool: True if filename_cc and filename_h belong to the same module. + string: the additional prefix needed to open the header file. + """ + fileinfo_cc = FileInfo(filename_cc) + if not fileinfo_cc.Extension().lstrip('.') in GetNonHeaderExtensions(): + return (False, '') + + fileinfo_h = FileInfo(filename_h) + if not fileinfo_h.Extension().lstrip('.') in GetHeaderExtensions(): + return (False, '') + + filename_cc = filename_cc[:-(len(fileinfo_cc.Extension()))] + matched_test_suffix = Search(_TEST_FILE_SUFFIX, fileinfo_cc.BaseName()) + if matched_test_suffix: + filename_cc = filename_cc[:-len(matched_test_suffix.group(1))] + + filename_cc = filename_cc.replace('/public/', '/') + filename_cc = filename_cc.replace('/internal/', '/') + + filename_h = filename_h[:-(len(fileinfo_h.Extension()))] + if filename_h.endswith('-inl'): + filename_h = filename_h[:-len('-inl')] + filename_h = filename_h.replace('/public/', '/') + filename_h = filename_h.replace('/internal/', '/') + + files_belong_to_same_module = filename_cc.endswith(filename_h) + common_path = '' + if files_belong_to_same_module: + common_path = filename_cc[:-len(filename_h)] + return files_belong_to_same_module, common_path + + +def UpdateIncludeState(filename, include_dict, io=codecs): + """Fill up the include_dict with new includes found from the file. + + Args: + filename: the name of the header to read. + include_dict: a dictionary in which the headers are inserted. + io: The io factory to use to read the file. Provided for testability. + + Returns: + True if a header was successfully added. False otherwise. + """ + headerfile = None + try: + headerfile = io.open(filename, 'r', 'utf8', 'replace') + except IOError: + return False + linenum = 0 + for line in headerfile: + linenum += 1 + clean_line = CleanseComments(line) + match = _RE_PATTERN_INCLUDE.search(clean_line) + if match: + include = match.group(2) + include_dict.setdefault(include, linenum) + return True + + +def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error, + io=codecs): + """Reports for missing stl includes. + + This function will output warnings to make sure you are including the headers + necessary for the stl containers and functions that you use. We only give one + reason to include a header. For example, if you use both equal_to<> and + less<> in a .h file, only one (the latter in the file) of these will be + reported as a reason to include the . + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + include_state: An _IncludeState instance. + error: The function to call with any errors found. + io: The IO factory to use to read the header file. Provided for unittest + injection. + """ + required = {} # A map of header name to linenumber and the template entity. + # Example of required: { '': (1219, 'less<>') } + + for linenum in range(clean_lines.NumLines()): + line = clean_lines.elided[linenum] + if not line or line[0] == '#': + continue + + # String is special -- it is a non-templatized type in STL. + matched = _RE_PATTERN_STRING.search(line) + if matched: + # Don't warn about strings in non-STL namespaces: + # (We check only the first match per line; good enough.) + prefix = line[:matched.start()] + if prefix.endswith('std::') or not prefix.endswith('::'): + required[''] = (linenum, 'string') + + for pattern, template, header in _re_pattern_headers_maybe_templates: + if pattern.search(line): + required[header] = (linenum, template) + + # The following function is just a speed up, no semantics are changed. + if not '<' in line: # Reduces the cpu time usage by skipping lines. + continue + + for pattern, template, header in _re_pattern_templates: + matched = pattern.search(line) + if matched: + # Don't warn about IWYU in non-STL namespaces: + # (We check only the first match per line; good enough.) + prefix = line[:matched.start()] + if prefix.endswith('std::') or not prefix.endswith('::'): + required[header] = (linenum, template) + + # The policy is that if you #include something in foo.h you don't need to + # include it again in foo.cc. Here, we will look at possible includes. + # Let's flatten the include_state include_list and copy it into a dictionary. + include_dict = dict([item for sublist in include_state.include_list + for item in sublist]) + + # Did we find the header for this file (if any) and successfully load it? + header_found = False + + # Use the absolute path so that matching works properly. + abs_filename = FileInfo(filename).FullName() + + # For Emacs's flymake. + # If cpplint is invoked from Emacs's flymake, a temporary file is generated + # by flymake and that file name might end with '_flymake.cc'. In that case, + # restore original file name here so that the corresponding header file can be + # found. + # e.g. If the file name is 'foo_flymake.cc', we should search for 'foo.h' + # instead of 'foo_flymake.h' + abs_filename = re.sub(r'_flymake\.cc$', '.cc', abs_filename) + + # include_dict is modified during iteration, so we iterate over a copy of + # the keys. + header_keys = list(include_dict.keys()) + for header in header_keys: + (same_module, common_path) = FilesBelongToSameModule(abs_filename, header) + fullpath = common_path + header + if same_module and UpdateIncludeState(fullpath, include_dict, io): + header_found = True + + # If we can't find the header file for a .cc, assume it's because we don't + # know where to look. In that case we'll give up as we're not sure they + # didn't include it in the .h file. + # TODO(unknown): Do a better job of finding .h files so we are confident that + # not having the .h file means there isn't one. + if not header_found: + for extension in GetNonHeaderExtensions(): + if filename.endswith('.' + extension): + return + + # All the lines have been processed, report the errors found. + for required_header_unstripped in sorted(required, key=required.__getitem__): + template = required[required_header_unstripped][1] + if required_header_unstripped.strip('<>"') not in include_dict: + error(filename, required[required_header_unstripped][0], + 'build/include_what_you_use', 4, + 'Add #include ' + required_header_unstripped + ' for ' + template) + + +_RE_PATTERN_EXPLICIT_MAKEPAIR = re.compile(r'\bmake_pair\s*<') + + +def CheckMakePairUsesDeduction(filename, clean_lines, linenum, error): + """Check that make_pair's template arguments are deduced. + + G++ 4.6 in C++11 mode fails badly if make_pair's template arguments are + specified explicitly, and such use isn't intended in any case. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + match = _RE_PATTERN_EXPLICIT_MAKEPAIR.search(line) + if match: + error(filename, linenum, 'build/explicit_make_pair', + 4, # 4 = high confidence + 'For C++11-compatibility, omit template arguments from make_pair' + ' OR use pair directly OR if appropriate, construct a pair directly') + + +def CheckRedundantVirtual(filename, clean_lines, linenum, error): + """Check if line contains a redundant "virtual" function-specifier. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + # Look for "virtual" on current line. + line = clean_lines.elided[linenum] + virtual = Match(r'^(.*)(\bvirtual\b)(.*)$', line) + if not virtual: return + + # Ignore "virtual" keywords that are near access-specifiers. These + # are only used in class base-specifier and do not apply to member + # functions. + if (Search(r'\b(public|protected|private)\s+$', virtual.group(1)) or + Match(r'^\s+(public|protected|private)\b', virtual.group(3))): + return + + # Ignore the "virtual" keyword from virtual base classes. Usually + # there is a column on the same line in these cases (virtual base + # classes are rare in google3 because multiple inheritance is rare). + if Match(r'^.*[^:]:[^:].*$', line): return + + # Look for the next opening parenthesis. This is the start of the + # parameter list (possibly on the next line shortly after virtual). + # TODO(unknown): doesn't work if there are virtual functions with + # decltype() or other things that use parentheses, but csearch suggests + # that this is rare. + end_col = -1 + end_line = -1 + start_col = len(virtual.group(2)) + for start_line in xrange(linenum, min(linenum + 3, clean_lines.NumLines())): + line = clean_lines.elided[start_line][start_col:] + parameter_list = Match(r'^([^(]*)\(', line) + if parameter_list: + # Match parentheses to find the end of the parameter list + (_, end_line, end_col) = CloseExpression( + clean_lines, start_line, start_col + len(parameter_list.group(1))) + break + start_col = 0 + + if end_col < 0: + return # Couldn't find end of parameter list, give up + + # Look for "override" or "final" after the parameter list + # (possibly on the next few lines). + for i in xrange(end_line, min(end_line + 3, clean_lines.NumLines())): + line = clean_lines.elided[i][end_col:] + match = Search(r'\b(override|final)\b', line) + if match: + error(filename, linenum, 'readability/inheritance', 4, + ('"virtual" is redundant since function is ' + 'already declared as "%s"' % match.group(1))) + + # Set end_col to check whole lines after we are done with the + # first line. + end_col = 0 + if Search(r'[^\w]\s*$', line): + break + + +def CheckRedundantOverrideOrFinal(filename, clean_lines, linenum, error): + """Check if line contains a redundant "override" or "final" virt-specifier. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + # Look for closing parenthesis nearby. We need one to confirm where + # the declarator ends and where the virt-specifier starts to avoid + # false positives. + line = clean_lines.elided[linenum] + declarator_end = line.rfind(')') + if declarator_end >= 0: + fragment = line[declarator_end:] + else: + if linenum > 1 and clean_lines.elided[linenum - 1].rfind(')') >= 0: + fragment = line + else: + return + + # Check that at most one of "override" or "final" is present, not both + if Search(r'\boverride\b', fragment) and Search(r'\bfinal\b', fragment): + error(filename, linenum, 'readability/inheritance', 4, + ('"override" is redundant since function is ' + 'already declared as "final"')) + + + + +# Returns true if we are at a new block, and it is directly +# inside of a namespace. +def IsBlockInNameSpace(nesting_state, is_forward_declaration): + """Checks that the new block is directly in a namespace. + + Args: + nesting_state: The _NestingState object that contains info about our state. + is_forward_declaration: If the class is a forward declared class. + Returns: + Whether or not the new block is directly in a namespace. + """ + if is_forward_declaration: + return len(nesting_state.stack) >= 1 and ( + isinstance(nesting_state.stack[-1], _NamespaceInfo)) + + + return (len(nesting_state.stack) > 1 and + nesting_state.stack[-1].check_namespace_indentation and + isinstance(nesting_state.stack[-2], _NamespaceInfo)) + + +def ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, + raw_lines_no_comments, linenum): + """This method determines if we should apply our namespace indentation check. + + Args: + nesting_state: The current nesting state. + is_namespace_indent_item: If we just put a new class on the stack, True. + If the top of the stack is not a class, or we did not recently + add the class, False. + raw_lines_no_comments: The lines without the comments. + linenum: The current line number we are processing. + + Returns: + True if we should apply our namespace indentation check. Currently, it + only works for classes and namespaces inside of a namespace. + """ + + is_forward_declaration = IsForwardClassDeclaration(raw_lines_no_comments, + linenum) + + if not (is_namespace_indent_item or is_forward_declaration): + return False + + # If we are in a macro, we do not want to check the namespace indentation. + if IsMacroDefinition(raw_lines_no_comments, linenum): + return False + + return IsBlockInNameSpace(nesting_state, is_forward_declaration) + + +# Call this method if the line is directly inside of a namespace. +# If the line above is blank (excluding comments) or the start of +# an inner namespace, it cannot be indented. +def CheckItemIndentationInNamespace(filename, raw_lines_no_comments, linenum, + error): + line = raw_lines_no_comments[linenum] + if Match(r'^\s+', line): + error(filename, linenum, 'runtime/indentation_namespace', 4, + 'Do not indent within a namespace') + + +def ProcessLine(filename, file_extension, clean_lines, line, + include_state, function_state, nesting_state, error, + extra_check_functions=None): + """Processes a single line in the file. + + Args: + filename: Filename of the file that is being processed. + file_extension: The extension (dot not included) of the file. + clean_lines: An array of strings, each representing a line of the file, + with comments stripped. + line: Number of line being processed. + include_state: An _IncludeState instance in which the headers are inserted. + function_state: A _FunctionState instance which counts function lines, etc. + nesting_state: A NestingState instance which maintains information about + the current stack of nested blocks being parsed. + error: A callable to which errors are reported, which takes 4 arguments: + filename, line number, error level, and message + extra_check_functions: An array of additional check functions that will be + run on each source line. Each function takes 4 + arguments: filename, clean_lines, line, error + """ + raw_lines = clean_lines.raw_lines + ParseNolintSuppressions(filename, raw_lines[line], line, error) + nesting_state.Update(filename, clean_lines, line, error) + CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line, + error) + if nesting_state.InAsmBlock(): return + CheckForFunctionLengths(filename, clean_lines, line, function_state, error) + CheckForMultilineCommentsAndStrings(filename, clean_lines, line, error) + CheckStyle(filename, clean_lines, line, file_extension, nesting_state, error) + CheckLanguage(filename, clean_lines, line, file_extension, include_state, + nesting_state, error) + CheckForNonConstReference(filename, clean_lines, line, nesting_state, error) + CheckForNonStandardConstructs(filename, clean_lines, line, + nesting_state, error) + CheckVlogArguments(filename, clean_lines, line, error) + CheckPosixThreading(filename, clean_lines, line, error) + CheckInvalidIncrement(filename, clean_lines, line, error) + CheckMakePairUsesDeduction(filename, clean_lines, line, error) + CheckRedundantVirtual(filename, clean_lines, line, error) + CheckRedundantOverrideOrFinal(filename, clean_lines, line, error) + if extra_check_functions: + for check_fn in extra_check_functions: + check_fn(filename, clean_lines, line, error) + +def FlagCxx11Features(filename, clean_lines, linenum, error): + """Flag those c++11 features that we only allow in certain places. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line) + + # Flag unapproved C++ TR1 headers. + if include and include.group(1).startswith('tr1/'): + error(filename, linenum, 'build/c++tr1', 5, + ('C++ TR1 headers such as <%s> are unapproved.') % include.group(1)) + + # Flag unapproved C++11 headers. + if include and include.group(1) in ('cfenv', + 'condition_variable', + 'fenv.h', + 'future', + 'mutex', + 'thread', + 'chrono', + 'ratio', + 'regex', + 'system_error', + ): + error(filename, linenum, 'build/c++11', 5, + ('<%s> is an unapproved C++11 header.') % include.group(1)) + + # The only place where we need to worry about C++11 keywords and library + # features in preprocessor directives is in macro definitions. + if Match(r'\s*#', line) and not Match(r'\s*#\s*define\b', line): return + + # These are classes and free functions. The classes are always + # mentioned as std::*, but we only catch the free functions if + # they're not found by ADL. They're alphabetical by header. + for top_name in ( + # type_traits + 'alignment_of', + 'aligned_union', + ): + if Search(r'\bstd::%s\b' % top_name, line): + error(filename, linenum, 'build/c++11', 5, + ('std::%s is an unapproved C++11 class or function. Send c-style ' + 'an example of where it would make your code more readable, and ' + 'they may let you use it.') % top_name) + + +def FlagCxx14Features(filename, clean_lines, linenum, error): + """Flag those C++14 features that we restrict. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line) + + # Flag unapproved C++14 headers. + if include and include.group(1) in ('scoped_allocator', 'shared_mutex'): + error(filename, linenum, 'build/c++14', 5, + ('<%s> is an unapproved C++14 header.') % include.group(1)) + + +def ProcessFileData(filename, file_extension, lines, error, + extra_check_functions=None): + """Performs lint checks and reports any errors to the given error function. + + Args: + filename: Filename of the file that is being processed. + file_extension: The extension (dot not included) of the file. + lines: An array of strings, each representing a line of the file, with the + last element being empty if the file is terminated with a newline. + error: A callable to which errors are reported, which takes 4 arguments: + filename, line number, error level, and message + extra_check_functions: An array of additional check functions that will be + run on each source line. Each function takes 4 + arguments: filename, clean_lines, line, error + """ + lines = (['// marker so line numbers and indices both start at 1'] + lines + + ['// marker so line numbers end in a known way']) + + include_state = _IncludeState() + function_state = _FunctionState() + nesting_state = NestingState() + + ResetNolintSuppressions() + + CheckForCopyright(filename, lines, error) + ProcessGlobalSuppresions(lines) + RemoveMultiLineComments(filename, lines, error) + clean_lines = CleansedLines(lines) + + if file_extension in GetHeaderExtensions(): + CheckForHeaderGuard(filename, clean_lines, error) + + for line in range(clean_lines.NumLines()): + ProcessLine(filename, file_extension, clean_lines, line, + include_state, function_state, nesting_state, error, + extra_check_functions) + FlagCxx11Features(filename, clean_lines, line, error) + nesting_state.CheckCompletedBlocks(filename, error) + + CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error) + + # Check that the .cc file has included its header if it exists. + if _IsSourceExtension(file_extension): + CheckHeaderFileIncluded(filename, include_state, error) + + # We check here rather than inside ProcessLine so that we see raw + # lines rather than "cleaned" lines. + CheckForBadCharacters(filename, lines, error) + + CheckForNewlineAtEOF(filename, lines, error) + +def ProcessConfigOverrides(filename): + """ Loads the configuration files and processes the config overrides. + + Args: + filename: The name of the file being processed by the linter. + + Returns: + False if the current |filename| should not be processed further. + """ + + abs_filename = os.path.abspath(filename) + cfg_filters = [] + keep_looking = True + while keep_looking: + abs_path, base_name = os.path.split(abs_filename) + if not base_name: + break # Reached the root directory. + + cfg_file = os.path.join(abs_path, "CPPLINT.cfg") + abs_filename = abs_path + if not os.path.isfile(cfg_file): + continue + + try: + with open(cfg_file) as file_handle: + for line in file_handle: + line, _, _ = line.partition('#') # Remove comments. + if not line.strip(): + continue + + name, _, val = line.partition('=') + name = name.strip() + val = val.strip() + if name == 'set noparent': + keep_looking = False + elif name == 'filter': + cfg_filters.append(val) + elif name == 'exclude_files': + # When matching exclude_files pattern, use the base_name of + # the current file name or the directory name we are processing. + # For example, if we are checking for lint errors in /foo/bar/baz.cc + # and we found the .cfg file at /foo/CPPLINT.cfg, then the config + # file's "exclude_files" filter is meant to be checked against "bar" + # and not "baz" nor "bar/baz.cc". + if base_name: + pattern = re.compile(val) + if pattern.match(base_name): + _cpplint_state.PrintInfo('Ignoring "%s": file excluded by ' + '"%s". File path component "%s" matches pattern "%s"\n' % + (filename, cfg_file, base_name, val)) + return False + elif name == 'linelength': + global _line_length + try: + _line_length = int(val) + except ValueError: + _cpplint_state.PrintError('Line length must be numeric.') + elif name == 'extensions': + global _valid_extensions + try: + extensions = [ext.strip() for ext in val.split(',')] + _valid_extensions = set(extensions) + except ValueError: + sys.stderr.write('Extensions should be a comma-separated list of values;' + 'for example: extensions=hpp,cpp\n' + 'This could not be parsed: "%s"' % (val,)) + elif name == 'headers': + global _header_extensions + try: + extensions = [ext.strip() for ext in val.split(',')] + _header_extensions = set(extensions) + except ValueError: + sys.stderr.write('Extensions should be a comma-separated list of values;' + 'for example: extensions=hpp,cpp\n' + 'This could not be parsed: "%s"' % (val,)) + elif name == 'root': + global _root + _root = val + else: + _cpplint_state.PrintError( + 'Invalid configuration option (%s) in file %s\n' % + (name, cfg_file)) + + except IOError: + _cpplint_state.PrintError( + "Skipping config file '%s': Can't open for reading\n" % cfg_file) + keep_looking = False + + # Apply all the accumulated filters in reverse order (top-level directory + # config options having the least priority). + for cfg_filter in reversed(cfg_filters): + _AddFilters(cfg_filter) + + return True + + +def ProcessFile(filename, vlevel, extra_check_functions=None): + """Does google-lint on a single file. + + Args: + filename: The name of the file to parse. + + vlevel: The level of errors to report. Every error of confidence + >= verbose_level will be reported. 0 is a good default. + + extra_check_functions: An array of additional check functions that will be + run on each source line. Each function takes 4 + arguments: filename, clean_lines, line, error + """ + + _SetVerboseLevel(vlevel) + _BackupFilters() + + if not ProcessConfigOverrides(filename): + _RestoreFilters() + return + + lf_lines = [] + crlf_lines = [] + try: + # Support the UNIX convention of using "-" for stdin. Note that + # we are not opening the file with universal newline support + # (which codecs doesn't support anyway), so the resulting lines do + # contain trailing '\r' characters if we are reading a file that + # has CRLF endings. + # If after the split a trailing '\r' is present, it is removed + # below. + if filename == '-': + lines = codecs.StreamReaderWriter(sys.stdin, + codecs.getreader('utf8'), + codecs.getwriter('utf8'), + 'replace').read().split('\n') + else: + lines = codecs.open(filename, 'r', 'utf8', 'replace').read().split('\n') + + # Remove trailing '\r'. + # The -1 accounts for the extra trailing blank line we get from split() + for linenum in range(len(lines) - 1): + if lines[linenum].endswith('\r'): + lines[linenum] = lines[linenum].rstrip('\r') + crlf_lines.append(linenum + 1) + else: + lf_lines.append(linenum + 1) + + except IOError: + _cpplint_state.PrintError( + "Skipping input '%s': Can't open for reading\n" % filename) + _RestoreFilters() + return + + # Note, if no dot is found, this will give the entire filename as the ext. + file_extension = filename[filename.rfind('.') + 1:] + + # When reading from stdin, the extension is unknown, so no cpplint tests + # should rely on the extension. + if filename != '-' and file_extension not in GetAllExtensions(): + _cpplint_state.PrintError('Ignoring %s; not a valid file name ' + '(%s)\n' % (filename, ', '.join(GetAllExtensions()))) + else: + ProcessFileData(filename, file_extension, lines, Error, + extra_check_functions) + + # If end-of-line sequences are a mix of LF and CR-LF, issue + # warnings on the lines with CR. + # + # Don't issue any warnings if all lines are uniformly LF or CR-LF, + # since critique can handle these just fine, and the style guide + # doesn't dictate a particular end of line sequence. + # + # We can't depend on os.linesep to determine what the desired + # end-of-line sequence should be, since that will return the + # server-side end-of-line sequence. + if lf_lines and crlf_lines: + # Warn on every line with CR. An alternative approach might be to + # check whether the file is mostly CRLF or just LF, and warn on the + # minority, we bias toward LF here since most tools prefer LF. + for linenum in crlf_lines: + Error(filename, linenum, 'whitespace/newline', 1, + 'Unexpected \\r (^M) found; better to use only \\n') + + _cpplint_state.PrintInfo('Done processing %s\n' % filename) + _RestoreFilters() + + +def PrintUsage(message): + """Prints a brief usage string and exits, optionally with an error message. + + Args: + message: The optional error message. + """ + sys.stderr.write(_USAGE) + + if message: + sys.exit('\nFATAL ERROR: ' + message) + else: + sys.exit(0) + + +def PrintCategories(): + """Prints a list of all the error-categories used by error messages. + + These are the categories used to filter messages via --filter. + """ + sys.stderr.write(''.join(' %s\n' % cat for cat in _ERROR_CATEGORIES)) + sys.exit(0) + + +def ParseArguments(args): + """Parses the command line arguments. + + This may set the output format and verbosity level as side-effects. + + Args: + args: The command line arguments: + + Returns: + The list of filenames to lint. + """ + try: + (opts, filenames) = getopt.getopt(args, '', ['help', 'output=', 'verbose=', + 'counting=', + 'filter=', + 'root=', + 'repository=', + 'linelength=', + 'extensions=', + 'exclude=', + 'headers=', + 'quiet', + 'recursive']) + except getopt.GetoptError: + PrintUsage('Invalid arguments.') + + verbosity = _VerboseLevel() + output_format = _OutputFormat() + filters = '' + counting_style = '' + recursive = False + + for (opt, val) in opts: + if opt == '--help': + PrintUsage(None) + elif opt == '--output': + if val not in ('emacs', 'vs7', 'eclipse', 'junit'): + PrintUsage('The only allowed output formats are emacs, vs7, eclipse ' + 'and junit.') + output_format = val + elif opt == '--verbose': + verbosity = int(val) + elif opt == '--filter': + filters = val + if not filters: + PrintCategories() + elif opt == '--counting': + if val not in ('total', 'toplevel', 'detailed'): + PrintUsage('Valid counting options are total, toplevel, and detailed') + counting_style = val + elif opt == '--root': + global _root + _root = val + elif opt == '--repository': + global _repository + _repository = val + elif opt == '--linelength': + global _line_length + try: + _line_length = int(val) + except ValueError: + PrintUsage('Line length must be digits.') + elif opt == '--exclude': + global _excludes + if not _excludes: + _excludes = set() + _excludes.update(glob.glob(val)) + elif opt == '--extensions': + global _valid_extensions + try: + _valid_extensions = set(val.split(',')) + except ValueError: + PrintUsage('Extensions must be comma separated list.') + elif opt == '--headers': + global _header_extensions + try: + _header_extensions = set(val.split(',')) + except ValueError: + PrintUsage('Extensions must be comma separated list.') + elif opt == '--recursive': + recursive = True + elif opt == '--quiet': + global _quiet + _quiet = True + + if not filenames: + PrintUsage('No files were specified.') + + if recursive: + filenames = _ExpandDirectories(filenames) + + if _excludes: + filenames = _FilterExcludedFiles(filenames) + + _SetOutputFormat(output_format) + _SetVerboseLevel(verbosity) + _SetFilters(filters) + _SetCountingStyle(counting_style) + + return filenames + +def _ExpandDirectories(filenames): + """Searches a list of filenames and replaces directories in the list with + all files descending from those directories. Files with extensions not in + the valid extensions list are excluded. + + Args: + filenames: A list of files or directories + + Returns: + A list of all files that are members of filenames or descended from a + directory in filenames + """ + expanded = set() + for filename in filenames: + if not os.path.isdir(filename): + expanded.add(filename) + continue + + for root, _, files in os.walk(filename): + for loopfile in files: + fullname = os.path.join(root, loopfile) + if fullname.startswith('.' + os.path.sep): + fullname = fullname[len('.' + os.path.sep):] + expanded.add(fullname) + + filtered = [] + for filename in expanded: + if os.path.splitext(filename)[1][1:] in GetAllExtensions(): + filtered.append(filename) + + return filtered + +def _FilterExcludedFiles(filenames): + """Filters out files listed in the --exclude command line switch. File paths + in the switch are evaluated relative to the current working directory + """ + exclude_paths = [os.path.abspath(f) for f in _excludes] + return [f for f in filenames if os.path.abspath(f) not in exclude_paths] + +def main(): + filenames = ParseArguments(sys.argv[1:]) + backup_err = sys.stderr + try: + # Change stderr to write with replacement characters so we don't die + # if we try to print something containing non-ASCII characters. + sys.stderr = codecs.StreamReader(sys.stderr, 'replace') + + _cpplint_state.ResetErrorCounts() + for filename in filenames: + ProcessFile(filename, _cpplint_state.verbose_level) + _cpplint_state.PrintErrorCounts() + + if _cpplint_state.output_format == 'junit': + sys.stderr.write(_cpplint_state.FormatJUnitXML()) + + finally: + sys.stderr = backup_err + + sys.exit(_cpplint_state.error_count > 0) + + +if __name__ == '__main__': + main() + diff --git a/cpp/build-support/run_cpplint.py b/cpp/build-support/run_cpplint.py new file mode 100755 index 0000000000..2a33748279 --- /dev/null +++ b/cpp/build-support/run_cpplint.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import print_function +import lintutils +from subprocess import PIPE, STDOUT +import argparse +import multiprocessing as mp +import sys +import platform +from functools import partial + + +# NOTE(wesm): +# +# * readability/casting is disabled as it aggressively warns about functions +# with names like "int32", so "int32(x)", where int32 is a function name, +# warns with +_filters = ''' +-whitespace/comments +-readability/casting +-readability/todo +-readability/alt_tokens +-build/header_guard +-build/c++11 +-runtime/references +-build/include_order +'''.split() + + +def _get_chunk_key(filenames): + # lists are not hashable so key on the first filename in a chunk + return filenames[0] + + +def _check_some_files(completed_processes, filenames): + # cpplint outputs complaints in '/path:line_number: complaint' format, + # so we can scan its output to get a list of files to fix + result = completed_processes[_get_chunk_key(filenames)] + return lintutils.stdout_pathcolonline(result, filenames) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Runs cpplint on all of the source files.") + parser.add_argument("--cpplint_binary", + required=True, + help="Path to the cpplint binary") + parser.add_argument("--exclude_globs", + help="Filename containing globs for files " + "that should be excluded from the checks") + parser.add_argument("--source_dir", + required=True, + help="Root directory of the source code") + parser.add_argument("--quiet", default=False, + action="store_true", + help="If specified, only print errors") + arguments = parser.parse_args() + + exclude_globs = [] + if arguments.exclude_globs: + for line in open(arguments.exclude_globs): + exclude_globs.append(line.strip()) + + linted_filenames = [] + for path in lintutils.get_sources(arguments.source_dir, exclude_globs): + linted_filenames.append(str(path)) + + cmd = [ + arguments.cpplint_binary, + '--verbose=2', + '--linelength=90', + '--filter=' + ','.join(_filters) + ] + if (arguments.cpplint_binary.endswith('.py') and + platform.system() == 'Windows'): + # Windows doesn't support executable scripts; execute with + # sys.executable + cmd.insert(0, sys.executable) + if arguments.quiet: + cmd.append('--quiet') + else: + print("\n".join(map(lambda x: "Linting {}".format(x), + linted_filenames))) + + # lint files in chunks: each invocation of cpplint will process 16 files + chunks = lintutils.chunk(linted_filenames, 16) + cmds = [cmd + some for some in chunks] + results = lintutils.run_parallel(cmds, stdout=PIPE, stderr=STDOUT) + + error = False + # record completed processes (keyed by the first filename in the input + # chunk) for lookup in _check_some_files + completed_processes = { + _get_chunk_key(filenames): result + for filenames, result in zip(chunks, results) + } + checker = partial(_check_some_files, completed_processes) + pool = mp.Pool() + try: + # scan the outputs of various cpplint invocations in parallel to + # distill a list of problematic files + for problem_files, stdout in pool.imap(checker, chunks): + if problem_files: + if isinstance(stdout, bytes): + stdout = stdout.decode('utf8') + print(stdout, file=sys.stderr) + error = True + except Exception: + error = True + raise + finally: + pool.terminate() + pool.join() + + sys.exit(1 if error else 0) + From c13edb909e42d36941422c1f1f2740924910b419 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 20 Sep 2019 17:35:31 +0800 Subject: [PATCH 069/217] MS-578 makesure milvus5.0 dont crack 0.3.1 data Former-commit-id: e3c81573e4a28ac51b564db548dd29fee92e1915 --- cpp/src/db/Utils.cpp | 36 ++++ cpp/src/db/Utils.h | 11 + cpp/src/db/meta/MetaFactory.cpp | 46 ++--- cpp/src/db/meta/MySQLMetaImpl.cpp | 309 ++++++++++++++++++++--------- cpp/src/db/meta/MySQLMetaImpl.h | 3 + cpp/src/db/meta/SqliteMetaImpl.cpp | 29 ++- cpp/src/db/meta/SqliteMetaImpl.h | 4 +- cpp/src/utils/Error.h | 1 + 8 files changed, 305 insertions(+), 134 deletions(-) diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index 26f93563dc..41ebe61887 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -21,6 +21,7 @@ #include #include +#include #include namespace zilliz { @@ -195,6 +196,41 @@ meta::DateT GetDate() { return GetDate(std::time(nullptr), 0); } +// URI format: dialect://username:password@host:port/database +Status ParseMetaUri(const std::string& uri, MetaUriInfo& info) { + std::string dialect_regex = "(.*)"; + std::string username_tegex = "(.*)"; + std::string password_regex = "(.*)"; + std::string host_regex = "(.*)"; + std::string port_regex = "(.*)"; + std::string db_name_regex = "(.*)"; + std::string uri_regex_str = + dialect_regex + "\\:\\/\\/" + + username_tegex + "\\:" + + password_regex + "\\@" + + host_regex + "\\:" + + port_regex + "\\/" + + db_name_regex; + + std::regex uri_regex(uri_regex_str); + std::smatch pieces_match; + + if (std::regex_match(uri, pieces_match, uri_regex)) { + info.dialect_ = pieces_match[1].str(); + info.username_ = pieces_match[2].str(); + info.password_ = pieces_match[3].str(); + info.host_ = pieces_match[4].str(); + info.port_ = pieces_match[5].str(); + info.db_name_ = pieces_match[6].str(); + + //TODO: verify host, port... + } else { + return Status(DB_INVALID_META_URI, "Invalid meta uri: " + uri); + } + + return Status::OK(); +} + } // namespace utils } // namespace engine } // namespace milvus diff --git a/cpp/src/db/Utils.h b/cpp/src/db/Utils.h index f27273c6fe..39bdfbfbc1 100644 --- a/cpp/src/db/Utils.h +++ b/cpp/src/db/Utils.h @@ -44,6 +44,17 @@ meta::DateT GetDate(const std::time_t &t, int day_delta = 0); meta::DateT GetDate(); meta::DateT GetDateWithDelta(int day_delta); +struct MetaUriInfo { + std::string dialect_; + std::string username_; + std::string password_; + std::string host_; + std::string port_; + std::string db_name_; +}; + +Status ParseMetaUri(const std::string& uri, MetaUriInfo& info); + } // namespace utils } // namespace engine } // namespace milvus diff --git a/cpp/src/db/meta/MetaFactory.cpp b/cpp/src/db/meta/MetaFactory.cpp index 2f498941d4..34b553ea94 100644 --- a/cpp/src/db/meta/MetaFactory.cpp +++ b/cpp/src/db/meta/MetaFactory.cpp @@ -20,13 +20,14 @@ #include "MySQLMetaImpl.h" #include "utils/Log.h" #include "utils/Exception.h" +#include "db/Utils.h" #include #include #include #include #include -#include +#include namespace zilliz { namespace milvus { @@ -49,38 +50,23 @@ namespace engine { meta::MetaPtr MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) { std::string uri = metaOptions.backend_uri; - std::string dialectRegex = "(.*)"; - std::string usernameRegex = "(.*)"; - std::string passwordRegex = "(.*)"; - std::string hostRegex = "(.*)"; - std::string portRegex = "(.*)"; - std::string dbNameRegex = "(.*)"; - std::string uriRegexStr = dialectRegex + "\\:\\/\\/" + - usernameRegex + "\\:" + - passwordRegex + "\\@" + - hostRegex + "\\:" + - portRegex + "\\/" + - dbNameRegex; - std::regex uriRegex(uriRegexStr); - std::smatch pieces_match; - - if (std::regex_match(uri, pieces_match, uriRegex)) { - std::string dialect = pieces_match[1].str(); - std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower); - if (dialect.find("mysql") != std::string::npos) { - ENGINE_LOG_INFO << "Using MySQL"; - return std::make_shared(metaOptions, mode); - } else if (dialect.find("sqlite") != std::string::npos) { - ENGINE_LOG_INFO << "Using SQLite"; - return std::make_shared(metaOptions); - } else { - ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << dialect; - throw InvalidArgumentException("URI dialect is not mysql / sqlite"); - } - } else { + utils::MetaUriInfo uri_info; + auto status = utils::ParseMetaUri(uri, uri_info); + if(!status.ok()) { ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri; throw InvalidArgumentException("Wrong URI format "); } + + if (strcasecmp(uri_info.dialect_.c_str(), "mysql") == 0) { + ENGINE_LOG_INFO << "Using MySQL"; + return std::make_shared(metaOptions, mode); + } else if (strcasecmp(uri_info.dialect_.c_str(), "sqlite") == 0) { + ENGINE_LOG_INFO << "Using SQLite"; + return std::make_shared(metaOptions); + } else { + ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << uri_info.dialect_; + throw InvalidArgumentException("URI dialect is not mysql / sqlite"); + } } } // namespace engine diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 282df32eb1..a070b19fb1 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -19,21 +19,22 @@ #include "db/IDGenerator.h" #include "db/Utils.h" #include "utils/Log.h" +#include "utils/Exception.h" #include "MetaConsts.h" #include "metrics/Metrics.h" #include #include #include -#include #include #include #include #include #include #include - -#include "mysql++/mysql++.h" +#include +#include +#include namespace zilliz { @@ -56,8 +57,112 @@ Status HandleException(const std::string &desc, const char* what = nullptr) { } } +class MetaField { +public: + MetaField(const std::string& name, const std::string& type, const std::string& setting) + : name_(name), + type_(type), + setting_(setting) { + } + + std::string name() const { + return name_; + } + + std::string ToString() const { + return name_ + " " + type_ + " " + setting_; + } + + // mysql field type has additional information. for instance, a filed type is defined as 'BIGINT' + // we get the type from sql is 'bigint(20)', so we need to ignore the '(20)' + bool IsEqual(const MetaField& field) const { + size_t name_len_min = field.name_.length() > name_.length() ? name_.length() : field.name_.length(); + size_t type_len_min = field.type_.length() > type_.length() ? type_.length() : field.type_.length(); + return strncasecmp(field.name_.c_str(), name_.c_str(), name_len_min) == 0 && + strncasecmp(field.type_.c_str(), type_.c_str(), type_len_min) == 0; + } + +private: + std::string name_; + std::string type_; + std::string setting_; +}; + +using MetaFields = std::vector; +class MetaSchema { +public: + MetaSchema(const std::string& name, const MetaFields& fields) + : name_(name), + fields_(fields) { + } + + std::string name() const { + return name_; + } + + std::string ToString() const { + std::string result; + for(auto& field : fields_) { + if(!result.empty()) { + result += ","; + } + result += field.ToString(); + } + return result; + } + + //if the outer fields contains all this MetaSchema fields, return true + //otherwise return false + bool IsEqual(const MetaFields& fields) const { + std::vector found_field; + for(const auto& this_field : fields_) { + for(const auto& outer_field : fields) { + if(this_field.IsEqual(outer_field)) { + found_field.push_back(this_field.name()); + break; + } + } + } + + return found_field.size() == fields_.size(); + } + +private: + std::string name_; + MetaFields fields_; +}; + +//Tables schema +static const MetaSchema TABLES_SCHEMA(META_TABLES, { + MetaField("id", "BIGINT", "PRIMARY KEY AUTO_INCREMENT"), + MetaField("table_id", "VARCHAR(255)", "UNIQUE NOT NULL"), + MetaField("state", "INT", "NOT NULL"), + MetaField("dimension", "SMALLINT", "NOT NULL"), + MetaField("created_on", "BIGINT", "NOT NULL"), + MetaField("flag", "BIGINT", "DEFAULT 0 NOT NULL"), + MetaField("index_file_size", "BIGINT", "DEFAULT 1024 NOT NULL"), + MetaField("engine_type", "INT", "DEFAULT 1 NOT NULL"), + MetaField("nlist", "INT", "DEFAULT 16384 NOT NULL"), + MetaField("metric_type", "INT", "DEFAULT 1 NOT NULL"), +}); + +//TableFiles schema +static const MetaSchema TABLEFILES_SCHEMA(META_TABLEFILES, { + MetaField("id", "BIGINT", "PRIMARY KEY AUTO_INCREMENT"), + MetaField("table_id", "VARCHAR(255)", "NOT NULL"), + MetaField("engine_type", "INT", "DEFAULT 1 NOT NULL"), + MetaField("file_id", "VARCHAR(255)", "NOT NULL"), + MetaField("file_type", "INT", "DEFAULT 0 NOT NULL"), + MetaField("file_size", "BIGINT", "DEFAULT 0 NOT NULL"), + MetaField("row_count", "BIGINT", "DEFAULT 0 NOT NULL"), + MetaField("updated_time", "BIGINT", "NOT NULL"), + MetaField("created_on", "BIGINT", "NOT NULL"), + MetaField("date", "INT", "DEFAULT -1 NOT NULL"), +}); + } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_, const int &mode) : options_(options_), mode_(mode) { @@ -84,7 +189,56 @@ Status MySQLMetaImpl::NextFileId(std::string &file_id) { return Status::OK(); } +void MySQLMetaImpl::ValidateMetaSchema() { + if(nullptr == mysql_connection_pool_) { + return; + } + + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return; + } + + auto validate_func = [&](const MetaSchema& schema) { + Query query_statement = connectionPtr->query(); + query_statement << "DESC " << schema.name() << ";"; + + MetaFields exist_fields; + + try { + StoreQueryResult res = query_statement.store(); + for (size_t i = 0; i < res.num_rows(); i++) { + const Row &row = res[i]; + std::string name, type; + row["Field"].to_string(name); + row["Type"].to_string(type); + + exist_fields.push_back(MetaField(name, type, "")); + } + } catch (std::exception &e) { + ENGINE_LOG_DEBUG << "Meta table '" << schema.name() << "' not exist and will be created"; + } + + if(exist_fields.empty()) { + return true; + } + + return schema.IsEqual(exist_fields); + }; + + //verify Tables + if (!validate_func(TABLES_SCHEMA)) { + throw Exception(DB_INCOMPATIB_META, "Meta Tables schema is created by Milvus old version"); + } + + //verufy TableFiles + if (!validate_func(TABLEFILES_SCHEMA)) { + throw Exception(DB_INCOMPATIB_META, "Meta TableFiles schema is created by Milvus old version"); + } +} + Status MySQLMetaImpl::Initialize() { + //step 1: create db root path if (!boost::filesystem::is_directory(options_.path)) { auto ret = boost::filesystem::create_directory(options_.path); if (!ret) { @@ -96,108 +250,79 @@ Status MySQLMetaImpl::Initialize() { std::string uri = options_.backend_uri; - std::string dialectRegex = "(.*)"; - std::string usernameRegex = "(.*)"; - std::string passwordRegex = "(.*)"; - std::string hostRegex = "(.*)"; - std::string portRegex = "(.*)"; - std::string dbNameRegex = "(.*)"; - std::string uriRegexStr = dialectRegex + "\\:\\/\\/" + - usernameRegex + "\\:" + - passwordRegex + "\\@" + - hostRegex + "\\:" + - portRegex + "\\/" + - dbNameRegex; - std::regex uriRegex(uriRegexStr); - std::smatch pieces_match; + //step 2: parse and check meta uri + utils::MetaUriInfo uri_info; + auto status = utils::ParseMetaUri(uri, uri_info); + if(!status.ok()) { + std::string msg = "Wrong URI format: " + uri; + ENGINE_LOG_ERROR << msg; + throw Exception(DB_INVALID_META_URI, msg); + } - if (std::regex_match(uri, pieces_match, uriRegex)) { - std::string dialect = pieces_match[1].str(); - std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower); - if (dialect.find("mysql") == std::string::npos) { - return Status(DB_ERROR, "URI's dialect is not MySQL"); + if (strcasecmp(uri_info.dialect_.c_str(), "mysql") != 0) { + std::string msg = "URI's dialect is not MySQL"; + ENGINE_LOG_ERROR << msg; + throw Exception(DB_INVALID_META_URI, msg); + } + + //step 3: connect mysql + int thread_hint = std::thread::hardware_concurrency(); + int max_pool_size = (thread_hint == 0) ? 8 : thread_hint; + unsigned int port = 0; + if (!uri_info.port_.empty()) { + port = std::stoi(uri_info.port_); + } + + mysql_connection_pool_ = + std::make_shared(uri_info.db_name_, uri_info.username_, + uri_info.password_, uri_info.host_, port, max_pool_size); + ENGINE_LOG_DEBUG << "MySQL connection pool: maximum pool size = " << std::to_string(max_pool_size); + + //step 4: validate to avoid open old version schema + ValidateMetaSchema(); + + //step 5: create meta tables + try { + + if (mode_ != DBOptions::MODE::READ_ONLY) { + CleanUp(); } - std::string username = pieces_match[2].str(); - std::string password = pieces_match[3].str(); - std::string serverAddress = pieces_match[4].str(); - unsigned int port = 0; - if (!pieces_match[5].str().empty()) { - port = std::stoi(pieces_match[5].str()); - } - std::string dbName = pieces_match[6].str(); + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - int threadHint = std::thread::hardware_concurrency(); - int maxPoolSize = threadHint == 0 ? 8 : threadHint; - mysql_connection_pool_ = - std::make_shared(dbName, username, password, serverAddress, port, maxPoolSize); - - ENGINE_LOG_DEBUG << "MySQL connection pool: maximum pool size = " << std::to_string(maxPoolSize); - try { - - if (mode_ != DBOptions::MODE::READ_ONLY) { - CleanUp(); + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to database server"); } - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } + if (!connectionPtr->thread_aware()) { + ENGINE_LOG_ERROR << "MySQL++ wasn't built with thread awareness! Can't run without it."; + return Status(DB_ERROR, "MySQL++ wasn't built with thread awareness! Can't run without it."); + } + Query InitializeQuery = connectionPtr->query(); + InitializeQuery << "CREATE TABLE IF NOT EXISTS " << + TABLES_SCHEMA.name() << " (" << TABLES_SCHEMA.ToString() + ");"; - if (!connectionPtr->thread_aware()) { - ENGINE_LOG_ERROR << "MySQL++ wasn't built with thread awareness! Can't run without it."; - return Status(DB_ERROR, "MySQL++ wasn't built with thread awareness! Can't run without it."); - } - Query InitializeQuery = connectionPtr->query(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); - InitializeQuery << "CREATE TABLE IF NOT EXISTS " << - META_TABLES << " " << - "(id BIGINT PRIMARY KEY AUTO_INCREMENT, " << - "table_id VARCHAR(255) UNIQUE NOT NULL, " << - "state INT NOT NULL, " << - "dimension SMALLINT NOT NULL, " << - "created_on BIGINT NOT NULL, " << - "flag BIGINT DEFAULT 0 NOT NULL, " << - "index_file_size BIGINT DEFAULT 1024 NOT NULL, " << - "engine_type INT DEFAULT 1 NOT NULL, " << - "nlist INT DEFAULT 16384 NOT NULL, " << - "metric_type INT DEFAULT 1 NOT NULL);"; + if (!InitializeQuery.exec()) { + return HandleException("Initialization Error", InitializeQuery.error()); + } - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); + InitializeQuery << "CREATE TABLE IF NOT EXISTS " << + TABLEFILES_SCHEMA.name() << " (" << TABLEFILES_SCHEMA.ToString() + ");"; - if (!InitializeQuery.exec()) { - return HandleException("Initialization Error", InitializeQuery.error()); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); - InitializeQuery << "CREATE TABLE IF NOT EXISTS " << - META_TABLEFILES << " " << - "(id BIGINT PRIMARY KEY AUTO_INCREMENT, " << - "table_id VARCHAR(255) NOT NULL, " << - "engine_type INT DEFAULT 1 NOT NULL, " << - "file_id VARCHAR(255) NOT NULL, " << - "file_type INT DEFAULT 0 NOT NULL, " << - "file_size BIGINT DEFAULT 0 NOT NULL, " << - "row_count BIGINT DEFAULT 0 NOT NULL, " << - "updated_time BIGINT NOT NULL, " << - "created_on BIGINT NOT NULL, " << - "date INT DEFAULT -1 NOT NULL);"; + if (!InitializeQuery.exec()) { + return HandleException("Initialization Error", InitializeQuery.error()); + } + } //Scoped Connection - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); - - if (!InitializeQuery.exec()) { - return HandleException("Initialization Error", InitializeQuery.error()); - } - } //Scoped Connection - - } catch (std::exception &e) { - return HandleException("GENERAL ERROR DURING INITIALIZATION", e.what()); - } - } else { - ENGINE_LOG_ERROR << "Wrong URI format. URI = " << uri; - return Status(DB_ERROR, "Wrong URI format"); + } catch (std::exception &e) { + return HandleException("GENERAL ERROR DURING INITIALIZATION", e.what()); } return Status::OK(); @@ -1843,7 +1968,7 @@ Status MySQLMetaImpl::DropAll() { } Query dropTableQuery = connectionPtr->query(); - dropTableQuery << "DROP TABLE IF EXISTS " << META_TABLES << ", " << META_TABLEFILES << ";"; + dropTableQuery << "DROP TABLE IF EXISTS " << TABLES_SCHEMA.name() << ", " << TABLEFILES_SCHEMA.name() << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str(); diff --git a/cpp/src/db/meta/MySQLMetaImpl.h b/cpp/src/db/meta/MySQLMetaImpl.h index dca109b231..4950b604a3 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.h +++ b/cpp/src/db/meta/MySQLMetaImpl.h @@ -103,8 +103,11 @@ class MySQLMetaImpl : public Meta { Status NextFileId(std::string &file_id); Status NextTableId(std::string &table_id); Status DiscardFiles(long long to_discard_size); + + void ValidateMetaSchema(); Status Initialize(); +private: const DBMetaOptions options_; const int mode_; diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index a28038d019..7131a47800 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -84,7 +84,6 @@ inline auto StoragePrototype(const std::string &path) { using ConnectorT = decltype(StoragePrototype("")); static std::unique_ptr ConnectorPtr; -using ConditionT = decltype(c(&TableFileSchema::id_) == 1UL); SqliteMetaImpl::SqliteMetaImpl(const DBMetaOptions &options_) : options_(options_) { @@ -111,6 +110,23 @@ Status SqliteMetaImpl::NextFileId(std::string &file_id) { return Status::OK(); } +void SqliteMetaImpl::ValidateMetaSchema() { + if(ConnectorPtr == nullptr) { + return; + } + + //old meta could be recreated since schema changed, throw exception if meta schema is not compatible + auto ret = ConnectorPtr->sync_schema_simulate(); + if(ret.find(META_TABLES) != ret.end() + && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLES]) { + throw Exception(DB_INCOMPATIB_META, "Meta Tables schema is created by Milvus old version"); + } + if(ret.find(META_TABLEFILES) != ret.end() + && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLEFILES]) { + throw Exception(DB_INCOMPATIB_META, "Meta TableFiles schema is created by Milvus old version"); + } +} + Status SqliteMetaImpl::Initialize() { if (!boost::filesystem::is_directory(options_.path)) { auto ret = boost::filesystem::create_directory(options_.path); @@ -123,16 +139,7 @@ Status SqliteMetaImpl::Initialize() { ConnectorPtr = std::make_unique(StoragePrototype(options_.path + "/meta.sqlite")); - //old meta could be recreated since schema changed, throw exception if meta schema is not compatible - auto ret = ConnectorPtr->sync_schema_simulate(); - if(ret.find(META_TABLES) != ret.end() - && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLES]) { - throw Exception(DB_INCOMPATIB_META, "Meta schema is created by Milvus old version"); - } - if(ret.find(META_TABLEFILES) != ret.end() - && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLEFILES]) { - throw Exception(DB_INCOMPATIB_META, "Meta schema is created by Milvus old version"); - } + ValidateMetaSchema(); ConnectorPtr->sync_schema(); ConnectorPtr->open_forever(); // thread safe option diff --git a/cpp/src/db/meta/SqliteMetaImpl.h b/cpp/src/db/meta/SqliteMetaImpl.h index b1465dd95a..db7a70fe53 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.h +++ b/cpp/src/db/meta/SqliteMetaImpl.h @@ -97,10 +97,12 @@ class SqliteMetaImpl : public Meta { Status NextFileId(std::string &file_id); Status NextTableId(std::string &table_id); Status DiscardFiles(long to_discard_size); + + void ValidateMetaSchema(); Status Initialize(); +private: const DBMetaOptions options_; - std::mutex meta_mutex_; }; // DBMetaImpl diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index 672aa92848..e903dcb2fb 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -87,6 +87,7 @@ constexpr ErrorCode DB_NOT_FOUND = ToDbErrorCode(3); constexpr ErrorCode DB_ALREADY_EXIST = ToDbErrorCode(4); constexpr ErrorCode DB_INVALID_PATH = ToDbErrorCode(5); constexpr ErrorCode DB_INCOMPATIB_META = ToDbErrorCode(6); +constexpr ErrorCode DB_INVALID_META_URI = ToDbErrorCode(7); //knowhere error code constexpr ErrorCode KNOWHERE_ERROR = ToKnowhereErrorCode(1); From cfc42c5471584f9bd73fbed0f4c8fdd48e6afdf5 Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Fri, 20 Sep 2019 18:02:08 +0800 Subject: [PATCH 070/217] MS-576 Scheduler refactor Former-commit-id: dc9317412c5dc72191381d5a634e7e9ff14f98b2 --- cpp/CHANGELOG.md | 1 + cpp/src/scheduler/Scheduler.cpp | 129 ++++-------------- cpp/src/scheduler/Scheduler.h | 2 + cpp/src/scheduler/action/Action.h | 10 ++ .../scheduler/action/PushTaskToNeighbour.cpp | 80 +++++++++++ .../examples/grpcsimple/src/ClientTest.cpp | 2 +- 6 files changed, 118 insertions(+), 106 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 667399cc5d..09929ed8d0 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -147,6 +147,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-539 - Remove old task code - MS-546 - Add simple mode resource_config - MS-570 - Add prometheus docker-compose file +- MS-576 - Scheduler refactor ## New Feature - MS-343 - Implement ResourceMgr diff --git a/cpp/src/scheduler/Scheduler.cpp b/cpp/src/scheduler/Scheduler.cpp index 6aa4501e29..ff66e6ac21 100644 --- a/cpp/src/scheduler/Scheduler.cpp +++ b/cpp/src/scheduler/Scheduler.cpp @@ -16,7 +16,7 @@ // under the License. -#include +#include "src/cache/GpuCacheMgr.h" #include "event/LoadCompletedEvent.h" #include "Scheduler.h" #include "action/Action.h" @@ -33,6 +33,14 @@ Scheduler::Scheduler(ResourceMgrWPtr res_mgr) if (auto mgr = res_mgr_.lock()) { mgr->RegisterSubscriber(std::bind(&Scheduler::PostEvent, this, std::placeholders::_1)); } + event_register_.insert(std::make_pair(static_cast(EventType::START_UP), + std::bind(&Scheduler::OnStartUp, this, std::placeholders::_1))); + event_register_.insert(std::make_pair(static_cast(EventType::LOAD_COMPLETED), + std::bind(&Scheduler::OnLoadCompleted, this, std::placeholders::_1))); + event_register_.insert(std::make_pair(static_cast(EventType::TASK_TABLE_UPDATED), + std::bind(&Scheduler::OnTaskTableUpdated, this, std::placeholders::_1))); + event_register_.insert(std::make_pair(static_cast(EventType::FINISH_TASK), + std::bind(&Scheduler::OnFinishTask, this, std::placeholders::_1))); } @@ -84,40 +92,8 @@ Scheduler::worker_function() { void Scheduler::Process(const EventPtr &event) { - switch (event->Type()) { - case EventType::START_UP: { - OnStartUp(event); - break; - } - case EventType::LOAD_COMPLETED: { - OnLoadCompleted(event); - break; - } - case EventType::FINISH_TASK: { - OnFinishTask(event); - break; - } - case EventType::TASK_TABLE_UPDATED: { - OnTaskTableUpdated(event); - break; - } - default: { - // TODO: logging - break; - } - } -} - - -void -Scheduler::OnStartUp(const EventPtr &event) { - if (auto resource = event->resource_.lock()) { - resource->WakeupLoader(); - } -} - -void -Scheduler::OnFinishTask(const EventPtr &event) { + auto process_event = event_register_.at(static_cast(event->Type())); + process_event(event); } // TODO: refactor the function @@ -130,79 +106,11 @@ Scheduler::OnLoadCompleted(const EventPtr &event) { auto task_table_type = load_completed_event->task_table_item_->task->label()->Type(); switch (task_table_type) { case TaskLabelType::DEFAULT: { - if (not resource->HasExecutor() && load_completed_event->task_table_item_->Move()) { - auto task = load_completed_event->task_table_item_->task; - auto search_task = std::static_pointer_cast(task); - bool moved = false; - - // to support test task, REFACTOR - if (auto index_engine = search_task->index_engine_) { - auto location = index_engine->GetLocation(); - - for (auto i = 0; i < res_mgr_.lock()->GetNumGpuResource(); ++i) { - auto index = zilliz::milvus::cache::GpuCacheMgr::GetInstance(i)->GetIndex(location); - if (index != nullptr) { - moved = true; - auto dest_resource = res_mgr_.lock()->GetResource(ResourceType::GPU, i); - Action::PushTaskToResource(load_completed_event->task_table_item_->task, dest_resource); - break; - } - } - } - - if (not moved) { - Action::PushTaskToNeighbourRandomly(task, resource); - } - } + Action::DefaultLabelTaskScheduler(res_mgr_, resource, load_completed_event); break; } case TaskLabelType::SPECIFIED_RESOURCE: { - // support next version -// auto self = event->resource_.lock(); -// auto task = load_completed_event->task_table_item_->task; -// -// // if this resource is disk, assign it to smallest cost resource -// if (self->type() == ResourceType::DISK) { -// // step 1: calculate shortest path per resource, from disk to compute resource -// auto compute_resources = res_mgr_.lock()->GetComputeResources(); -// std::vector> paths; -// std::vector transport_costs; -// for (auto &res : compute_resources) { -// std::vector path; -// uint64_t transport_cost = ShortestPath(self, res, res_mgr_.lock(), path); -// transport_costs.push_back(transport_cost); -// paths.emplace_back(path); -// } -// -// // step 2: select min cost, cost(resource) = avg_cost * task_to_do + transport_cost -// uint64_t min_cost = std::numeric_limits::max(); -// uint64_t min_cost_idx = 0; -// for (uint64_t i = 0; i < compute_resources.size(); ++i) { -// if (compute_resources[i]->TotalTasks() == 0) { -// min_cost_idx = i; -// break; -// } -// uint64_t cost = compute_resources[i]->TaskAvgCost() * compute_resources[i]->NumOfTaskToExec() -// + transport_costs[i]; -// if (min_cost > cost) { -// min_cost = cost; -// min_cost_idx = i; -// } -// } -// -// // step 3: set path in task -// Path task_path(paths[min_cost_idx], paths[min_cost_idx].size() - 1); -// task->path() = task_path; -// } -// -// if (self->name() == task->path().Last()) { -// self->WakeupLoader(); -// } else { -// auto next_res_name = task->path().Next(); -// auto next_res = res_mgr_.lock()->GetResource(next_res_name); -// load_completed_event->task_table_item_->Move(); -// next_res->task_table().Put(task); -// } + Action::SpecifiedResourceLabelTaskScheduler(res_mgr_, resource, load_completed_event); break; } case TaskLabelType::BROADCAST: { @@ -216,6 +124,17 @@ Scheduler::OnLoadCompleted(const EventPtr &event) { } } +void +Scheduler::OnStartUp(const EventPtr &event) { + if (auto resource = event->resource_.lock()) { + resource->WakeupLoader(); + } +} + +void +Scheduler::OnFinishTask(const EventPtr &event) { +} + void Scheduler::OnTaskTableUpdated(const EventPtr &event) { if (auto resource = event->resource_.lock()) { diff --git a/cpp/src/scheduler/Scheduler.h b/cpp/src/scheduler/Scheduler.h index 0de80d5cc5..feb1374a8b 100644 --- a/cpp/src/scheduler/Scheduler.h +++ b/cpp/src/scheduler/Scheduler.h @@ -122,6 +122,8 @@ private: private: bool running_; + std::unordered_map> event_register_; + ResourceMgrWPtr res_mgr_; std::queue event_queue_; std::thread worker_thread_; diff --git a/cpp/src/scheduler/action/Action.h b/cpp/src/scheduler/action/Action.h index 4af8e1cfee..57fc8fe686 100644 --- a/cpp/src/scheduler/action/Action.h +++ b/cpp/src/scheduler/action/Action.h @@ -18,6 +18,7 @@ #pragma once #include "../resource/Resource.h" +#include "../ResourceMgr.h" namespace zilliz { @@ -34,6 +35,15 @@ public: static void PushTaskToResource(const TaskPtr &task, const ResourcePtr &dest); + + static void + DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr resource, std::shared_ptr event); + + static void + SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, + ResourcePtr resource, + std::shared_ptr event); + }; diff --git a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp index 05f997e96d..76beaa3e07 100644 --- a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp @@ -18,6 +18,8 @@ #include #include +#include "../Algorithm.h" +#include "src/cache/GpuCacheMgr.h" #include "Action.h" @@ -101,6 +103,84 @@ Action::PushTaskToResource(const TaskPtr& task, const ResourcePtr& dest) { dest->task_table().Put(task); } +void +Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr, + ResourcePtr resource, + std::shared_ptr event) { + if (not resource->HasExecutor() && event->task_table_item_->Move()) { + auto task = event->task_table_item_->task; + auto search_task = std::static_pointer_cast(task); + bool moved = false; + + //to support test task, REFACTOR + if (auto index_engine = search_task->index_engine_) { + auto location = index_engine->GetLocation(); + + for (auto i = 0; i < res_mgr.lock()->GetNumGpuResource(); ++i) { + auto index = zilliz::milvus::cache::GpuCacheMgr::GetInstance(i)->GetIndex(location); + if (index != nullptr) { + moved = true; + auto dest_resource = res_mgr.lock()->GetResource(ResourceType::GPU, i); + PushTaskToResource(event->task_table_item_->task, dest_resource); + break; + } + } + } + + if (not moved) { + PushTaskToNeighbourRandomly(task, resource); + } + } +} + +void +Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, + ResourcePtr resource, + std::shared_ptr event) { + auto task = event->task_table_item_->task; + if (resource->type() == ResourceType::DISK) { + // step 1: calculate shortest path per resource, from disk to compute resource + auto compute_resources = res_mgr.lock()->GetComputeResources(); + std::vector> paths; + std::vector transport_costs; + for (auto &res : compute_resources) { + std::vector path; + uint64_t transport_cost = ShortestPath(resource, res, res_mgr.lock(), path); + transport_costs.push_back(transport_cost); + paths.emplace_back(path); + } + + // step 2: select min cost, cost(resource) = avg_cost * task_to_do + transport_cost + uint64_t min_cost = std::numeric_limits::max(); + uint64_t min_cost_idx = 0; + for (uint64_t i = 0; i < compute_resources.size(); ++i) { + if (compute_resources[i]->TotalTasks() == 0) { + min_cost_idx = i; + break; + } + uint64_t cost = compute_resources[i]->TaskAvgCost() * compute_resources[i]->NumOfTaskToExec() + + transport_costs[i]; + if (min_cost > cost) { + min_cost = cost; + min_cost_idx = i; + } + } + + // step 3: set path in task + Path task_path(paths[min_cost_idx], paths[min_cost_idx].size() - 1); + task->path() = task_path; + } + + if (resource->name() == task->path().Last()) { + resource->WakeupLoader(); + } else { + auto next_res_name = task->path().Next(); + auto next_res = res_mgr.lock()->GetResource(next_res_name); + event->task_table_item_->Move(); + next_res->task_table().Put(task); + } +} + } } } diff --git a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp index ff3c6edaf5..6b11946150 100644 --- a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp @@ -39,7 +39,7 @@ constexpr int64_t BATCH_ROW_COUNT = 100000; constexpr int64_t NQ = 5; constexpr int64_t TOP_K = 10; constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different -constexpr int64_t ADD_VECTOR_LOOP = 10; +constexpr int64_t ADD_VECTOR_LOOP = 1; constexpr int64_t SECONDS_EACH_HOUR = 3600; #define BLOCK_SPLITER std::cout << "===========================================" << std::endl; From 1b4ce04c08e84bff8bfcd5f8bb1fc6d6285afc92 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 20 Sep 2019 19:29:06 +0800 Subject: [PATCH 071/217] refine code Former-commit-id: 8e2af64ed8fe7517069944eed3545d13398a3ed7 --- cpp/src/cache/LRU.h | 62 ++++++++++++------------ cpp/src/db/DBFactory.cpp | 2 +- cpp/src/db/DBImpl.cpp | 10 ++-- cpp/src/db/Options.h | 16 +++---- cpp/src/db/Utils.cpp | 20 ++++---- cpp/src/db/insert/MemManagerImpl.cpp | 2 +- cpp/src/db/meta/MetaFactory.cpp | 4 +- cpp/src/db/meta/MySQLMetaImpl.cpp | 72 ++++++++++++++-------------- cpp/src/db/meta/MySQLMetaImpl.h | 2 +- cpp/src/db/meta/SqliteMetaImpl.cpp | 10 ++-- cpp/src/server/DBWrapper.cpp | 20 ++++---- cpp/src/server/Server.cpp | 14 +++--- cpp/src/server/Server.h | 2 +- cpp/src/utils/ThreadPool.h | 44 ++++++++--------- cpp/unittest/db/meta_tests.cpp | 8 ++-- cpp/unittest/db/misc_test.cpp | 12 ++--- cpp/unittest/db/mysql_meta_test.cpp | 8 ++-- cpp/unittest/db/utils.cpp | 36 +++++++------- cpp/unittest/metrics/utils.cpp | 4 +- 19 files changed, 174 insertions(+), 174 deletions(-) diff --git a/cpp/src/cache/LRU.h b/cpp/src/cache/LRU.h index f117425d2b..a4f4dcf5f2 100644 --- a/cpp/src/cache/LRU.h +++ b/cpp/src/cache/LRU.h @@ -34,78 +34,78 @@ public: typedef typename std::list::iterator list_iterator_t; typedef typename std::list::reverse_iterator reverse_list_iterator_t; - LRU(size_t max_size) : _max_size(max_size) {} + LRU(size_t max_size) : max_size_(max_size) {} void put(const key_t& key, const value_t& value) { - auto it = _cache_items_map.find(key); - _cache_items_list.push_front(key_value_pair_t(key, value)); - if (it != _cache_items_map.end()) { - _cache_items_list.erase(it->second); - _cache_items_map.erase(it); + auto it = cache_items_map_.find(key); + cache_items_list_.push_front(key_value_pair_t(key, value)); + if (it != cache_items_map_.end()) { + cache_items_list_.erase(it->second); + cache_items_map_.erase(it); } - _cache_items_map[key] = _cache_items_list.begin(); + cache_items_map_[key] = cache_items_list_.begin(); - if (_cache_items_map.size() > _max_size) { - auto last = _cache_items_list.end(); + if (cache_items_map_.size() > max_size_) { + auto last = cache_items_list_.end(); last--; - _cache_items_map.erase(last->first); - _cache_items_list.pop_back(); + cache_items_map_.erase(last->first); + cache_items_list_.pop_back(); } } const value_t& get(const key_t& key) { - auto it = _cache_items_map.find(key); - if (it == _cache_items_map.end()) { + auto it = cache_items_map_.find(key); + if (it == cache_items_map_.end()) { throw std::range_error("There is no such key in cache"); } else { - _cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second); + cache_items_list_.splice(cache_items_list_.begin(), cache_items_list_, it->second); return it->second->second; } } void erase(const key_t& key) { - auto it = _cache_items_map.find(key); - if (it != _cache_items_map.end()) { - _cache_items_list.erase(it->second); - _cache_items_map.erase(it); + auto it = cache_items_map_.find(key); + if (it != cache_items_map_.end()) { + cache_items_list_.erase(it->second); + cache_items_map_.erase(it); } } bool exists(const key_t& key) const { - return _cache_items_map.find(key) != _cache_items_map.end(); + return cache_items_map_.find(key) != cache_items_map_.end(); } size_t size() const { - return _cache_items_map.size(); + return cache_items_map_.size(); } list_iterator_t begin() { - _iter = _cache_items_list.begin(); - return _iter; + iter_ = cache_items_list_.begin(); + return iter_; } list_iterator_t end() { - return _cache_items_list.end(); + return cache_items_list_.end(); } reverse_list_iterator_t rbegin() { - return _cache_items_list.rbegin(); + return cache_items_list_.rbegin(); } reverse_list_iterator_t rend() { - return _cache_items_list.rend(); + return cache_items_list_.rend(); } void clear() { - _cache_items_list.clear(); - _cache_items_map.clear(); + cache_items_list_.clear(); + cache_items_map_.clear(); } private: - std::list _cache_items_list; - std::unordered_map _cache_items_map; - size_t _max_size; - list_iterator_t _iter; + std::list cache_items_list_; + std::unordered_map cache_items_map_; + size_t max_size_; + list_iterator_t iter_; }; } // cache diff --git a/cpp/src/db/DBFactory.cpp b/cpp/src/db/DBFactory.cpp index 91e03278d6..2eec49fc7a 100644 --- a/cpp/src/db/DBFactory.cpp +++ b/cpp/src/db/DBFactory.cpp @@ -36,7 +36,7 @@ namespace engine { DBOptions DBFactory::BuildOption() { auto meta = MetaFactory::BuildOption(); DBOptions options; - options.meta = meta; + options.meta_ = meta; return options; } diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index d253ddb146..8323564024 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -56,7 +56,7 @@ DBImpl::DBImpl(const DBOptions& options) shutting_down_(true), compact_thread_pool_(1, 1), index_thread_pool_(1, 1) { - meta_ptr_ = MetaFactory::Build(options.meta, options.mode); + meta_ptr_ = MetaFactory::Build(options.meta_, options.mode_); mem_mgr_ = MemManagerFactory::Build(meta_ptr_, options_); Start(); } @@ -77,7 +77,7 @@ Status DBImpl::Start() { shutting_down_.store(false, std::memory_order_release); //for distribute version, some nodes are read only - if (options_.mode != DBOptions::MODE::READ_ONLY) { + if (options_.mode_ != DBOptions::MODE::READ_ONLY) { ENGINE_LOG_TRACE << "StartTimerTasks"; bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this); } @@ -98,7 +98,7 @@ Status DBImpl::Stop() { //wait compaction/buildindex finish bg_timer_thread_.join(); - if (options_.mode != DBOptions::MODE::READ_ONLY) { + if (options_.mode_ != DBOptions::MODE::READ_ONLY) { meta_ptr_->CleanUp(); } @@ -649,7 +649,7 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { bool has_merge = false; for (auto& kv : raw_files) { auto files = kv.second; - if (files.size() < options_.merge_trigger_number) { + if (files.size() < options_.merge_trigger_number_) { ENGINE_LOG_DEBUG << "Files number not greater equal than merge trigger number, skip merge action"; continue; } @@ -684,7 +684,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes - if (options_.mode == DBOptions::MODE::CLUSTER) { + if (options_.mode_ == DBOptions::MODE::CLUSTER) { ttl = meta::D_SEC; } meta_ptr_->CleanUpFilesWithTTL(ttl); diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 663ac3fd09..2ba8c36a32 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -52,10 +52,10 @@ private: }; struct DBMetaOptions { - std::string path; - std::vector slave_paths; - std::string backend_uri; - ArchiveConf archive_conf = ArchiveConf("delete"); + std::string path_; + std::vector slave_paths_; + std::string backend_uri_; + ArchiveConf archive_conf_ = ArchiveConf("delete"); }; // DBMetaOptions struct DBOptions { @@ -65,11 +65,11 @@ struct DBOptions { READ_ONLY } MODE; - uint16_t merge_trigger_number = 2; - DBMetaOptions meta; - int mode = MODE::SINGLE; + uint16_t merge_trigger_number_ = 2; + DBMetaOptions meta_; + int mode_ = MODE::SINGLE; - size_t insert_buffer_size = 4 * ONE_GB; + size_t insert_buffer_size_ = 4 * ONE_GB; bool insert_cache_immediately_ = false; }; // Options diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index 41ebe61887..cf61bf6b3d 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -43,8 +43,8 @@ std::string ConstructParentFolder(const std::string& db_path, const meta::TableF } std::string GetTableFileParentFolder(const DBMetaOptions& options, const meta::TableFileSchema& table_file) { - uint64_t path_count = options.slave_paths.size() + 1; - std::string target_path = options.path; + uint64_t path_count = options.slave_paths_.size() + 1; + std::string target_path = options.path_; uint64_t index = 0; if(meta::TableFileSchema::NEW_INDEX == table_file.file_type_) { @@ -61,7 +61,7 @@ std::string GetTableFileParentFolder(const DBMetaOptions& options, const meta::T } if (index > 0) { - target_path = options.slave_paths[index - 1]; + target_path = options.slave_paths_[index - 1]; } return ConstructParentFolder(target_path, table_file); @@ -78,7 +78,7 @@ long GetMicroSecTimeStamp() { } Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id) { - std::string db_path = options.path; + std::string db_path = options.path_; std::string table_path = db_path + TABLES_FOLDER + table_id; auto status = server::CommonUtil::CreateDirectory(table_path); if (!status.ok()) { @@ -86,7 +86,7 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id return status; } - for(auto& path : options.slave_paths) { + for(auto& path : options.slave_paths_) { table_path = path + TABLES_FOLDER + table_id; status = server::CommonUtil::CreateDirectory(table_path); if (!status.ok()) { @@ -99,8 +99,8 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id } Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force) { - std::vector paths = options.slave_paths; - paths.push_back(options.path); + std::vector paths = options.slave_paths_; + paths.push_back(options.path_); for(auto& path : paths) { std::string table_path = path + TABLES_FOLDER + table_id; @@ -132,13 +132,13 @@ Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& } Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { - std::string parent_path = ConstructParentFolder(options.path, table_file); + std::string parent_path = ConstructParentFolder(options.path_, table_file); std::string file_path = parent_path + "/" + table_file.file_id_; if(boost::filesystem::exists(file_path)) { table_file.location_ = file_path; return Status::OK(); } else { - for(auto& path : options.slave_paths) { + for(auto& path : options.slave_paths_) { parent_path = ConstructParentFolder(path, table_file); file_path = parent_path + "/" + table_file.file_id_; if(boost::filesystem::exists(file_path)) { @@ -149,7 +149,7 @@ Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& tab } std::string msg = "Table file doesn't exist: " + file_path; - ENGINE_LOG_ERROR << msg << " in path: " << options.path + ENGINE_LOG_ERROR << msg << " in path: " << options.path_ << " for table: " << table_file.table_id_; return Status(DB_ERROR, msg); diff --git a/cpp/src/db/insert/MemManagerImpl.cpp b/cpp/src/db/insert/MemManagerImpl.cpp index 35dd227b6c..ecc14583f9 100644 --- a/cpp/src/db/insert/MemManagerImpl.cpp +++ b/cpp/src/db/insert/MemManagerImpl.cpp @@ -43,7 +43,7 @@ Status MemManagerImpl::InsertVectors(const std::string &table_id_, const float *vectors_, IDNumbers &vector_ids_) { - while (GetCurrentMem() > options_.insert_buffer_size) { + while (GetCurrentMem() > options_.insert_buffer_size_) { std::this_thread::sleep_for(std::chrono::milliseconds(1)); } diff --git a/cpp/src/db/meta/MetaFactory.cpp b/cpp/src/db/meta/MetaFactory.cpp index 34b553ea94..8837d54722 100644 --- a/cpp/src/db/meta/MetaFactory.cpp +++ b/cpp/src/db/meta/MetaFactory.cpp @@ -43,12 +43,12 @@ namespace engine { } DBMetaOptions meta; - meta.path = p; + meta.path_ = p; return meta; } meta::MetaPtr MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) { - std::string uri = metaOptions.backend_uri; + std::string uri = metaOptions.backend_uri_; utils::MetaUriInfo uri_info; auto status = utils::ParseMetaUri(uri, uri_info); diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index a070b19fb1..3e54686ed0 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -194,7 +194,7 @@ void MySQLMetaImpl::ValidateMetaSchema() { return; } - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return; } @@ -239,16 +239,16 @@ void MySQLMetaImpl::ValidateMetaSchema() { Status MySQLMetaImpl::Initialize() { //step 1: create db root path - if (!boost::filesystem::is_directory(options_.path)) { - auto ret = boost::filesystem::create_directory(options_.path); + if (!boost::filesystem::is_directory(options_.path_)) { + auto ret = boost::filesystem::create_directory(options_.path_); if (!ret) { - std::string msg = "Failed to create db directory " + options_.path; + std::string msg = "Failed to create db directory " + options_.path_; ENGINE_LOG_ERROR << msg; return Status(DB_META_TRANSACTION_FAILED, msg); } } - std::string uri = options_.backend_uri; + std::string uri = options_.backend_uri_; //step 2: parse and check meta uri utils::MetaUriInfo uri_info; @@ -289,7 +289,7 @@ Status MySQLMetaImpl::Initialize() { } { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -351,7 +351,7 @@ Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, dateListStr = dateListStr.substr(0, dateListStr.size() - 2); //remove the last ", " { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -383,7 +383,7 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -462,7 +462,7 @@ Status MySQLMetaImpl::FilesByType(const std::string &table_id, StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -542,7 +542,7 @@ Status MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableI server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -601,7 +601,7 @@ Status MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -633,7 +633,7 @@ Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -673,7 +673,7 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -736,7 +736,7 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -773,7 +773,7 @@ Status MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -807,7 +807,7 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { server::MetricCollector metric; StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -860,7 +860,7 @@ Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { server::MetricCollector metric; StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -895,7 +895,7 @@ Status MySQLMetaImpl::AllTables(std::vector &table_schema_array) { server::MetricCollector metric; StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -976,7 +976,7 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { std::string date = std::to_string(file_schema.date_); { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -1015,7 +1015,7 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { server::MetricCollector metric; StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -1099,7 +1099,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, server::MetricCollector metric; StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -1218,7 +1218,7 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -1309,7 +1309,7 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, try { StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -1378,7 +1378,7 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, // PXU TODO: Support Swap Status MySQLMetaImpl::Archive() { - auto &criterias = options_.archive_conf.GetCriterias(); + auto &criterias = options_.archive_conf_.GetCriterias(); if (criterias.empty()) { return Status::OK(); } @@ -1391,7 +1391,7 @@ Status MySQLMetaImpl::Archive() { long now = utils::GetMicroSecTimeStamp(); try { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -1432,7 +1432,7 @@ Status MySQLMetaImpl::Size(uint64_t &result) { try { StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -1472,7 +1472,7 @@ Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { server::MetricCollector metric; bool status; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -1537,7 +1537,7 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -1605,7 +1605,7 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { try { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -1636,7 +1636,7 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -1721,7 +1721,7 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -1793,7 +1793,7 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -1844,7 +1844,7 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -1874,7 +1874,7 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { Status MySQLMetaImpl::CleanUp() { try { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -1925,7 +1925,7 @@ Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); @@ -1961,7 +1961,7 @@ Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { Status MySQLMetaImpl::DropAll() { try { ENGINE_LOG_DEBUG << "Drop all mysql meta"; - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); diff --git a/cpp/src/db/meta/MySQLMetaImpl.h b/cpp/src/db/meta/MySQLMetaImpl.h index 4950b604a3..b845b45a8c 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.h +++ b/cpp/src/db/meta/MySQLMetaImpl.h @@ -112,7 +112,7 @@ private: const int mode_; std::shared_ptr mysql_connection_pool_; - bool safe_grab = false; + bool safe_grab_ = false; // std::mutex connectionMutex_; }; // DBMetaImpl diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 7131a47800..aa8f16f5a8 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -128,16 +128,16 @@ void SqliteMetaImpl::ValidateMetaSchema() { } Status SqliteMetaImpl::Initialize() { - if (!boost::filesystem::is_directory(options_.path)) { - auto ret = boost::filesystem::create_directory(options_.path); + if (!boost::filesystem::is_directory(options_.path_)) { + auto ret = boost::filesystem::create_directory(options_.path_); if (!ret) { - std::string msg = "Failed to create db directory " + options_.path; + std::string msg = "Failed to create db directory " + options_.path_; ENGINE_LOG_ERROR << msg; return Status(DB_INVALID_PATH, msg); } } - ConnectorPtr = std::make_unique(StoragePrototype(options_.path + "/meta.sqlite")); + ConnectorPtr = std::make_unique(StoragePrototype(options_.path_ + "/meta.sqlite")); ValidateMetaSchema(); @@ -886,7 +886,7 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, // PXU TODO: Support Swap Status SqliteMetaImpl::Archive() { - auto &criterias = options_.archive_conf.GetCriterias(); + auto &criterias = options_.archive_conf_.GetCriterias(); if (criterias.size() == 0) { return Status::OK(); } diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 1409fff597..e7767aac1c 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -38,12 +38,12 @@ Status DBWrapper::StartService() { //db config engine::DBOptions opt; ConfigNode& db_config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); - opt.meta.backend_uri = db_config.GetValue(CONFIG_DB_URL); + opt.meta_.backend_uri_ = db_config.GetValue(CONFIG_DB_URL); std::string db_path = db_config.GetValue(CONFIG_DB_PATH); - opt.meta.path = db_path + "/db"; + opt.meta_.path_ = db_path + "/db"; std::string db_slave_path = db_config.GetValue(CONFIG_DB_SLAVE_PATH); - StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta.slave_paths); + StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta_.slave_paths_); // cache config ConfigNode& cache_config = ServerConfig::GetInstance().GetConfig(CONFIG_CACHE); @@ -52,13 +52,13 @@ Status DBWrapper::StartService() { ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); std::string mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); if (mode == "single") { - opt.mode = engine::DBOptions::MODE::SINGLE; + opt.mode_ = engine::DBOptions::MODE::SINGLE; } else if (mode == "cluster") { - opt.mode = engine::DBOptions::MODE::CLUSTER; + opt.mode_ = engine::DBOptions::MODE::CLUSTER; } else if (mode == "read_only") { - opt.mode = engine::DBOptions::MODE::READ_ONLY; + opt.mode_ = engine::DBOptions::MODE::READ_ONLY; } else { std::cerr << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" << std::endl; @@ -91,16 +91,16 @@ Status DBWrapper::StartService() { if(days > 0) { criterial[engine::ARCHIVE_CONF_DAYS] = days; } - opt.meta.archive_conf.SetCriterias(criterial); + opt.meta_.archive_conf_.SetCriterias(criterial); //create db root folder - Status status = CommonUtil::CreateDirectory(opt.meta.path); + Status status = CommonUtil::CreateDirectory(opt.meta_.path_); if(!status.ok()) { - std::cerr << "ERROR! Failed to create database root path: " << opt.meta.path << std::endl; + std::cerr << "ERROR! Failed to create database root path: " << opt.meta_.path_ << std::endl; kill(0, SIGUSR1); } - for(auto& path : opt.meta.slave_paths) { + for(auto& path : opt.meta_.slave_paths_) { status = CommonUtil::CreateDirectory(path); if(!status.ok()) { std::cerr << "ERROR! Failed to create database slave path: " << path << std::endl; diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 2d0a4e5c99..da6d96e217 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -136,18 +136,18 @@ Server::Daemonize() { stderr = fopen("/dev/null", "w+"); // Try to write PID of daemon to lockfile if (!pid_filename_.empty()) { - pid_fd = open(pid_filename_.c_str(), O_RDWR | O_CREAT, 0640); - if (pid_fd < 0) { + pid_fd_ = open(pid_filename_.c_str(), O_RDWR | O_CREAT, 0640); + if (pid_fd_ < 0) { std::cerr << "Can't open filename: " + pid_filename_ + ", Error: " + strerror(errno); exit(EXIT_FAILURE); } - if (lockf(pid_fd, F_TLOCK, 0) < 0) { + if (lockf(pid_fd_, F_TLOCK, 0) < 0) { std::cerr << "Can't lock filename: " + pid_filename_ + ", Error: " + strerror(errno); exit(EXIT_FAILURE); } std::string pid_file_context = std::to_string(getpid()); - ssize_t res = write(pid_fd, pid_file_context.c_str(), pid_file_context.size()); + ssize_t res = write(pid_fd_, pid_file_context.c_str(), pid_file_context.size()); if (res != 0) { return; } @@ -209,13 +209,13 @@ Server::Stop() { std::cerr << "Milvus server is going to shutdown ..." << std::endl; /* Unlock and close lockfile */ - if (pid_fd != -1) { - int ret = lockf(pid_fd, F_ULOCK, 0); + if (pid_fd_ != -1) { + int ret = lockf(pid_fd_, F_ULOCK, 0); if (ret != 0) { std::cerr << "Can't lock file: " << strerror(errno) << std::endl; exit(0); } - ret = close(pid_fd); + ret = close(pid_fd_); if (ret != 0) { std::cerr << "Can't close file: " << strerror(errno) << std::endl; exit(0); diff --git a/cpp/src/server/Server.h b/cpp/src/server/Server.h index 30ea95a7bb..710170f8e9 100644 --- a/cpp/src/server/Server.h +++ b/cpp/src/server/Server.h @@ -52,7 +52,7 @@ class Server { private: int64_t daemonized_ = 0; - int pid_fd = -1; + int pid_fd_ = -1; std::string pid_filename_; std::string config_filename_; std::string log_config_file_; diff --git a/cpp/src/utils/ThreadPool.h b/cpp/src/utils/ThreadPool.h index cb8755fa6b..8060ee4cf6 100644 --- a/cpp/src/utils/ThreadPool.h +++ b/cpp/src/utils/ThreadPool.h @@ -45,17 +45,17 @@ public: private: // need to keep track of threads so we can join them - std::vector workers; + std::vector workers_; // the task queue - std::queue > tasks; + std::queue > tasks_; - size_t max_queue_size; + size_t max_queue_size_; // synchronization - std::mutex queue_mutex; + std::mutex queue_mutex_; - std::condition_variable condition; + std::condition_variable condition_; bool stop; }; @@ -63,23 +63,23 @@ private: // the constructor just launches some amount of workers inline ThreadPool::ThreadPool(size_t threads, size_t queue_size) - : max_queue_size(queue_size), stop(false) { + : max_queue_size_(queue_size), stop(false) { for (size_t i = 0; i < threads; ++i) - workers.emplace_back( + workers_.emplace_back( [this] { for (;;) { std::function task; { - std::unique_lock lock(this->queue_mutex); - this->condition.wait(lock, - [this] { return this->stop || !this->tasks.empty(); }); - if (this->stop && this->tasks.empty()) + std::unique_lock lock(this->queue_mutex_); + this->condition_.wait(lock, + [this] { return this->stop || !this->tasks_.empty(); }); + if (this->stop && this->tasks_.empty()) return; - task = std::move(this->tasks.front()); - this->tasks.pop(); + task = std::move(this->tasks_.front()); + this->tasks_.pop(); } - this->condition.notify_all(); + this->condition_.notify_all(); task(); } @@ -99,27 +99,27 @@ auto ThreadPool::enqueue(F &&f, Args &&... args) std::future res = task->get_future(); { - std::unique_lock lock(queue_mutex); - this->condition.wait(lock, - [this] { return this->tasks.size() < max_queue_size; }); + std::unique_lock lock(queue_mutex_); + this->condition_.wait(lock, + [this] { return this->tasks_.size() < max_queue_size_; }); // don't allow enqueueing after stopping the pool if (stop) throw std::runtime_error("enqueue on stopped ThreadPool"); - tasks.emplace([task]() { (*task)(); }); + tasks_.emplace([task]() { (*task)(); }); } - condition.notify_all(); + condition_.notify_all(); return res; } // the destructor joins all threads inline ThreadPool::~ThreadPool() { { - std::unique_lock lock(queue_mutex); + std::unique_lock lock(queue_mutex_); stop = true; } - condition.notify_all(); - for (std::thread &worker: workers) + condition_.notify_all(); + for (std::thread &worker: workers_) worker.join(); } diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 7a6ee31243..23897408bc 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -117,11 +117,11 @@ TEST_F(MetaTest, TABLE_FILE_TEST) { TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { srand(time(0)); DBMetaOptions options; - options.path = "/tmp/milvus_test"; + options.path_ = "/tmp/milvus_test"; int days_num = rand() % 100; std::stringstream ss; ss << "days:" << days_num; - options.archive_conf = ArchiveConf("delete", ss.str()); + options.archive_conf_ = ArchiveConf("delete", ss.str()); meta::SqliteMetaImpl impl(options); auto table_id = "meta_test_table"; @@ -168,8 +168,8 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { TEST_F(MetaTest, ARCHIVE_TEST_DISK) { DBMetaOptions options; - options.path = "/tmp/milvus_test"; - options.archive_conf = ArchiveConf("delete", "disk:11"); + options.path_ = "/tmp/milvus_test"; + options.archive_conf_ = ArchiveConf("delete", "disk:11"); meta::SqliteMetaImpl impl(options); auto table_id = "meta_test_group"; diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index dd388dab7a..762e9944ee 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -77,7 +77,7 @@ TEST(DBMiscTest, OPTIONS_TEST) { TEST(DBMiscTest, META_TEST) { engine::DBMetaOptions options; - options.path = "/tmp/milvus_test"; + options.path_ = "/tmp/milvus_test"; engine::meta::SqliteMetaImpl impl(options); time_t tt; @@ -89,15 +89,15 @@ TEST(DBMiscTest, META_TEST) { TEST(DBMiscTest, UTILS_TEST) { engine::DBMetaOptions options; - options.path = "/tmp/milvus_test/main"; - options.slave_paths.push_back("/tmp/milvus_test/slave_1"); - options.slave_paths.push_back("/tmp/milvus_test/slave_2"); + options.path_ = "/tmp/milvus_test/main"; + options.slave_paths_.push_back("/tmp/milvus_test/slave_1"); + options.slave_paths_.push_back("/tmp/milvus_test/slave_2"); const std::string TABLE_NAME = "test_tbl"; auto status = engine::utils::CreateTablePath(options, TABLE_NAME); ASSERT_TRUE(status.ok()); - ASSERT_TRUE(boost::filesystem::exists(options.path)); - for(auto& path : options.slave_paths) { + ASSERT_TRUE(boost::filesystem::exists(options.path_)); + for(auto& path : options.slave_paths_) { ASSERT_TRUE(boost::filesystem::exists(path)); } diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index a7c9837f14..46b030235e 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -121,12 +121,12 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) { TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) { srand(time(0)); - DBMetaOptions options = GetOptions().meta; + DBMetaOptions options = GetOptions().meta_; int days_num = rand() % 100; std::stringstream ss; ss << "days:" << days_num; - options.archive_conf = ArchiveConf("delete", ss.str()); + options.archive_conf_ = ArchiveConf("delete", ss.str()); int mode = DBOptions::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); @@ -184,9 +184,9 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) { } TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) { - DBMetaOptions options = GetOptions().meta; + DBMetaOptions options = GetOptions().meta_; - options.archive_conf = ArchiveConf("delete", "disk:11"); + options.archive_conf_ = ArchiveConf("delete", "disk:11"); int mode = DBOptions::MODE::SINGLE; auto impl = meta::MySQLMetaImpl(options, mode); auto table_id = "meta_test_group"; diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index e9950a8574..0fb47df13c 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -71,8 +71,8 @@ void BaseTest::TearDown() { engine::DBOptions BaseTest::GetOptions() { auto options = engine::DBFactory::BuildOption(); - options.meta.path = "/tmp/milvus_test"; - options.meta.backend_uri = "sqlite://:@:/"; + options.meta_.path_ = "/tmp/milvus_test"; + options.meta_.backend_uri_ = "sqlite://:@:/"; return options; } @@ -110,15 +110,15 @@ void DBTest::TearDown() { BaseTest::TearDown(); auto options = GetOptions(); - boost::filesystem::remove_all(options.meta.path); + boost::filesystem::remove_all(options.meta_.path_); } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// engine::DBOptions DBTest2::GetOptions() { auto options = engine::DBFactory::BuildOption(); - options.meta.path = "/tmp/milvus_test"; - options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1"); - options.meta.backend_uri = "sqlite://:@:/"; + options.meta_.path_ = "/tmp/milvus_test"; + options.meta_.archive_conf_ = engine::ArchiveConf("delete", "disk:1"); + options.meta_.backend_uri_ = "sqlite://:@:/"; return options; } @@ -127,7 +127,7 @@ void MetaTest::SetUp() { BaseTest::SetUp(); auto options = GetOptions(); - impl_ = std::make_shared(options.meta); + impl_ = std::make_shared(options.meta_); } void MetaTest::TearDown() { @@ -136,17 +136,17 @@ void MetaTest::TearDown() { BaseTest::TearDown(); auto options = GetOptions(); - boost::filesystem::remove_all(options.meta.path); + boost::filesystem::remove_all(options.meta_.path_); } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// engine::DBOptions MySqlDBTest::GetOptions() { auto options = engine::DBFactory::BuildOption(); - options.meta.path = "/tmp/milvus_test"; - options.meta.backend_uri = DBTestEnvironment::getURI(); + options.meta_.path_ = "/tmp/milvus_test"; + options.meta_.backend_uri_ = DBTestEnvironment::getURI(); - if(options.meta.backend_uri.empty()) { - options.meta.backend_uri = "mysql://root:Fantast1c@192.168.1.194:3306/"; + if(options.meta_.backend_uri_.empty()) { + options.meta_.backend_uri_ = "mysql://root:Fantast1c@192.168.1.194:3306/"; } return options; @@ -157,7 +157,7 @@ void MySqlMetaTest::SetUp() { BaseTest::SetUp(); auto options = GetOptions(); - impl_ = std::make_shared(options.meta, options.mode); + impl_ = std::make_shared(options.meta_, options.mode_); } void MySqlMetaTest::TearDown() { @@ -166,16 +166,16 @@ void MySqlMetaTest::TearDown() { BaseTest::TearDown(); auto options = GetOptions(); - boost::filesystem::remove_all(options.meta.path); + boost::filesystem::remove_all(options.meta_.path_); } engine::DBOptions MySqlMetaTest::GetOptions() { auto options = engine::DBFactory::BuildOption(); - options.meta.path = "/tmp/milvus_test"; - options.meta.backend_uri = DBTestEnvironment::getURI(); + options.meta_.path_ = "/tmp/milvus_test"; + options.meta_.backend_uri_ = DBTestEnvironment::getURI(); - if(options.meta.backend_uri.empty()) { - options.meta.backend_uri = "mysql://root:Fantast1c@192.168.1.194:3306/"; + if(options.meta_.backend_uri_.empty()) { + options.meta_.backend_uri_ = "mysql://root:Fantast1c@192.168.1.194:3306/"; } return options; diff --git a/cpp/unittest/metrics/utils.cpp b/cpp/unittest/metrics/utils.cpp index 49ccc53d30..8d59a1e9c2 100644 --- a/cpp/unittest/metrics/utils.cpp +++ b/cpp/unittest/metrics/utils.cpp @@ -54,8 +54,8 @@ void MetricTest::InitLog() { engine::DBOptions MetricTest::GetOptions() { auto options = engine::DBFactory::BuildOption(); - options.meta.path = "/tmp/milvus_test"; - options.meta.backend_uri = "sqlite://:@:/"; + options.meta_.path_ = "/tmp/milvus_test"; + options.meta_.backend_uri_ = "sqlite://:@:/"; return options; } From d64389c7e3fdf832c9da015ef98283784e8ea886 Mon Sep 17 00:00:00 2001 From: wxyu Date: Sat, 21 Sep 2019 11:29:04 +0800 Subject: [PATCH 072/217] MS-585 Update namespace in scheduler Former-commit-id: 073ab3c91a97415efcb7d3be516a6bc00ee07a7a --- cpp/CHANGELOG.md | 1 + cpp/src/db/DBImpl.cpp | 4 +- cpp/src/scheduler/Algorithm.cpp | 2 +- cpp/src/scheduler/Algorithm.h | 2 +- cpp/src/scheduler/Definition.h | 48 +++++++++++++++++++ cpp/src/scheduler/JobMgr.h | 3 -- cpp/src/scheduler/ResourceFactory.cpp | 2 +- cpp/src/scheduler/ResourceFactory.h | 2 +- cpp/src/scheduler/ResourceMgr.cpp | 2 +- cpp/src/scheduler/ResourceMgr.h | 2 +- cpp/src/scheduler/SchedInst.cpp | 2 +- cpp/src/scheduler/SchedInst.h | 2 +- cpp/src/scheduler/Scheduler.cpp | 2 +- cpp/src/scheduler/Scheduler.h | 2 +- cpp/src/scheduler/TaskCreator.cpp | 4 +- cpp/src/scheduler/TaskCreator.h | 4 -- cpp/src/scheduler/TaskTable.cpp | 2 +- cpp/src/scheduler/TaskTable.h | 2 +- cpp/src/scheduler/Utils.cpp | 2 +- cpp/src/scheduler/Utils.h | 2 +- cpp/src/scheduler/action/Action.h | 2 +- .../scheduler/action/PushTaskToNeighbour.cpp | 2 +- cpp/src/scheduler/event/Event.h | 2 +- cpp/src/scheduler/event/EventDump.cpp | 2 +- cpp/src/scheduler/event/FinishTaskEvent.h | 2 +- cpp/src/scheduler/event/LoadCompletedEvent.h | 2 +- cpp/src/scheduler/event/StartUpEvent.h | 2 +- .../scheduler/event/TaskTableUpdatedEvent.h | 2 +- cpp/src/scheduler/resource/Connection.h | 2 +- cpp/src/scheduler/resource/CpuResource.cpp | 2 +- cpp/src/scheduler/resource/CpuResource.h | 2 +- cpp/src/scheduler/resource/DiskResource.cpp | 2 +- cpp/src/scheduler/resource/DiskResource.h | 2 +- cpp/src/scheduler/resource/GpuResource.cpp | 2 +- cpp/src/scheduler/resource/GpuResource.h | 2 +- cpp/src/scheduler/resource/Node.cpp | 2 +- cpp/src/scheduler/resource/Node.h | 2 +- cpp/src/scheduler/resource/Resource.cpp | 2 +- cpp/src/scheduler/resource/Resource.h | 2 +- cpp/src/scheduler/resource/TestResource.cpp | 2 +- cpp/src/scheduler/resource/TestResource.h | 2 +- cpp/src/scheduler/task/DeleteTask.cpp | 2 +- cpp/src/scheduler/task/DeleteTask.h | 2 +- cpp/src/scheduler/task/Path.h | 2 +- cpp/src/scheduler/task/SearchTask.cpp | 8 ++-- cpp/src/scheduler/task/SearchTask.h | 8 ++-- cpp/src/scheduler/task/Task.h | 2 +- cpp/src/scheduler/task/TestTask.cpp | 5 +- cpp/src/scheduler/task/TestTask.h | 4 +- cpp/src/scheduler/tasklabel/BroadcastLabel.h | 2 +- cpp/src/scheduler/tasklabel/DefaultLabel.h | 2 +- cpp/src/scheduler/tasklabel/SpecResLabel.h | 2 +- cpp/src/scheduler/tasklabel/TaskLabel.h | 2 +- cpp/src/server/Server.cpp | 4 +- cpp/src/server/grpc_impl/GrpcRequestTask.cpp | 2 +- cpp/unittest/db/search_test.cpp | 36 +++++++------- cpp/unittest/db/utils.cpp | 22 ++++----- cpp/unittest/scheduler/algorithm_test.cpp | 2 +- cpp/unittest/scheduler/event_test.cpp | 2 +- cpp/unittest/scheduler/node_test.cpp | 2 +- cpp/unittest/scheduler/normal_test.cpp | 4 +- .../scheduler/resource_factory_test.cpp | 2 +- cpp/unittest/scheduler/resource_mgr_test.cpp | 4 +- cpp/unittest/scheduler/resource_test.cpp | 10 ++-- cpp/unittest/scheduler/schedinst_test.cpp | 2 +- cpp/unittest/scheduler/scheduler_test.cpp | 21 ++++---- cpp/unittest/scheduler/task_test.cpp | 2 +- cpp/unittest/scheduler/tasktable_test.cpp | 6 +-- 68 files changed, 169 insertions(+), 127 deletions(-) create mode 100644 cpp/src/scheduler/Definition.h diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 09929ed8d0..5fb91c72d2 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -19,6 +19,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-566 - Refactor cmake - MS-555 - Remove old scheduler - MS-578 - Makesure milvus5.0 don't crack 0.3.1 data +- MS-585 - Update namespace in scheduler ## New Feature diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 8323564024..a763c9199c 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -133,9 +133,9 @@ Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& date meta_ptr_->DeleteTable(table_id); //soft delete table //scheduler will determine when to delete table files - auto nres = ResMgrInst::GetInstance()->GetNumOfComputeResource(); + auto nres = scheduler::ResMgrInst::GetInstance()->GetNumOfComputeResource(); scheduler::DeleteJobPtr job = std::make_shared(0, table_id, meta_ptr_, nres); - JobMgrInst::GetInstance()->Put(job); + scheduler::JobMgrInst::GetInstance()->Put(job); job->WaitAndDelete(); } else { meta_ptr_->DropPartitionsByDates(table_id, dates); diff --git a/cpp/src/scheduler/Algorithm.cpp b/cpp/src/scheduler/Algorithm.cpp index 0a9aefbce0..b388c17bdd 100644 --- a/cpp/src/scheduler/Algorithm.cpp +++ b/cpp/src/scheduler/Algorithm.cpp @@ -20,7 +20,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { constexpr uint64_t MAXINT = std::numeric_limits::max(); diff --git a/cpp/src/scheduler/Algorithm.h b/cpp/src/scheduler/Algorithm.h index 763ef8ad5d..be77cf351f 100644 --- a/cpp/src/scheduler/Algorithm.h +++ b/cpp/src/scheduler/Algorithm.h @@ -24,7 +24,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { uint64_t ShortestPath(const ResourcePtr &src, diff --git a/cpp/src/scheduler/Definition.h b/cpp/src/scheduler/Definition.h new file mode 100644 index 0000000000..9ac647de88 --- /dev/null +++ b/cpp/src/scheduler/Definition.h @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "db/meta/MetaTypes.h" +#include "db/engine/EngineFactory.h" +#include "db/engine/ExecutionEngine.h" + + +namespace zilliz { +namespace milvus { +namespace scheduler { + +using TableFileSchemaPtr = engine::meta::TableFileSchemaPtr; +using TableFileSchema = engine::meta::TableFileSchema; + +using ExecutionEnginePtr = engine::ExecutionEnginePtr; +using EngineFactory = engine::EngineFactory; +using EngineType = engine::EngineType; +using MetricType = engine::MetricType; + +} +} +} diff --git a/cpp/src/scheduler/JobMgr.h b/cpp/src/scheduler/JobMgr.h index 1d8720c4b4..8d22b5d4f5 100644 --- a/cpp/src/scheduler/JobMgr.h +++ b/cpp/src/scheduler/JobMgr.h @@ -36,9 +36,6 @@ namespace zilliz { namespace milvus { namespace scheduler { -using engine::TaskPtr; -using engine::ResourceMgrPtr; - class JobMgr { public: explicit diff --git a/cpp/src/scheduler/ResourceFactory.cpp b/cpp/src/scheduler/ResourceFactory.cpp index 40cd00f206..066b47aad7 100644 --- a/cpp/src/scheduler/ResourceFactory.cpp +++ b/cpp/src/scheduler/ResourceFactory.cpp @@ -21,7 +21,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { std::shared_ptr ResourceFactory::Create(const std::string &name, diff --git a/cpp/src/scheduler/ResourceFactory.h b/cpp/src/scheduler/ResourceFactory.h index 58a29497b0..39fb79154b 100644 --- a/cpp/src/scheduler/ResourceFactory.h +++ b/cpp/src/scheduler/ResourceFactory.h @@ -28,7 +28,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class ResourceFactory { public: diff --git a/cpp/src/scheduler/ResourceMgr.cpp b/cpp/src/scheduler/ResourceMgr.cpp index dafc15713a..0f9fab8e8d 100644 --- a/cpp/src/scheduler/ResourceMgr.cpp +++ b/cpp/src/scheduler/ResourceMgr.cpp @@ -22,7 +22,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { void diff --git a/cpp/src/scheduler/ResourceMgr.h b/cpp/src/scheduler/ResourceMgr.h index ebe0894770..8bd045ab7d 100644 --- a/cpp/src/scheduler/ResourceMgr.h +++ b/cpp/src/scheduler/ResourceMgr.h @@ -30,7 +30,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class ResourceMgr { public: diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index 3b14158e58..a13640073d 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -25,7 +25,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { ResourceMgrPtr ResMgrInst::instance = nullptr; std::mutex ResMgrInst::mutex_; diff --git a/cpp/src/scheduler/SchedInst.h b/cpp/src/scheduler/SchedInst.h index 010a7e84f5..d23c200b05 100644 --- a/cpp/src/scheduler/SchedInst.h +++ b/cpp/src/scheduler/SchedInst.h @@ -27,7 +27,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class ResMgrInst { public: diff --git a/cpp/src/scheduler/Scheduler.cpp b/cpp/src/scheduler/Scheduler.cpp index ff66e6ac21..abaccd4fcb 100644 --- a/cpp/src/scheduler/Scheduler.cpp +++ b/cpp/src/scheduler/Scheduler.cpp @@ -25,7 +25,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { Scheduler::Scheduler(ResourceMgrWPtr res_mgr) : running_(false), diff --git a/cpp/src/scheduler/Scheduler.h b/cpp/src/scheduler/Scheduler.h index feb1374a8b..b591fdd256 100644 --- a/cpp/src/scheduler/Scheduler.h +++ b/cpp/src/scheduler/Scheduler.h @@ -30,7 +30,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { // TODO: refactor, not friendly to unittest, logical in framework code diff --git a/cpp/src/scheduler/TaskCreator.cpp b/cpp/src/scheduler/TaskCreator.cpp index 1d471135d6..0d3b2a8a96 100644 --- a/cpp/src/scheduler/TaskCreator.cpp +++ b/cpp/src/scheduler/TaskCreator.cpp @@ -45,7 +45,7 @@ TaskCreator::Create(const SearchJobPtr &job) { std::vector tasks; for (auto &index_file : job->index_files()) { auto task = std::make_shared(index_file.second); - task->label() = std::make_shared(); + task->label() = std::make_shared(); task->job_ = job; tasks.emplace_back(task); } @@ -57,7 +57,7 @@ std::vector TaskCreator::Create(const DeleteJobPtr &job) { std::vector tasks; auto task = std::make_shared(job); - task->label() = std::make_shared(); + task->label() = std::make_shared(); task->job_ = job; tasks.emplace_back(task); diff --git a/cpp/src/scheduler/TaskCreator.h b/cpp/src/scheduler/TaskCreator.h index b0e947600d..d4e87bd0a2 100644 --- a/cpp/src/scheduler/TaskCreator.h +++ b/cpp/src/scheduler/TaskCreator.h @@ -39,10 +39,6 @@ namespace zilliz { namespace milvus { namespace scheduler { -using engine::TaskPtr; -using engine::XSearchTask; -using engine::XDeleteTask; - class TaskCreator { public: static std::vector diff --git a/cpp/src/scheduler/TaskTable.cpp b/cpp/src/scheduler/TaskTable.cpp index 1678665593..badcb46c2d 100644 --- a/cpp/src/scheduler/TaskTable.cpp +++ b/cpp/src/scheduler/TaskTable.cpp @@ -27,7 +27,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { std::string ToString(TaskTableItemState state) { diff --git a/cpp/src/scheduler/TaskTable.h b/cpp/src/scheduler/TaskTable.h index c4be442d11..c9557c8f9e 100644 --- a/cpp/src/scheduler/TaskTable.h +++ b/cpp/src/scheduler/TaskTable.h @@ -27,7 +27,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { enum class TaskTableItemState { INVALID, diff --git a/cpp/src/scheduler/Utils.cpp b/cpp/src/scheduler/Utils.cpp index b011751105..f207845c20 100644 --- a/cpp/src/scheduler/Utils.cpp +++ b/cpp/src/scheduler/Utils.cpp @@ -24,7 +24,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { uint64_t get_current_timestamp() { diff --git a/cpp/src/scheduler/Utils.h b/cpp/src/scheduler/Utils.h index 066695b35b..f1056f104a 100644 --- a/cpp/src/scheduler/Utils.h +++ b/cpp/src/scheduler/Utils.h @@ -21,7 +21,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { uint64_t get_current_timestamp(); diff --git a/cpp/src/scheduler/action/Action.h b/cpp/src/scheduler/action/Action.h index 57fc8fe686..9f2fa9f6c5 100644 --- a/cpp/src/scheduler/action/Action.h +++ b/cpp/src/scheduler/action/Action.h @@ -23,7 +23,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class Action { public: diff --git a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp index 76beaa3e07..d53903ba2d 100644 --- a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp @@ -25,7 +25,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { std::vector get_neighbours(const ResourcePtr &self) { diff --git a/cpp/src/scheduler/event/Event.h b/cpp/src/scheduler/event/Event.h index af8be5ecf0..b00bd7f902 100644 --- a/cpp/src/scheduler/event/Event.h +++ b/cpp/src/scheduler/event/Event.h @@ -21,7 +21,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { enum class EventType { START_UP, diff --git a/cpp/src/scheduler/event/EventDump.cpp b/cpp/src/scheduler/event/EventDump.cpp index 384fd98e8d..3d06dfd5f5 100644 --- a/cpp/src/scheduler/event/EventDump.cpp +++ b/cpp/src/scheduler/event/EventDump.cpp @@ -25,7 +25,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { std::ostream &operator<<(std::ostream &out, const Event &event) { out << event.Dump(); diff --git a/cpp/src/scheduler/event/FinishTaskEvent.h b/cpp/src/scheduler/event/FinishTaskEvent.h index 3c88a62693..3ffa855f30 100644 --- a/cpp/src/scheduler/event/FinishTaskEvent.h +++ b/cpp/src/scheduler/event/FinishTaskEvent.h @@ -22,7 +22,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class FinishTaskEvent : public Event { public: diff --git a/cpp/src/scheduler/event/LoadCompletedEvent.h b/cpp/src/scheduler/event/LoadCompletedEvent.h index 7e7abaa697..286f26723f 100644 --- a/cpp/src/scheduler/event/LoadCompletedEvent.h +++ b/cpp/src/scheduler/event/LoadCompletedEvent.h @@ -23,7 +23,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class LoadCompletedEvent : public Event { public: diff --git a/cpp/src/scheduler/event/StartUpEvent.h b/cpp/src/scheduler/event/StartUpEvent.h index 0ae4db5ce0..089eb1377b 100644 --- a/cpp/src/scheduler/event/StartUpEvent.h +++ b/cpp/src/scheduler/event/StartUpEvent.h @@ -22,7 +22,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class StartUpEvent : public Event { public: diff --git a/cpp/src/scheduler/event/TaskTableUpdatedEvent.h b/cpp/src/scheduler/event/TaskTableUpdatedEvent.h index 766df9254d..70ee28fdab 100644 --- a/cpp/src/scheduler/event/TaskTableUpdatedEvent.h +++ b/cpp/src/scheduler/event/TaskTableUpdatedEvent.h @@ -22,7 +22,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class TaskTableUpdatedEvent : public Event { public: diff --git a/cpp/src/scheduler/resource/Connection.h b/cpp/src/scheduler/resource/Connection.h index ae7bf88e86..38fbc65bb5 100644 --- a/cpp/src/scheduler/resource/Connection.h +++ b/cpp/src/scheduler/resource/Connection.h @@ -23,7 +23,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class Connection { public: diff --git a/cpp/src/scheduler/resource/CpuResource.cpp b/cpp/src/scheduler/resource/CpuResource.cpp index 863a427096..3933bbb5a0 100644 --- a/cpp/src/scheduler/resource/CpuResource.cpp +++ b/cpp/src/scheduler/resource/CpuResource.cpp @@ -21,7 +21,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { std::ostream &operator<<(std::ostream &out, const CpuResource &resource) { out << resource.Dump(); diff --git a/cpp/src/scheduler/resource/CpuResource.h b/cpp/src/scheduler/resource/CpuResource.h index 19bac6090d..1f6f235621 100644 --- a/cpp/src/scheduler/resource/CpuResource.h +++ b/cpp/src/scheduler/resource/CpuResource.h @@ -24,7 +24,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class CpuResource : public Resource { public: diff --git a/cpp/src/scheduler/resource/DiskResource.cpp b/cpp/src/scheduler/resource/DiskResource.cpp index bb07f96b96..6495396fdb 100644 --- a/cpp/src/scheduler/resource/DiskResource.cpp +++ b/cpp/src/scheduler/resource/DiskResource.cpp @@ -20,7 +20,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { std::ostream &operator<<(std::ostream &out, const DiskResource &resource) { out << resource.Dump(); diff --git a/cpp/src/scheduler/resource/DiskResource.h b/cpp/src/scheduler/resource/DiskResource.h index 8c8a6ae46a..b9e7a4035c 100644 --- a/cpp/src/scheduler/resource/DiskResource.h +++ b/cpp/src/scheduler/resource/DiskResource.h @@ -23,7 +23,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class DiskResource : public Resource { public: diff --git a/cpp/src/scheduler/resource/GpuResource.cpp b/cpp/src/scheduler/resource/GpuResource.cpp index c8959cf393..c3981e65cb 100644 --- a/cpp/src/scheduler/resource/GpuResource.cpp +++ b/cpp/src/scheduler/resource/GpuResource.cpp @@ -21,7 +21,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { std::ostream &operator<<(std::ostream &out, const GpuResource &resource) { out << resource.Dump(); diff --git a/cpp/src/scheduler/resource/GpuResource.h b/cpp/src/scheduler/resource/GpuResource.h index 12947a35fa..f08e767c51 100644 --- a/cpp/src/scheduler/resource/GpuResource.h +++ b/cpp/src/scheduler/resource/GpuResource.h @@ -23,7 +23,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class GpuResource : public Resource { public: diff --git a/cpp/src/scheduler/resource/Node.cpp b/cpp/src/scheduler/resource/Node.cpp index d4c42a5a97..719bfb1427 100644 --- a/cpp/src/scheduler/resource/Node.cpp +++ b/cpp/src/scheduler/resource/Node.cpp @@ -22,7 +22,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { Node::Node() { static std::atomic_uint_fast8_t counter(0); diff --git a/cpp/src/scheduler/resource/Node.h b/cpp/src/scheduler/resource/Node.h index 311ec8156a..46506da10d 100644 --- a/cpp/src/scheduler/resource/Node.h +++ b/cpp/src/scheduler/resource/Node.h @@ -27,7 +27,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class Node; diff --git a/cpp/src/scheduler/resource/Resource.cpp b/cpp/src/scheduler/resource/Resource.cpp index b094bb1c06..22e7eb8502 100644 --- a/cpp/src/scheduler/resource/Resource.cpp +++ b/cpp/src/scheduler/resource/Resource.cpp @@ -22,7 +22,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { std::ostream & operator<<(std::ostream &out, const Resource &resource) { diff --git a/cpp/src/scheduler/resource/Resource.h b/cpp/src/scheduler/resource/Resource.h index f49ea71b5c..0b0eaae304 100644 --- a/cpp/src/scheduler/resource/Resource.h +++ b/cpp/src/scheduler/resource/Resource.h @@ -37,7 +37,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { // TODO(wxyu): Storage, Route, Executor enum class ResourceType { diff --git a/cpp/src/scheduler/resource/TestResource.cpp b/cpp/src/scheduler/resource/TestResource.cpp index ebdcb403bd..b49ea12d77 100644 --- a/cpp/src/scheduler/resource/TestResource.cpp +++ b/cpp/src/scheduler/resource/TestResource.cpp @@ -20,7 +20,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { std::ostream &operator<<(std::ostream &out, const TestResource &resource) { out << resource.Dump(); diff --git a/cpp/src/scheduler/resource/TestResource.h b/cpp/src/scheduler/resource/TestResource.h index b8f749ba58..861efbd825 100644 --- a/cpp/src/scheduler/resource/TestResource.h +++ b/cpp/src/scheduler/resource/TestResource.h @@ -23,7 +23,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class TestResource : public Resource { public: diff --git a/cpp/src/scheduler/task/DeleteTask.cpp b/cpp/src/scheduler/task/DeleteTask.cpp index b1b1697f0e..65a7a537fd 100644 --- a/cpp/src/scheduler/task/DeleteTask.cpp +++ b/cpp/src/scheduler/task/DeleteTask.cpp @@ -21,7 +21,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { XDeleteTask::XDeleteTask(const scheduler::DeleteJobPtr &delete_job) : Task(TaskType::DeleteTask), delete_job_(delete_job) {} diff --git a/cpp/src/scheduler/task/DeleteTask.h b/cpp/src/scheduler/task/DeleteTask.h index f21a1a96e4..b64f9a977f 100644 --- a/cpp/src/scheduler/task/DeleteTask.h +++ b/cpp/src/scheduler/task/DeleteTask.h @@ -23,7 +23,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class XDeleteTask : public Task { public: diff --git a/cpp/src/scheduler/task/Path.h b/cpp/src/scheduler/task/Path.h index 64df58b46c..a463494b77 100644 --- a/cpp/src/scheduler/task/Path.h +++ b/cpp/src/scheduler/task/Path.h @@ -23,7 +23,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class Path { public: diff --git a/cpp/src/scheduler/task/SearchTask.cpp b/cpp/src/scheduler/task/SearchTask.cpp index e39cd00d2e..284152f14e 100644 --- a/cpp/src/scheduler/task/SearchTask.cpp +++ b/cpp/src/scheduler/task/SearchTask.cpp @@ -27,7 +27,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { static constexpr size_t PARALLEL_REDUCE_THRESHOLD = 10000; static constexpr size_t PARALLEL_REDUCE_BATCH = 1000; @@ -79,8 +79,8 @@ std::mutex XSearchTask::merge_mutex_; void CollectFileMetrics(int file_type, size_t file_size) { switch (file_type) { - case meta::TableFileSchema::RAW: - case meta::TableFileSchema::TO_INDEX: { + case TableFileSchema::RAW: + case TableFileSchema::TO_INDEX: { server::Metrics::GetInstance().RawFileSizeHistogramObserve(file_size); server::Metrics::GetInstance().RawFileSizeTotalIncrement(file_size); server::Metrics::GetInstance().RawFileSizeGaugeSet(file_size); @@ -95,7 +95,7 @@ CollectFileMetrics(int file_type, size_t file_size) { } } -XSearchTask::XSearchTask(meta::TableFileSchemaPtr file) +XSearchTask::XSearchTask(TableFileSchemaPtr file) : Task(TaskType::SearchTask), file_(file) { if (file_) { index_engine_ = EngineFactory::Build(file_->dimension_, diff --git a/cpp/src/scheduler/task/SearchTask.h b/cpp/src/scheduler/task/SearchTask.h index e54e742e24..6f0ec6bc53 100644 --- a/cpp/src/scheduler/task/SearchTask.h +++ b/cpp/src/scheduler/task/SearchTask.h @@ -18,19 +18,19 @@ #pragma once #include "Task.h" -#include "db/meta/MetaTypes.h" #include "scheduler/job/SearchJob.h" +#include "scheduler/Definition.h" namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { // TODO: rewrite class XSearchTask : public Task { public: explicit - XSearchTask(meta::TableFileSchemaPtr file); + XSearchTask(TableFileSchemaPtr file); void Load(LoadType type, uint8_t device_id) override; @@ -56,7 +56,7 @@ public: scheduler::ResultSet &result_target); public: - meta::TableFileSchemaPtr file_; + TableFileSchemaPtr file_; size_t index_id_ = 0; int index_type_ = 0; diff --git a/cpp/src/scheduler/task/Task.h b/cpp/src/scheduler/task/Task.h index c4a2c1cb4c..785ef9cfb7 100644 --- a/cpp/src/scheduler/task/Task.h +++ b/cpp/src/scheduler/task/Task.h @@ -28,7 +28,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { enum class LoadType { DISK2CPU, diff --git a/cpp/src/scheduler/task/TestTask.cpp b/cpp/src/scheduler/task/TestTask.cpp index 5d5ac976b5..32e0a8587e 100644 --- a/cpp/src/scheduler/task/TestTask.cpp +++ b/cpp/src/scheduler/task/TestTask.cpp @@ -22,10 +22,9 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { - -TestTask::TestTask(meta::TableFileSchemaPtr &file) : XSearchTask(file) {} +TestTask::TestTask(TableFileSchemaPtr &file) : XSearchTask(file) {} void TestTask::Load(LoadType type, uint8_t device_id) { diff --git a/cpp/src/scheduler/task/TestTask.h b/cpp/src/scheduler/task/TestTask.h index e489fc537f..3367506d88 100644 --- a/cpp/src/scheduler/task/TestTask.h +++ b/cpp/src/scheduler/task/TestTask.h @@ -22,12 +22,12 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class TestTask : public XSearchTask { public: explicit - TestTask(meta::TableFileSchemaPtr& file); + TestTask(TableFileSchemaPtr& file); public: void diff --git a/cpp/src/scheduler/tasklabel/BroadcastLabel.h b/cpp/src/scheduler/tasklabel/BroadcastLabel.h index 2faa49acd1..1c89460df0 100644 --- a/cpp/src/scheduler/tasklabel/BroadcastLabel.h +++ b/cpp/src/scheduler/tasklabel/BroadcastLabel.h @@ -24,7 +24,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class BroadcastLabel : public TaskLabel { diff --git a/cpp/src/scheduler/tasklabel/DefaultLabel.h b/cpp/src/scheduler/tasklabel/DefaultLabel.h index d1619684b9..d61028117d 100644 --- a/cpp/src/scheduler/tasklabel/DefaultLabel.h +++ b/cpp/src/scheduler/tasklabel/DefaultLabel.h @@ -24,7 +24,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class DefaultLabel : public TaskLabel { public: diff --git a/cpp/src/scheduler/tasklabel/SpecResLabel.h b/cpp/src/scheduler/tasklabel/SpecResLabel.h index 3955625412..8e94a7677b 100644 --- a/cpp/src/scheduler/tasklabel/SpecResLabel.h +++ b/cpp/src/scheduler/tasklabel/SpecResLabel.h @@ -29,7 +29,7 @@ using ResourceWPtr = std::weak_ptr; namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class SpecResLabel : public TaskLabel { public: diff --git a/cpp/src/scheduler/tasklabel/TaskLabel.h b/cpp/src/scheduler/tasklabel/TaskLabel.h index da8049ebf9..5d43adb7db 100644 --- a/cpp/src/scheduler/tasklabel/TaskLabel.h +++ b/cpp/src/scheduler/tasklabel/TaskLabel.h @@ -21,7 +21,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { enum class TaskLabelType { DEFAULT, // means can be executed in any resource diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index da6d96e217..8dacf04d41 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -252,7 +252,7 @@ Server::LoadConfig() { void Server::StartService() { engine::KnowhereResource::Initialize(); - engine::StartSchedulerService(); + scheduler::StartSchedulerService(); DBWrapper::GetInstance().StartService(); grpc::GrpcServer::GetInstance().Start(); } @@ -261,7 +261,7 @@ void Server::StopService() { grpc::GrpcServer::GetInstance().Stop(); DBWrapper::GetInstance().StopService(); - engine::StopSchedulerService(); + scheduler::StopSchedulerService(); engine::KnowhereResource::Finalize(); } diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index ad0abd713d..0a8a84855d 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -772,7 +772,7 @@ CmdTask::OnExecute() { if (cmd_ == "version") { result_ = MILVUS_VERSION; } else if (cmd_ == "tasktable") { - result_ = engine::ResMgrInst::GetInstance()->DumpTaskTables(); + result_ = scheduler::ResMgrInst::GetInstance()->DumpTaskTables(); } else { result_ = "OK"; diff --git a/cpp/unittest/db/search_test.cpp b/cpp/unittest/db/search_test.cpp index 295650f2d2..b1c83c6db7 100644 --- a/cpp/unittest/db/search_test.cpp +++ b/cpp/unittest/db/search_test.cpp @@ -127,23 +127,23 @@ TEST(DBSearchTest, TOPK_TEST) { std::vector target_ids; std::vector target_distence; scheduler::ResultSet src_result; - auto status = engine::XSearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result); + auto status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result); ASSERT_FALSE(status.ok()); ASSERT_TRUE(src_result.empty()); BuildResult(NQ, TOP_K, ascending, target_ids, target_distence); - status = engine::XSearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result); + status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result); ASSERT_TRUE(status.ok()); ASSERT_EQ(src_result.size(), NQ); scheduler::ResultSet target_result; - status = engine::XSearchTask::TopkResult(target_result, TOP_K, ascending, target_result); + status = scheduler::XSearchTask::TopkResult(target_result, TOP_K, ascending, target_result); ASSERT_TRUE(status.ok()); - status = engine::XSearchTask::TopkResult(target_result, TOP_K, ascending, src_result); + status = scheduler::XSearchTask::TopkResult(target_result, TOP_K, ascending, src_result); ASSERT_FALSE(status.ok()); - status = engine::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result); + status = scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result); ASSERT_TRUE(status.ok()); ASSERT_TRUE(src_result.empty()); ASSERT_EQ(target_result.size(), NQ); @@ -153,10 +153,10 @@ TEST(DBSearchTest, TOPK_TEST) { uint64_t wrong_topk = TOP_K - 10; BuildResult(NQ, wrong_topk, ascending, src_ids, src_distence); - status = engine::XSearchTask::ClusterResult(src_ids, src_distence, NQ, wrong_topk, src_result); + status = scheduler::XSearchTask::ClusterResult(src_ids, src_distence, NQ, wrong_topk, src_result); ASSERT_TRUE(status.ok()); - status = engine::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result); + status = scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result); ASSERT_TRUE(status.ok()); for(uint64_t i = 0; i < NQ; i++) { ASSERT_EQ(target_result[i].size(), TOP_K); @@ -165,7 +165,7 @@ TEST(DBSearchTest, TOPK_TEST) { wrong_topk = TOP_K + 10; BuildResult(NQ, wrong_topk, ascending, src_ids, src_distence); - status = engine::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result); + status = scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result); ASSERT_TRUE(status.ok()); for(uint64_t i = 0; i < NQ; i++) { ASSERT_EQ(target_result[i].size(), TOP_K); @@ -183,15 +183,15 @@ TEST(DBSearchTest, MERGE_TEST) { uint64_t src_count = 5, target_count = 8; BuildResult(1, src_count, ascending, src_ids, src_distence); BuildResult(1, target_count, ascending, target_ids, target_distence); - auto status = engine::XSearchTask::ClusterResult(src_ids, src_distence, 1, src_count, src_result); + auto status = scheduler::XSearchTask::ClusterResult(src_ids, src_distence, 1, src_count, src_result); ASSERT_TRUE(status.ok()); - status = engine::XSearchTask::ClusterResult(target_ids, target_distence, 1, target_count, target_result); + status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, 1, target_count, target_result); ASSERT_TRUE(status.ok()); { scheduler::Id2DistanceMap src = src_result[0]; scheduler::Id2DistanceMap target = target_result[0]; - status = engine::XSearchTask::MergeResult(src, target, 10, ascending); + status = scheduler::XSearchTask::MergeResult(src, target, 10, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), 10); CheckResult(src_result[0], target_result[0], target, ascending); @@ -200,7 +200,7 @@ TEST(DBSearchTest, MERGE_TEST) { { scheduler::Id2DistanceMap src = src_result[0]; scheduler::Id2DistanceMap target; - status = engine::XSearchTask::MergeResult(src, target, 10, ascending); + status = scheduler::XSearchTask::MergeResult(src, target, 10, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), src_count); ASSERT_TRUE(src.empty()); @@ -210,7 +210,7 @@ TEST(DBSearchTest, MERGE_TEST) { { scheduler::Id2DistanceMap src = src_result[0]; scheduler::Id2DistanceMap target = target_result[0]; - status = engine::XSearchTask::MergeResult(src, target, 30, ascending); + status = scheduler::XSearchTask::MergeResult(src, target, 30, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), src_count + target_count); CheckResult(src_result[0], target_result[0], target, ascending); @@ -219,7 +219,7 @@ TEST(DBSearchTest, MERGE_TEST) { { scheduler::Id2DistanceMap target = src_result[0]; scheduler::Id2DistanceMap src = target_result[0]; - status = engine::XSearchTask::MergeResult(src, target, 30, ascending); + status = scheduler::XSearchTask::MergeResult(src, target, 30, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), src_count + target_count); CheckResult(src_result[0], target_result[0], target, ascending); @@ -242,7 +242,7 @@ TEST(DBSearchTest, PARALLEL_CLUSTER_TEST) { BuildResult(nq, topk, ascending, target_ids, target_distence); rc.RecordSection("build id/dietance map"); - auto status = engine::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result); + auto status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result); ASSERT_TRUE(status.ok()); ASSERT_EQ(src_result.size(), nq); @@ -282,14 +282,14 @@ TEST(DBSearchTest, PARALLEL_TOPK_TEST) { TimeRecorder rc("DoCluster"); BuildResult(nq, topk, ascending, target_ids, target_distence); - auto status = engine::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result); + auto status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result); rc.RecordSection("cluster result"); BuildResult(nq, insufficient_topk, ascending, insufficient_ids, insufficient_distence); - status = engine::XSearchTask::ClusterResult(target_ids, target_distence, nq, insufficient_topk, insufficient_result); + status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, nq, insufficient_topk, insufficient_result); rc.RecordSection("cluster result"); - engine::XSearchTask::TopkResult(insufficient_result, topk, ascending, src_result); + scheduler::XSearchTask::TopkResult(insufficient_result, topk, ascending, src_result); ASSERT_TRUE(status.ok()); rc.RecordSection("topk"); diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 0fb47df13c..08e3c8f090 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -80,20 +80,20 @@ engine::DBOptions BaseTest::GetOptions() { void DBTest::SetUp() { BaseTest::SetUp(); - auto res_mgr = engine::ResMgrInst::GetInstance(); + auto res_mgr = scheduler::ResMgrInst::GetInstance(); res_mgr->Clear(); - res_mgr->Add(engine::ResourceFactory::Create("disk", "DISK", 0, true, false)); - res_mgr->Add(engine::ResourceFactory::Create("cpu", "CPU", 0, true, false)); - res_mgr->Add(engine::ResourceFactory::Create("gtx1660", "GPU", 0, true, true)); + res_mgr->Add(scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false)); + res_mgr->Add(scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, false)); + res_mgr->Add(scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true)); - auto default_conn = engine::Connection("IO", 500.0); - auto PCIE = engine::Connection("IO", 11000.0); + auto default_conn = scheduler::Connection("IO", 500.0); + auto PCIE = scheduler::Connection("IO", 11000.0); res_mgr->Connect("disk", "cpu", default_conn); res_mgr->Connect("cpu", "gtx1660", PCIE); res_mgr->Start(); - engine::SchedInst::GetInstance()->Start(); + scheduler::SchedInst::GetInstance()->Start(); - engine::JobMgrInst::GetInstance()->Start(); + scheduler::JobMgrInst::GetInstance()->Start(); auto options = GetOptions(); db_ = engine::DBFactory::Build(options); @@ -103,9 +103,9 @@ void DBTest::TearDown() { db_->Stop(); db_->DropAll(); - engine::JobMgrInst::GetInstance()->Stop(); - engine::SchedInst::GetInstance()->Stop(); - engine::ResMgrInst::GetInstance()->Stop(); + scheduler::JobMgrInst::GetInstance()->Stop(); + scheduler::SchedInst::GetInstance()->Stop(); + scheduler::ResMgrInst::GetInstance()->Stop(); BaseTest::TearDown(); diff --git a/cpp/unittest/scheduler/algorithm_test.cpp b/cpp/unittest/scheduler/algorithm_test.cpp index 5272edf0d7..fe619c4981 100644 --- a/cpp/unittest/scheduler/algorithm_test.cpp +++ b/cpp/unittest/scheduler/algorithm_test.cpp @@ -26,7 +26,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class AlgorithmTest : public testing::Test { protected: diff --git a/cpp/unittest/scheduler/event_test.cpp b/cpp/unittest/scheduler/event_test.cpp index 895aabe19d..34d4c2ce23 100644 --- a/cpp/unittest/scheduler/event_test.cpp +++ b/cpp/unittest/scheduler/event_test.cpp @@ -25,7 +25,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { TEST(EventTest, START_UP_EVENT) { ResourceWPtr res(ResourcePtr(nullptr)); diff --git a/cpp/unittest/scheduler/node_test.cpp b/cpp/unittest/scheduler/node_test.cpp index 36dd10b73d..b87611d3ca 100644 --- a/cpp/unittest/scheduler/node_test.cpp +++ b/cpp/unittest/scheduler/node_test.cpp @@ -20,7 +20,7 @@ #include -using namespace zilliz::milvus::engine; +using namespace zilliz::milvus::scheduler; class NodeTest : public ::testing::Test { protected: diff --git a/cpp/unittest/scheduler/normal_test.cpp b/cpp/unittest/scheduler/normal_test.cpp index 02ed3dae69..535fa96731 100644 --- a/cpp/unittest/scheduler/normal_test.cpp +++ b/cpp/unittest/scheduler/normal_test.cpp @@ -26,7 +26,7 @@ #include -using namespace zilliz::milvus::engine; +using namespace zilliz::milvus::scheduler; TEST(NormalTest, INST_TEST) { @@ -46,7 +46,7 @@ TEST(NormalTest, INST_TEST) { const uint64_t NUM_TASK = 1000; std::vector> tasks; - meta::TableFileSchemaPtr dummy = nullptr; + TableFileSchemaPtr dummy = nullptr; auto disks = res_mgr->GetDiskResources(); ASSERT_FALSE(disks.empty()); diff --git a/cpp/unittest/scheduler/resource_factory_test.cpp b/cpp/unittest/scheduler/resource_factory_test.cpp index 1bf39f669f..e2a84257eb 100644 --- a/cpp/unittest/scheduler/resource_factory_test.cpp +++ b/cpp/unittest/scheduler/resource_factory_test.cpp @@ -20,7 +20,7 @@ #include -using namespace zilliz::milvus::engine; +using namespace zilliz::milvus::scheduler; TEST(ResourceFactoryTest, CREATE) { auto disk = ResourceFactory::Create("ssd", "DISK", 0); diff --git a/cpp/unittest/scheduler/resource_mgr_test.cpp b/cpp/unittest/scheduler/resource_mgr_test.cpp index 8aaef5bdf5..c2be785a00 100644 --- a/cpp/unittest/scheduler/resource_mgr_test.cpp +++ b/cpp/unittest/scheduler/resource_mgr_test.cpp @@ -27,7 +27,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { /************ ResourceMgrBaseTest ************/ @@ -187,7 +187,7 @@ TEST_F(ResourceMgrAdvanceTest, REGISTER_SUBSCRIBER) { flag = true; }; mgr1_->RegisterSubscriber(callback); - meta::TableFileSchemaPtr dummy = nullptr; + TableFileSchemaPtr dummy = nullptr; disk_res->task_table().Put(std::make_shared(dummy)); sleep(1); ASSERT_TRUE(flag); diff --git a/cpp/unittest/scheduler/resource_test.cpp b/cpp/unittest/scheduler/resource_test.cpp index 1fdeb13062..b335a601db 100644 --- a/cpp/unittest/scheduler/resource_test.cpp +++ b/cpp/unittest/scheduler/resource_test.cpp @@ -29,7 +29,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { /************ ResourceBaseTest ************/ class ResourceBaseTest : public testing::Test { @@ -180,7 +180,7 @@ protected: TEST_F(ResourceAdvanceTest, DISK_RESOURCE_TEST) { const uint64_t NUM = 100; std::vector> tasks; - meta::TableFileSchemaPtr dummy = nullptr; + TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { auto task = std::make_shared(dummy); tasks.push_back(task); @@ -205,7 +205,7 @@ TEST_F(ResourceAdvanceTest, DISK_RESOURCE_TEST) { TEST_F(ResourceAdvanceTest, CPU_RESOURCE_TEST) { const uint64_t NUM = 100; std::vector> tasks; - meta::TableFileSchemaPtr dummy = nullptr; + TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { auto task = std::make_shared(dummy); tasks.push_back(task); @@ -230,7 +230,7 @@ TEST_F(ResourceAdvanceTest, CPU_RESOURCE_TEST) { TEST_F(ResourceAdvanceTest, GPU_RESOURCE_TEST) { const uint64_t NUM = 100; std::vector> tasks; - meta::TableFileSchemaPtr dummy = nullptr; + TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { auto task = std::make_shared(dummy); tasks.push_back(task); @@ -255,7 +255,7 @@ TEST_F(ResourceAdvanceTest, GPU_RESOURCE_TEST) { TEST_F(ResourceAdvanceTest, TEST_RESOURCE_TEST) { const uint64_t NUM = 100; std::vector> tasks; - meta::TableFileSchemaPtr dummy = nullptr; + TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < NUM; ++i) { auto task = std::make_shared(dummy); tasks.push_back(task); diff --git a/cpp/unittest/scheduler/schedinst_test.cpp b/cpp/unittest/scheduler/schedinst_test.cpp index 27b6a17bbb..26ecce9f64 100644 --- a/cpp/unittest/scheduler/schedinst_test.cpp +++ b/cpp/unittest/scheduler/schedinst_test.cpp @@ -24,7 +24,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class SchedInstTest : public testing::Test { diff --git a/cpp/unittest/scheduler/scheduler_test.cpp b/cpp/unittest/scheduler/scheduler_test.cpp index ebe3a1f20d..dd2c8e8cb9 100644 --- a/cpp/unittest/scheduler/scheduler_test.cpp +++ b/cpp/unittest/scheduler/scheduler_test.cpp @@ -15,22 +15,23 @@ // specific language governing permissions and limitations // under the License. -#include "scheduler/Scheduler.h" #include -#include "src/scheduler/tasklabel/DefaultLabel.h" -#include "cache/DataObj.h" -#include "cache/GpuCacheMgr.h" + +#include "scheduler/Scheduler.h" +#include "scheduler/tasklabel/DefaultLabel.h" +#include "scheduler/tasklabel/SpecResLabel.h" #include "scheduler/task/TestTask.h" #include "scheduler/ResourceFactory.h" #include "scheduler/resource/Resource.h" +#include "cache/DataObj.h" +#include "cache/GpuCacheMgr.h" #include "utils/Error.h" -#include "src/wrapper/vec_index.h" -#include "scheduler/tasklabel/SpecResLabel.h" +#include "wrapper/vec_index.h" namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { class MockVecIndex : public engine::VecIndex { public: @@ -155,7 +156,7 @@ insert_dummy_index_into_gpu_cache(uint64_t device_id) { TEST_F(SchedulerTest, ON_LOAD_COMPLETED) { const uint64_t NUM = 10; std::vector> tasks; - meta::TableFileSchemaPtr dummy = std::make_shared(); + TableFileSchemaPtr dummy = std::make_shared(); dummy->location_ = "location"; insert_dummy_index_into_gpu_cache(1); @@ -175,7 +176,7 @@ TEST_F(SchedulerTest, ON_LOAD_COMPLETED) { TEST_F(SchedulerTest, PUSH_TASK_TO_NEIGHBOUR_RANDOMLY_TEST) { const uint64_t NUM = 10; std::vector> tasks; - meta::TableFileSchemaPtr dummy1 = std::make_shared(); + TableFileSchemaPtr dummy1 = std::make_shared(); dummy1->location_ = "location"; tasks.clear(); @@ -246,7 +247,7 @@ protected: TEST_F(SchedulerTest2, SPECIFIED_RESOURCE_TEST) { const uint64_t NUM = 10; std::vector> tasks; - meta::TableFileSchemaPtr dummy = std::make_shared(); + TableFileSchemaPtr dummy = std::make_shared(); dummy->location_ = "location"; for (uint64_t i = 0; i < NUM; ++i) { diff --git a/cpp/unittest/scheduler/task_test.cpp b/cpp/unittest/scheduler/task_test.cpp index 8a8b455ae5..17d1e28cd6 100644 --- a/cpp/unittest/scheduler/task_test.cpp +++ b/cpp/unittest/scheduler/task_test.cpp @@ -22,7 +22,7 @@ namespace zilliz { namespace milvus { -namespace engine { +namespace scheduler { TEST(TaskTest, INVALID_INDEX) { diff --git a/cpp/unittest/scheduler/tasktable_test.cpp b/cpp/unittest/scheduler/tasktable_test.cpp index 0c3f65e146..5c64a72351 100644 --- a/cpp/unittest/scheduler/tasktable_test.cpp +++ b/cpp/unittest/scheduler/tasktable_test.cpp @@ -21,7 +21,7 @@ #include -using namespace zilliz::milvus::engine; +using namespace zilliz::milvus::scheduler; /************ TaskTableBaseTest ************/ @@ -169,7 +169,7 @@ class TaskTableBaseTest : public ::testing::Test { protected: void SetUp() override { - meta::TableFileSchemaPtr dummy = nullptr; + TableFileSchemaPtr dummy = nullptr; invalid_task_ = nullptr; task1_ = std::make_shared(dummy); task2_ = std::make_shared(dummy); @@ -339,7 +339,7 @@ class TaskTableAdvanceTest : public ::testing::Test { protected: void SetUp() override { - meta::TableFileSchemaPtr dummy = nullptr; + TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < 8; ++i) { auto task = std::make_shared(dummy); table1_.Put(task); From 22563d83d7efce599799ae57ac52b736e61220a2 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Sat, 21 Sep 2019 15:02:35 +0800 Subject: [PATCH 073/217] MS-574 add GET config APIs Former-commit-id: 6011f32d650d26367ab84f4bf8521889881f2629 --- cpp/conf/server_config.template | 8 +- cpp/src/cache/CpuCacheMgr.cpp | 17 +- cpp/src/cache/GpuCacheMgr.cpp | 17 +- cpp/src/db/engine/ExecutionEngineImpl.cpp | 3 +- cpp/src/metrics/Metrics.cpp | 6 +- cpp/src/metrics/PrometheusMetrics.cpp | 9 +- cpp/src/scheduler/SchedInst.cpp | 7 +- cpp/src/server/DBWrapper.cpp | 34 ++-- cpp/src/server/Server.cpp | 9 +- cpp/src/server/ServerConfig.cpp | 220 +++++++++++++++++++--- cpp/src/server/ServerConfig.h | 45 ++++- cpp/src/server/grpc_impl/GrpcServer.cpp | 8 +- cpp/src/wrapper/KnowhereResource.cpp | 9 +- 13 files changed, 280 insertions(+), 112 deletions(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index a96d2d81cf..6364698100 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -5,13 +5,13 @@ server_config: time_zone: UTC+8 db_config: - db_path: @MILVUS_DB_PATH@ # milvus database path - db_slave_path: # secondary database path, split by semicolon + path: @MILVUS_DB_PATH@ # milvus database path + slave_path: # secondary database path, split by semicolon # URI format: dialect://username:password@host:port/database # All parts except dialect are optional, but you MUST include the delimiters # Currently dialect supports mysql or sqlite - db_backend_url: sqlite://:@:/ + backend_url: sqlite://:@:/ archive_disk_threshold: 0 # GB, file will be archived when disk usage exceed, 0 for no limit archive_days_threshold: 0 # DAYS, older files will be archived, 0 for no limit @@ -27,7 +27,7 @@ metric_config: cache_config: cpu_mem_capacity: 16 # GB, CPU memory size used for cache cpu_mem_threshold: 0.85 # percent of data kept when cache cleanup triggered - insert_immediately: false # whether load data into cache when insert + cache_insert_data: false # whether load data into cache when insert engine_config: blas_threshold: 20 diff --git a/cpp/src/cache/CpuCacheMgr.cpp b/cpp/src/cache/CpuCacheMgr.cpp index ee30494192..7141394fa1 100644 --- a/cpp/src/cache/CpuCacheMgr.cpp +++ b/cpp/src/cache/CpuCacheMgr.cpp @@ -29,19 +29,16 @@ namespace { } CpuCacheMgr::CpuCacheMgr() { - server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); - int64_t cap = - config.GetInt64Value(server::CONFIG_CACHE_CPU_MEM_CAPACITY, std::stoi(server::CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT)); - cap *= unit; + server::ServerConfig& config = server::ServerConfig::GetInstance(); + int64_t cap = config.GetCacheConfigCpuMemCapacity() * unit; cache_ = std::make_shared>(cap, 1UL<<32); - double free_percent = - config.GetDoubleValue(server::CONFIG_CACHE_CPU_MEM_THRESHOLD, std::stof(server::CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT)); - if(free_percent > 0.0 && free_percent <= 1.0) { - cache_->set_freemem_percent(free_percent); + float cpu_mem_threshold = config.GetCacheConfigCpuMemThreshold(); + if (cpu_mem_threshold > 0.0 && cpu_mem_threshold <= 1.0) { + cache_->set_freemem_percent(cpu_mem_threshold); } else { - SERVER_LOG_ERROR << "Invalid cache_free_percent: " << free_percent << - ", defaultly set to " << cache_->freemem_percent(); + SERVER_LOG_ERROR << "Invalid cpu_mem_threshold: " << cpu_mem_threshold + << ", by default set to " << cache_->freemem_percent(); } } diff --git a/cpp/src/cache/GpuCacheMgr.cpp b/cpp/src/cache/GpuCacheMgr.cpp index 58c7db3fd6..ba5665f5f2 100644 --- a/cpp/src/cache/GpuCacheMgr.cpp +++ b/cpp/src/cache/GpuCacheMgr.cpp @@ -33,20 +33,17 @@ namespace { } GpuCacheMgr::GpuCacheMgr() { - server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); + server::ServerConfig& config = server::ServerConfig::GetInstance(); - int64_t cap = - config.GetInt64Value(server::CONFIG_CACHE_GPU_MEM_CAPACITY, std::stoi(server::CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT)); - cap *= G_BYTE; + int32_t cap = config.GetCacheConfigGpuMemCapacity() * G_BYTE; cache_ = std::make_shared>(cap, 1UL<<32); - double free_percent = - config.GetDoubleValue(server::CONFIG_CACHE_GPU_MEM_THRESHOLD, std::stof(server::CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT)); - if (free_percent > 0.0 && free_percent <= 1.0) { - cache_->set_freemem_percent(free_percent); + float gpu_mem_threshold = config.GetCacheConfigGpuMemThreshold(); + if (gpu_mem_threshold > 0.0 && gpu_mem_threshold <= 1.0) { + cache_->set_freemem_percent(gpu_mem_threshold); } else { - SERVER_LOG_ERROR << "Invalid gpu_cache_free_percent: " << free_percent << - ", defaultly set to " << cache_->freemem_percent(); + SERVER_LOG_ERROR << "Invalid gpu_mem_threshold: " << gpu_mem_threshold + << ", by default set to " << cache_->freemem_percent(); } } diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 65483690d6..960f3baa89 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -335,8 +335,7 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { Status ExecutionEngineImpl::Init() { using namespace zilliz::milvus::server; ServerConfig &config = ServerConfig::GetInstance(); - ConfigNode server_config = config.GetConfig(CONFIG_DB); - gpu_num_ = server_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); + gpu_num_ = config.GetDBConfigBuildIndexGPU(); return Status::OK(); } diff --git a/cpp/src/metrics/Metrics.cpp b/cpp/src/metrics/Metrics.cpp index ba51d7fd94..59eaea6355 100644 --- a/cpp/src/metrics/Metrics.cpp +++ b/cpp/src/metrics/Metrics.cpp @@ -16,6 +16,7 @@ // under the License. #include "Metrics.h" +#include "server/ServerConfig.h" #include "PrometheusMetrics.h" @@ -31,9 +32,8 @@ Metrics::GetInstance() { MetricsBase & Metrics::CreateMetricsCollector() { - ConfigNode &config = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC); - std::string collector_type_str = - config.GetValue(CONFIG_METRIC_COLLECTOR, CONFIG_METRIC_COLLECTOR_DEFAULT); + ServerConfig &config = ServerConfig::GetInstance(); + std::string collector_type_str = config.GetMetricConfigCollector(); if (collector_type_str == "prometheus") { return PrometheusMetrics::GetInstance(); diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index cc8aca204f..b07e91ea32 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -29,15 +29,12 @@ namespace server { ErrorCode PrometheusMetrics::Init() { try { - ConfigNode &metric_config = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC); - startup_ = - metric_config.GetBoolValue(CONFIG_METRIC_AUTO_BOOTUP, std::stoi(CONFIG_METRIC_AUTO_BOOTUP_DEFAULT)); + ServerConfig &config = ServerConfig::GetInstance(); + startup_ = config.GetMetricConfigAutoBootup(); if (!startup_) return SERVER_SUCCESS; // Following should be read from config file. - ConfigNode &prometheus_config = metric_config.GetChild(CONFIG_METRIC_PROMETHEUS); - const std::string bind_address = - prometheus_config.GetValue(CONFIG_METRIC_PROMETHEUS_PORT, CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); + const std::string bind_address = config.GetMetricConfigPrometheusPort(); const std::string uri = std::string("/tmp/metrics"); const std::size_t num_threads = 2; diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index 0b1e7cd302..54b870100a 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -38,10 +38,9 @@ std::mutex JobMgrInst::mutex_; void load_simple_config() { - server::ConfigNode &config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_RESOURCE); - auto mode = config.GetValue(server::CONFIG_RESOURCE_MODE, server::CONFIG_RESOURCE_MODE_DEFAULT); - - auto pool = config.GetSequence(server::CONFIG_RESOURCE_POOL); + server::ServerConfig &config = server::ServerConfig::GetInstance(); + auto mode = config.GetResourceConfigMode(); + auto pool = config.GetResourceConfigPool(); bool cpu = false; std::set gpu_ids; for (auto &resource : pool) { diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 756438dfb3..3d958b15a4 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -35,23 +35,20 @@ DBWrapper::DBWrapper() { } Status DBWrapper::StartService() { + ServerConfig& config = ServerConfig::GetInstance(); + //db config engine::DBOptions opt; - ConfigNode& db_config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); - opt.meta.backend_uri = db_config.GetValue(CONFIG_DB_BACKEND_URL); - std::string db_path = db_config.GetValue(CONFIG_DB_PATH); - opt.meta.path = db_path + "/db"; + opt.meta.backend_uri = config.GetDBConfigBackendUrl(); + opt.meta.path = config.GetDBConfigPath() + "/db"; - std::string db_slave_path = db_config.GetValue(CONFIG_DB_SLAVE_PATH); + std::string db_slave_path = config.GetDBConfigSlavePath(); StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta.slave_paths); // cache config - ConfigNode& cache_config = ServerConfig::GetInstance().GetConfig(CONFIG_CACHE); - opt.insert_cache_immediately_ = - cache_config.GetBoolValue(CONFIG_CACHE_INSERT_IMMEDIATELY, std::stoi(CONFIG_CACHE_INSERT_IMMEDIATELY_DEFAULT)); + opt.insert_cache_immediately_ = config.GetCacheConfigCacheInsertData(); - ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); - std::string mode = serverConfig.GetValue(CONFIG_SERVER_MODE, CONFIG_SERVER_MODE_DEFAULT); + std::string mode = config.GetServerConfigMode(); if (mode == "single") { opt.mode = engine::DBOptions::MODE::SINGLE; } @@ -67,9 +64,7 @@ Status DBWrapper::StartService() { } // engine config - ConfigNode& engine_config = ServerConfig::GetInstance().GetConfig(CONFIG_ENGINE); - int32_t omp_thread = - engine_config.GetInt32Value(CONFIG_ENGINE_OMP_THREAD_NUM, std::stoi(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT)); + int32_t omp_thread = config.GetEngineConfigOmpThreadNum(); if(omp_thread > 0) { omp_set_num_threads(omp_thread); SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; @@ -82,19 +77,16 @@ Status DBWrapper::StartService() { } //init faiss global variable - faiss::distance_compute_blas_threshold = - engine_config.GetInt32Value(CONFIG_ENGINE_BLAS_THRESHOLD, std::stoi(CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT)); + faiss::distance_compute_blas_threshold = config.GetEngineConfigBlasThreshold(); //set archive config engine::ArchiveConf::CriteriaT criterial; - int64_t disk = - db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, std::stoi(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT)); - int64_t days = - db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, std::stoi(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT)); - if(disk > 0) { + int32_t disk = config.GetDBConfigArchiveDiskThreshold(); + int32_t days = config.GetDBConfigArchiveDaysThreshold(); + if (disk > 0) { criterial[engine::ARCHIVE_CONF_DISK] = disk; } - if(days > 0) { + if (days > 0) { criterial[engine::ARCHIVE_CONF_DAYS] = days; } opt.meta.archive_conf.SetCriterias(criterial); diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 7f85ed6509..d8924e9d7d 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -162,9 +162,7 @@ Server::Start() { /* log path is defined in Config file, so InitLog must be called after LoadConfig */ ServerConfig &config = ServerConfig::GetInstance(); - ConfigNode server_config = config.GetConfig(CONFIG_SERVER); - - std::string time_zone = server_config.GetValue(CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); + std::string time_zone = config.GetServerConfigTimeZone(); if (time_zone.length() == 3) { time_zone = "CUT"; } else { @@ -232,8 +230,9 @@ Server::Stop() { ErrorCode Server::LoadConfig() { - ServerConfig::GetInstance().LoadConfigFile(config_filename_); - auto status = ServerConfig::GetInstance().ValidateConfig(); + ServerConfig& server_config = ServerConfig::GetInstance(); + server_config.LoadConfigFile(config_filename_); + auto status = server_config.ValidateConfig(); if (!status.ok()) { std::cerr << "Failed to load config file: " << config_filename_ << std::endl; exit(0); diff --git a/cpp/src/server/ServerConfig.cpp b/cpp/src/server/ServerConfig.cpp index 0f51ecf6e2..35012192e8 100644 --- a/cpp/src/server/ServerConfig.cpp +++ b/cpp/src/server/ServerConfig.cpp @@ -42,30 +42,29 @@ ServerConfig::GetInstance() { } Status -ServerConfig::LoadConfigFile(const std::string &config_filename) { - std::string filename = config_filename; +ServerConfig::LoadConfigFile(const std::string &filename) { if (filename.empty()) { - std::cerr << "ERROR: a config file is required" << std::endl; - exit(1);//directly exit program if config file not specified + std::cerr << "ERROR: need specify config file" << std::endl; + exit(1); } - struct stat directoryStat; - int statOK = stat(filename.c_str(), &directoryStat); + struct stat dirStat; + int statOK = stat(filename.c_str(), &dirStat); if (statOK != 0) { - std::cerr << "ERROR: " << filename << " not found!" << std::endl; - exit(1);//directly exit program if config file not found + std::cerr << "ERROR: Config file not exist: " << filename << std::endl; + exit(1); } try { ConfigMgr *mgr = const_cast(ConfigMgr::GetInstance()); ErrorCode err = mgr->LoadConfigFile(filename); if (err != 0) { - std::cerr << "Server failed to load config file" << std::endl; - exit(1);//directly exit program if the config file is illegal + std::cerr << "Server failed to load config file: " << filename << std::endl; + exit(1); } } catch (YAML::Exception &e) { - std::cerr << "Server failed to load config file: " << std::endl; - exit(1);//directly exit program if the config file is illegal + std::cerr << "Server failed to load config file: " << filename << std::endl; + exit(1); } return Status::OK(); @@ -73,27 +72,25 @@ ServerConfig::LoadConfigFile(const std::string &config_filename) { Status ServerConfig::ValidateConfig() { - - bool okay = true; if (!CheckServerConfig().ok()) { - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Server config validation check fail"); } if (!CheckDBConfig().ok()) { - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "DB config validation check fail"); } if (!CheckMetricConfig().ok()) { - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Metric config validation check fail"); } if (!CheckCacheConfig().ok()) { - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Cache config validation check fail"); } if (!CheckEngineConfig().ok()) { - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Engine config validation check fail"); } if (!CheckResourceConfig().ok()) { - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Resource config validation check fail"); } - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Config validation not pass")); + return Status::OK(); } Status @@ -201,18 +198,18 @@ ServerConfig::CheckDBConfig() { okay = false; } - std::string insert_buffer_size_str = db_config.GetValue(CONFIG_DB_BUFFER_SIZE, CONFIG_DB_BUFFER_SIZE_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(insert_buffer_size_str).ok()) { - std::cerr << "ERROR: insert_buffer_size " << insert_buffer_size_str << " is not a number" << std::endl; + std::string buffer_size_str = db_config.GetValue(CONFIG_DB_BUFFER_SIZE, CONFIG_DB_BUFFER_SIZE_DEFAULT); + if (!ValidationUtil::ValidateStringIsNumber(buffer_size_str).ok()) { + std::cerr << "ERROR: buffer_size " << buffer_size_str << " is not a number" << std::endl; okay = false; } else { - uint64_t insert_buffer_size = (uint64_t) std::stol(insert_buffer_size_str); - insert_buffer_size *= GB; + uint64_t buffer_size = (uint64_t) std::stol(buffer_size_str); + buffer_size *= GB; unsigned long total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); - if (insert_buffer_size >= total_mem) { - std::cerr << "ERROR: insert_buffer_size exceed system memory" << std::endl; + if (buffer_size >= total_mem) { + std::cerr << "ERROR: buffer_size exceed system memory" << std::endl; okay = false; } } @@ -317,7 +314,7 @@ ServerConfig::CheckCacheConfig() { } std::string insert_cache_immediately_str = - cache_config.GetValue(CONFIG_CACHE_INSERT_IMMEDIATELY, CONFIG_CACHE_INSERT_IMMEDIATELY_DEFAULT); + cache_config.GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); if (!ValidationUtil::ValidateStringIsBool(insert_cache_immediately_str).ok()) { std::cerr << "ERROR: invalid insert_cache_immediately config: " << insert_cache_immediately_str << std::endl; okay = false; @@ -629,6 +626,171 @@ ServerConfig::GetConfig(const std::string &name) { return root_node.GetChild(name); } +/* server config */ +std::string +ServerConfig::GetServerConfigAddress() { + ConfigNode server_config = GetConfig(CONFIG_SERVER); + return server_config.GetValue(CONFIG_SERVER_ADDRESS, + CONFIG_SERVER_ADDRESS_DEFAULT); +} + +std::string +ServerConfig::GetServerConfigPort() { + ConfigNode server_config = GetConfig(CONFIG_SERVER); + return server_config.GetValue(CONFIG_SERVER_PORT, + CONFIG_SERVER_PORT_DEFAULT); +} + +std::string +ServerConfig::GetServerConfigMode() { + ConfigNode server_config = GetConfig(CONFIG_SERVER); + return server_config.GetValue(CONFIG_SERVER_MODE, + CONFIG_SERVER_MODE_DEFAULT); +} + +std::string +ServerConfig::GetServerConfigTimeZone() { + ConfigNode server_config = GetConfig(CONFIG_SERVER); + return server_config.GetValue(CONFIG_SERVER_TIME_ZONE, + CONFIG_SERVER_TIME_ZONE_DEFAULT); +} + +/* db config */ +std::string +ServerConfig::GetDBConfigPath() { + ConfigNode db_config = GetConfig(CONFIG_DB); + return db_config.GetValue(CONFIG_DB_PATH, + CONFIG_DB_PATH_DEFAULT); +} + +std::string +ServerConfig::GetDBConfigSlavePath() { + ConfigNode db_config = GetConfig(CONFIG_DB); + return db_config.GetValue(CONFIG_DB_SLAVE_PATH, + CONFIG_DB_SLAVE_PATH_DEFAULT); +} + +std::string +ServerConfig::GetDBConfigBackendUrl() { + ConfigNode db_config = GetConfig(CONFIG_DB); + return db_config.GetValue(CONFIG_DB_BACKEND_URL, + CONFIG_DB_BACKEND_URL_DEFAULT); +} + +int32_t +ServerConfig::GetDBConfigArchiveDiskThreshold() { + ConfigNode db_config = GetConfig(CONFIG_DB); + return db_config.GetInt32Value(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, + std::stoi(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT)); +} + +int32_t +ServerConfig::GetDBConfigArchiveDaysThreshold() { + ConfigNode db_config = GetConfig(CONFIG_DB); + return db_config.GetInt32Value(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, + std::stoi(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT)); +} + +int32_t +ServerConfig::GetDBConfigBufferSize() { + ConfigNode db_config = GetConfig(CONFIG_DB); + return db_config.GetInt32Value(CONFIG_DB_BUFFER_SIZE, + std::stoi(CONFIG_DB_BUFFER_SIZE_DEFAULT)); +} + +int32_t +ServerConfig::GetDBConfigBuildIndexGPU() { + ConfigNode db_config = GetConfig(CONFIG_DB); + return db_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, + std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); +} + +/* metric config */ +bool +ServerConfig::GetMetricConfigAutoBootup() { + ConfigNode metric_config = GetConfig(CONFIG_METRIC); + return metric_config.GetBoolValue(CONFIG_METRIC_AUTO_BOOTUP, + std::stoi(CONFIG_METRIC_AUTO_BOOTUP_DEFAULT)); +} + +std::string +ServerConfig::GetMetricConfigCollector() { + ConfigNode metric_config = GetConfig(CONFIG_METRIC); + return metric_config.GetValue(CONFIG_METRIC_COLLECTOR, + CONFIG_METRIC_COLLECTOR_DEFAULT); +} + +std::string +ServerConfig::GetMetricConfigPrometheusPort() { + ConfigNode metric_config = GetConfig(CONFIG_METRIC); + return metric_config.GetValue(CONFIG_METRIC_PROMETHEUS_PORT, + CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); +} + +/* cache config */ +int32_t +ServerConfig::GetCacheConfigCpuMemCapacity() { + ConfigNode cache_config = GetConfig(CONFIG_CACHE); + return cache_config.GetInt32Value(CONFIG_CACHE_CPU_MEM_CAPACITY, + std::stoi(CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT)); +} + +float +ServerConfig::GetCacheConfigCpuMemThreshold() { + ConfigNode cache_config = GetConfig(CONFIG_CACHE); + return cache_config.GetFloatValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, + std::stof(CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT)); +} + +int32_t +ServerConfig::GetCacheConfigGpuMemCapacity() { + ConfigNode cache_config = GetConfig(CONFIG_CACHE); + return cache_config.GetInt32Value(CONFIG_CACHE_GPU_MEM_CAPACITY, + std::stoi(CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT)); +} + +float +ServerConfig::GetCacheConfigGpuMemThreshold() { + ConfigNode cache_config = GetConfig(CONFIG_CACHE); + return cache_config.GetFloatValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, + std::stof(CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT)); +} + +bool +ServerConfig::GetCacheConfigCacheInsertData() { + ConfigNode cache_config = GetConfig(CONFIG_CACHE); + return cache_config.GetBoolValue(CONFIG_CACHE_CACHE_INSERT_DATA, + std::stoi(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT)); +} + +/* engine config */ +int32_t +ServerConfig::GetEngineConfigBlasThreshold() { + ConfigNode engine_config = GetConfig(CONFIG_ENGINE); + return engine_config.GetInt32Value(CONFIG_ENGINE_BLAS_THRESHOLD, + std::stoi(CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT)); +} + +int32_t +ServerConfig::GetEngineConfigOmpThreadNum() { + ConfigNode engine_config = GetConfig(CONFIG_ENGINE); + return engine_config.GetInt32Value(CONFIG_ENGINE_OMP_THREAD_NUM, + std::stoi(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT)); +} + +/* resource config */ +std::string +ServerConfig::GetResourceConfigMode() { + ConfigNode resource_config = GetConfig(CONFIG_RESOURCE); + return resource_config.GetValue(CONFIG_RESOURCE_MODE, + CONFIG_RESOURCE_MODE_DEFAULT); +} + +std::vector +ServerConfig::GetResourceConfigPool() { + ConfigNode resource_config = GetConfig(CONFIG_RESOURCE); + return resource_config.GetSequence(CONFIG_RESOURCE_POOL); +} } } diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index e515b74645..1453adde49 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -39,11 +39,11 @@ static const char* CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; /* db config */ static const char* CONFIG_DB = "db_config"; -static const char* CONFIG_DB_PATH = "db_path"; +static const char* CONFIG_DB_PATH = "path"; static const char* CONFIG_DB_PATH_DEFAULT = "/tmp/milvus"; -static const char* CONFIG_DB_SLAVE_PATH = "db_slave_path"; +static const char* CONFIG_DB_SLAVE_PATH = "slave_path"; static const char* CONFIG_DB_SLAVE_PATH_DEFAULT = ""; -static const char* CONFIG_DB_BACKEND_URL = "db_backend_url"; +static const char* CONFIG_DB_BACKEND_URL = "backend_url"; static const char* CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/"; static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold"; static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0"; @@ -64,8 +64,8 @@ static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD = "cpu_mem_threshold"; static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT = "0.85"; static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD = "gpu_mem_threshold"; static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT = "0.85"; -static const char* CONFIG_CACHE_INSERT_IMMEDIATELY = "insert_immediately"; -static const char* CONFIG_CACHE_INSERT_IMMEDIATELY_DEFAULT = "0"; +static const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; +static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "0"; /* metric config */ static const char* CONFIG_METRIC = "metric_config"; @@ -95,20 +95,51 @@ class ServerConfig { public: static ServerConfig &GetInstance(); - Status LoadConfigFile(const std::string& config_filename); + Status LoadConfigFile(const std::string& filename); Status ValidateConfig(); void PrintAll() const; + private: ConfigNode GetConfig(const std::string& name) const; ConfigNode& GetConfig(const std::string& name); - private: Status CheckServerConfig(); Status CheckDBConfig(); Status CheckMetricConfig(); Status CheckCacheConfig(); Status CheckEngineConfig(); Status CheckResourceConfig(); + + public: + std::string GetServerConfigAddress(); + std::string GetServerConfigPort(); + std::string GetServerConfigMode(); + std::string GetServerConfigTimeZone(); + + std::string GetDBConfigPath(); + std::string GetDBConfigSlavePath(); + std::string GetDBConfigBackendUrl(); + int32_t GetDBConfigArchiveDiskThreshold(); + int32_t GetDBConfigArchiveDaysThreshold(); + int32_t GetDBConfigBufferSize(); + int32_t GetDBConfigBuildIndexGPU(); + + bool GetMetricConfigAutoBootup(); + std::string GetMetricConfigCollector(); + std::string GetMetricConfigPrometheusPort(); + + int32_t GetCacheConfigCpuMemCapacity(); + float GetCacheConfigCpuMemThreshold(); + int32_t GetCacheConfigGpuMemCapacity(); + float GetCacheConfigGpuMemThreshold(); + bool GetCacheConfigCacheInsertData(); + + int32_t GetEngineConfigBlasThreshold(); + int32_t GetEngineConfigOmpThreadNum(); + + std::string GetResourceConfigMode(); + std::vector + GetResourceConfigPool(); }; } diff --git a/cpp/src/server/grpc_impl/GrpcServer.cpp b/cpp/src/server/grpc_impl/GrpcServer.cpp index 5160e0c8ad..e330f5483c 100644 --- a/cpp/src/server/grpc_impl/GrpcServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcServer.cpp @@ -74,12 +74,10 @@ GrpcServer::Stop() { Status GrpcServer::StartService() { ServerConfig &config = ServerConfig::GetInstance(); - ConfigNode server_config = config.GetConfig(CONFIG_SERVER); - ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); - std::string address = server_config.GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); - int32_t port = server_config.GetInt32Value(CONFIG_SERVER_PORT, std::stoi(CONFIG_SERVER_PORT_DEFAULT)); + std::string address = config.GetServerConfigAddress(); + std::string port = config.GetServerConfigPort(); - std::string server_address(address + ":" + std::to_string(port)); + std::string server_address(address + ":" + port); ::grpc::ServerBuilder builder; builder.SetOption(std::unique_ptr<::grpc::ServerBuilderOption>(new NoReusePortOption)); diff --git a/cpp/src/wrapper/KnowhereResource.cpp b/cpp/src/wrapper/KnowhereResource.cpp index e59b56b574..dfd75df253 100644 --- a/cpp/src/wrapper/KnowhereResource.cpp +++ b/cpp/src/wrapper/KnowhereResource.cpp @@ -38,16 +38,13 @@ ErrorCode KnowhereResource::Initialize() { GpuResourcesArray gpu_resources; //get build index gpu resource - server::ServerConfig& root_config = server::ServerConfig::GetInstance(); - server::ConfigNode& db_config = root_config.GetConfig(server::CONFIG_DB); + server::ServerConfig& config = server::ServerConfig::GetInstance(); - int32_t build_index_gpu = - db_config.GetInt32Value(server::CONFIG_DB_BUILD_INDEX_GPU, std::stoi(server::CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); + int32_t build_index_gpu = config.GetDBConfigBuildIndexGPU(); gpu_resources.insert(std::make_pair(build_index_gpu, GpuResourceSetting())); //get search gpu resource - server::ConfigNode& res_config = root_config.GetConfig(server::CONFIG_RESOURCE); - auto pool = res_config.GetSequence(server::CONFIG_RESOURCE_POOL); + auto pool = config.GetResourceConfigPool(); std::set gpu_ids; for (auto &resource : pool) { if (resource.length() < 4 || resource.substr(0, 3) != "gpu") { From ebda6cb4ef63ea74ce9a20f6bc80550301ec0a3b Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Sat, 21 Sep 2019 16:39:48 +0800 Subject: [PATCH 074/217] MS-574 rename ServerConfig to Config Former-commit-id: 2b0ef94293017c64a2f17616a3089e7f0cdd6b2b --- cpp/src/cache/CpuCacheMgr.cpp | 4 +- cpp/src/cache/GpuCacheMgr.cpp | 4 +- cpp/src/db/engine/ExecutionEngineImpl.cpp | 3 +- cpp/src/metrics/MetricBase.h | 6 -- cpp/src/metrics/Metrics.cpp | 4 +- cpp/src/metrics/PrometheusMetrics.cpp | 5 +- cpp/src/metrics/PrometheusMetrics.h | 19 +---- cpp/src/scheduler/SchedInst.cpp | 6 +- .../server/{ServerConfig.cpp => Config.cpp} | 82 +++++++++---------- cpp/src/server/{ServerConfig.h => Config.h} | 4 +- cpp/src/server/DBWrapper.cpp | 4 +- cpp/src/server/Server.cpp | 23 +++--- cpp/src/server/grpc_impl/GrpcRequestTask.cpp | 6 +- cpp/src/server/grpc_impl/GrpcServer.cpp | 4 +- cpp/src/utils/LogUtil.cpp | 4 +- cpp/src/wrapper/KnowhereResource.cpp | 4 +- cpp/unittest/db/engine_test.cpp | 1 - cpp/unittest/db/search_test.cpp | 7 +- cpp/unittest/main.cpp | 7 +- cpp/unittest/scheduler/schedinst_test.cpp | 4 +- cpp/unittest/scheduler/scheduler_test.cpp | 7 +- cpp/unittest/server/cache_test.cpp | 4 +- cpp/unittest/server/config_test.cpp | 1 - cpp/unittest/server/rpc_test.cpp | 1 - 24 files changed, 91 insertions(+), 123 deletions(-) rename cpp/src/server/{ServerConfig.cpp => Config.cpp} (94%) rename cpp/src/server/{ServerConfig.h => Config.h} (98%) diff --git a/cpp/src/cache/CpuCacheMgr.cpp b/cpp/src/cache/CpuCacheMgr.cpp index 7141394fa1..9a69bf599f 100644 --- a/cpp/src/cache/CpuCacheMgr.cpp +++ b/cpp/src/cache/CpuCacheMgr.cpp @@ -17,7 +17,7 @@ #include "CpuCacheMgr.h" -#include "server/ServerConfig.h" +#include "server/Config.h" #include "utils/Log.h" namespace zilliz { @@ -29,7 +29,7 @@ namespace { } CpuCacheMgr::CpuCacheMgr() { - server::ServerConfig& config = server::ServerConfig::GetInstance(); + server::Config& config = server::Config::GetInstance(); int64_t cap = config.GetCacheConfigCpuMemCapacity() * unit; cache_ = std::make_shared>(cap, 1UL<<32); diff --git a/cpp/src/cache/GpuCacheMgr.cpp b/cpp/src/cache/GpuCacheMgr.cpp index ba5665f5f2..bc6c3e13db 100644 --- a/cpp/src/cache/GpuCacheMgr.cpp +++ b/cpp/src/cache/GpuCacheMgr.cpp @@ -19,7 +19,7 @@ #include #include "utils/Log.h" #include "GpuCacheMgr.h" -#include "server/ServerConfig.h" +#include "server/Config.h" namespace zilliz { namespace milvus { @@ -33,7 +33,7 @@ namespace { } GpuCacheMgr::GpuCacheMgr() { - server::ServerConfig& config = server::ServerConfig::GetInstance(); + server::Config& config = server::Config::GetInstance(); int32_t cap = config.GetCacheConfigGpuMemCapacity() * G_BYTE; cache_ = std::make_shared>(cap, 1UL<<32); diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 960f3baa89..742d21a0be 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -26,6 +26,7 @@ #include "src/wrapper/vec_index.h" #include "src/wrapper/vec_impl.h" #include "knowhere/common/Exception.h" +#include "server/Config.h" #include @@ -334,7 +335,7 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { // TODO(linxj): remove. Status ExecutionEngineImpl::Init() { using namespace zilliz::milvus::server; - ServerConfig &config = ServerConfig::GetInstance(); + server::Config &config = server::Config::GetInstance(); gpu_num_ = config.GetDBConfigBuildIndexGPU(); return Status::OK(); diff --git a/cpp/src/metrics/MetricBase.h b/cpp/src/metrics/MetricBase.h index aa181fb3f3..8a3f11d7df 100644 --- a/cpp/src/metrics/MetricBase.h +++ b/cpp/src/metrics/MetricBase.h @@ -19,7 +19,6 @@ #pragma once #include "utils/Error.h" -#include "server/ServerConfig.h" #include "SystemInfo.h" namespace zilliz { @@ -83,11 +82,6 @@ class MetricsBase{ virtual void CPUTemperature() {}; }; - - - - - } } } \ No newline at end of file diff --git a/cpp/src/metrics/Metrics.cpp b/cpp/src/metrics/Metrics.cpp index 59eaea6355..6c6df56b17 100644 --- a/cpp/src/metrics/Metrics.cpp +++ b/cpp/src/metrics/Metrics.cpp @@ -16,7 +16,7 @@ // under the License. #include "Metrics.h" -#include "server/ServerConfig.h" +#include "server/Config.h" #include "PrometheusMetrics.h" @@ -32,7 +32,7 @@ Metrics::GetInstance() { MetricsBase & Metrics::CreateMetricsCollector() { - ServerConfig &config = ServerConfig::GetInstance(); + Config &config = Config::GetInstance(); std::string collector_type_str = config.GetMetricConfigCollector(); if (collector_type_str == "prometheus") { diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index b07e91ea32..14d2de6732 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -16,8 +16,9 @@ // under the License. -#include +#include "cache/GpuCacheMgr.h" #include "PrometheusMetrics.h" +#include "server/Config.h" #include "utils/Log.h" #include "SystemInfo.h" @@ -29,7 +30,7 @@ namespace server { ErrorCode PrometheusMetrics::Init() { try { - ServerConfig &config = ServerConfig::GetInstance(); + Config &config = Config::GetInstance(); startup_ = config.GetMetricConfigAutoBootup(); if (!startup_) return SERVER_SUCCESS; diff --git a/cpp/src/metrics/PrometheusMetrics.h b/cpp/src/metrics/PrometheusMetrics.h index 5a3379305a..4ee6699b77 100644 --- a/cpp/src/metrics/PrometheusMetrics.h +++ b/cpp/src/metrics/PrometheusMetrics.h @@ -17,15 +17,13 @@ #pragma once -#include "utils/Error.h" #include #include - - #include #include #include -#include "server/ServerConfig.h" + +#include "utils/Error.h" #include "MetricBase.h" @@ -38,10 +36,6 @@ namespace zilliz { namespace milvus { namespace server { - - - - class PrometheusMetrics: public MetricsBase { public: @@ -107,11 +101,6 @@ class PrometheusMetrics: public MetricsBase { void GPUTemperature() override; void CPUTemperature() override; - - - - - std::shared_ptr &exposer_ptr() {return exposer_ptr_; } // prometheus::Exposer& exposer() { return exposer_;} std::shared_ptr ®istry_ptr() {return registry_; } @@ -125,8 +114,6 @@ class PrometheusMetrics: public MetricsBase { // .Register(*registry_); // prometheus::Counter &connection_total_ = connect_request_.Add({}); - - ////all from DBImpl.cpp using BucketBoundaries = std::vector; //record add_group request @@ -472,10 +459,8 @@ class PrometheusMetrics: public MetricsBase { .Name("CPU_temperature") .Help("CPU temperature") .Register(*registry_); - }; - } } } diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index 54b870100a..5c4ec522da 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -17,7 +17,7 @@ #include "SchedInst.h" -#include "server/ServerConfig.h" +#include "server/Config.h" #include "ResourceFactory.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" #include "Utils.h" @@ -38,7 +38,7 @@ std::mutex JobMgrInst::mutex_; void load_simple_config() { - server::ServerConfig &config = server::ServerConfig::GetInstance(); + server::Config &config = server::Config::GetInstance(); auto mode = config.GetResourceConfigMode(); auto pool = config.GetResourceConfigPool(); bool cpu = false; @@ -81,7 +81,7 @@ load_simple_config() { void load_advance_config() { // try { -// server::ConfigNode &config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_RESOURCE); +// server::ConfigNode &config = server::Config::GetInstance().GetConfig(server::CONFIG_RESOURCE); // // if (config.GetChildren().empty()) throw "resource_config null exception"; // diff --git a/cpp/src/server/ServerConfig.cpp b/cpp/src/server/Config.cpp similarity index 94% rename from cpp/src/server/ServerConfig.cpp rename to cpp/src/server/Config.cpp index 35012192e8..9f4f456452 100644 --- a/cpp/src/server/ServerConfig.cpp +++ b/cpp/src/server/Config.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "ServerConfig.h" +#include "Config.h" #include #include @@ -35,14 +35,14 @@ namespace server { constexpr uint64_t MB = 1024 * 1024; constexpr uint64_t GB = MB * 1024; -ServerConfig & -ServerConfig::GetInstance() { - static ServerConfig config; - return config; +Config & +Config::GetInstance() { + static Config config_inst; + return config_inst; } Status -ServerConfig::LoadConfigFile(const std::string &filename) { +Config::LoadConfigFile(const std::string &filename) { if (filename.empty()) { std::cerr << "ERROR: need specify config file" << std::endl; exit(1); @@ -71,7 +71,7 @@ ServerConfig::LoadConfigFile(const std::string &filename) { } Status -ServerConfig::ValidateConfig() { +Config::ValidateConfig() { if (!CheckServerConfig().ok()) { return Status(SERVER_INVALID_ARGUMENT, "Server config validation check fail"); } @@ -94,7 +94,7 @@ ServerConfig::ValidateConfig() { } Status -ServerConfig::CheckServerConfig() { +Config::CheckServerConfig() { /* server_config: address: 0.0.0.0 # milvus server ip address (IPv4) @@ -154,7 +154,7 @@ ServerConfig::CheckServerConfig() { } Status -ServerConfig::CheckDBConfig() { +Config::CheckDBConfig() { /* db_config: db_path: @MILVUS_DB_PATH@ # milvus data storage path @@ -230,7 +230,7 @@ ServerConfig::CheckDBConfig() { } Status -ServerConfig::CheckMetricConfig() { +Config::CheckMetricConfig() { /* metric_config: is_startup: off # if monitoring start: on, off @@ -260,7 +260,7 @@ ServerConfig::CheckMetricConfig() { } Status -ServerConfig::CheckCacheConfig() { +Config::CheckCacheConfig() { /* cache_config: cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory @@ -361,7 +361,7 @@ ServerConfig::CheckCacheConfig() { } Status -ServerConfig::CheckEngineConfig() { +Config::CheckEngineConfig() { /* engine_config: use_blas_threshold: 20 @@ -396,7 +396,7 @@ ServerConfig::CheckEngineConfig() { } Status -ServerConfig::CheckResourceConfig() { +Config::CheckResourceConfig() { /* resource_config: mode: simple @@ -406,7 +406,7 @@ ServerConfig::CheckResourceConfig() { - gpu100 */ bool okay = true; - server::ConfigNode &config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_RESOURCE); + server::ConfigNode &config = GetConfig(server::CONFIG_RESOURCE); auto mode = config.GetValue(CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); if (mode != "simple") { std::cerr << "ERROR: invalid resource config: mode is " << mode << std::endl; @@ -422,7 +422,7 @@ ServerConfig::CheckResourceConfig() { } //Status -//ServerConfig::CheckResourceConfig() { +//Config::CheckResourceConfig() { /* resource_config: # resource list, length: 0~N @@ -604,7 +604,7 @@ ServerConfig::CheckResourceConfig() { //} void -ServerConfig::PrintAll() const { +Config::PrintAll() const { if (const ConfigMgr *mgr = ConfigMgr::GetInstance()) { std::string str = mgr->DumpString(); // SERVER_LOG_INFO << "\n" << str; @@ -613,14 +613,14 @@ ServerConfig::PrintAll() const { } ConfigNode -ServerConfig::GetConfig(const std::string &name) const { +Config::GetConfig(const std::string &name) const { const ConfigMgr *mgr = ConfigMgr::GetInstance(); const ConfigNode &root_node = mgr->GetRootNode(); return root_node.GetChild(name); } ConfigNode & -ServerConfig::GetConfig(const std::string &name) { +Config::GetConfig(const std::string &name) { ConfigMgr *mgr = ConfigMgr::GetInstance(); ConfigNode &root_node = mgr->GetRootNode(); return root_node.GetChild(name); @@ -628,28 +628,28 @@ ServerConfig::GetConfig(const std::string &name) { /* server config */ std::string -ServerConfig::GetServerConfigAddress() { +Config::GetServerConfigAddress() { ConfigNode server_config = GetConfig(CONFIG_SERVER); return server_config.GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); } std::string -ServerConfig::GetServerConfigPort() { +Config::GetServerConfigPort() { ConfigNode server_config = GetConfig(CONFIG_SERVER); return server_config.GetValue(CONFIG_SERVER_PORT, CONFIG_SERVER_PORT_DEFAULT); } std::string -ServerConfig::GetServerConfigMode() { +Config::GetServerConfigMode() { ConfigNode server_config = GetConfig(CONFIG_SERVER); return server_config.GetValue(CONFIG_SERVER_MODE, CONFIG_SERVER_MODE_DEFAULT); } std::string -ServerConfig::GetServerConfigTimeZone() { +Config::GetServerConfigTimeZone() { ConfigNode server_config = GetConfig(CONFIG_SERVER); return server_config.GetValue(CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); @@ -657,49 +657,49 @@ ServerConfig::GetServerConfigTimeZone() { /* db config */ std::string -ServerConfig::GetDBConfigPath() { +Config::GetDBConfigPath() { ConfigNode db_config = GetConfig(CONFIG_DB); return db_config.GetValue(CONFIG_DB_PATH, CONFIG_DB_PATH_DEFAULT); } std::string -ServerConfig::GetDBConfigSlavePath() { +Config::GetDBConfigSlavePath() { ConfigNode db_config = GetConfig(CONFIG_DB); return db_config.GetValue(CONFIG_DB_SLAVE_PATH, CONFIG_DB_SLAVE_PATH_DEFAULT); } std::string -ServerConfig::GetDBConfigBackendUrl() { +Config::GetDBConfigBackendUrl() { ConfigNode db_config = GetConfig(CONFIG_DB); return db_config.GetValue(CONFIG_DB_BACKEND_URL, CONFIG_DB_BACKEND_URL_DEFAULT); } int32_t -ServerConfig::GetDBConfigArchiveDiskThreshold() { +Config::GetDBConfigArchiveDiskThreshold() { ConfigNode db_config = GetConfig(CONFIG_DB); return db_config.GetInt32Value(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, std::stoi(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT)); } int32_t -ServerConfig::GetDBConfigArchiveDaysThreshold() { +Config::GetDBConfigArchiveDaysThreshold() { ConfigNode db_config = GetConfig(CONFIG_DB); return db_config.GetInt32Value(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, std::stoi(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT)); } int32_t -ServerConfig::GetDBConfigBufferSize() { +Config::GetDBConfigBufferSize() { ConfigNode db_config = GetConfig(CONFIG_DB); return db_config.GetInt32Value(CONFIG_DB_BUFFER_SIZE, std::stoi(CONFIG_DB_BUFFER_SIZE_DEFAULT)); } int32_t -ServerConfig::GetDBConfigBuildIndexGPU() { +Config::GetDBConfigBuildIndexGPU() { ConfigNode db_config = GetConfig(CONFIG_DB); return db_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); @@ -707,21 +707,21 @@ ServerConfig::GetDBConfigBuildIndexGPU() { /* metric config */ bool -ServerConfig::GetMetricConfigAutoBootup() { +Config::GetMetricConfigAutoBootup() { ConfigNode metric_config = GetConfig(CONFIG_METRIC); return metric_config.GetBoolValue(CONFIG_METRIC_AUTO_BOOTUP, std::stoi(CONFIG_METRIC_AUTO_BOOTUP_DEFAULT)); } std::string -ServerConfig::GetMetricConfigCollector() { +Config::GetMetricConfigCollector() { ConfigNode metric_config = GetConfig(CONFIG_METRIC); return metric_config.GetValue(CONFIG_METRIC_COLLECTOR, CONFIG_METRIC_COLLECTOR_DEFAULT); } std::string -ServerConfig::GetMetricConfigPrometheusPort() { +Config::GetMetricConfigPrometheusPort() { ConfigNode metric_config = GetConfig(CONFIG_METRIC); return metric_config.GetValue(CONFIG_METRIC_PROMETHEUS_PORT, CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); @@ -729,35 +729,35 @@ ServerConfig::GetMetricConfigPrometheusPort() { /* cache config */ int32_t -ServerConfig::GetCacheConfigCpuMemCapacity() { +Config::GetCacheConfigCpuMemCapacity() { ConfigNode cache_config = GetConfig(CONFIG_CACHE); return cache_config.GetInt32Value(CONFIG_CACHE_CPU_MEM_CAPACITY, std::stoi(CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT)); } float -ServerConfig::GetCacheConfigCpuMemThreshold() { +Config::GetCacheConfigCpuMemThreshold() { ConfigNode cache_config = GetConfig(CONFIG_CACHE); return cache_config.GetFloatValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, std::stof(CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT)); } int32_t -ServerConfig::GetCacheConfigGpuMemCapacity() { +Config::GetCacheConfigGpuMemCapacity() { ConfigNode cache_config = GetConfig(CONFIG_CACHE); return cache_config.GetInt32Value(CONFIG_CACHE_GPU_MEM_CAPACITY, std::stoi(CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT)); } float -ServerConfig::GetCacheConfigGpuMemThreshold() { +Config::GetCacheConfigGpuMemThreshold() { ConfigNode cache_config = GetConfig(CONFIG_CACHE); return cache_config.GetFloatValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, std::stof(CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT)); } bool -ServerConfig::GetCacheConfigCacheInsertData() { +Config::GetCacheConfigCacheInsertData() { ConfigNode cache_config = GetConfig(CONFIG_CACHE); return cache_config.GetBoolValue(CONFIG_CACHE_CACHE_INSERT_DATA, std::stoi(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT)); @@ -765,14 +765,14 @@ ServerConfig::GetCacheConfigCacheInsertData() { /* engine config */ int32_t -ServerConfig::GetEngineConfigBlasThreshold() { +Config::GetEngineConfigBlasThreshold() { ConfigNode engine_config = GetConfig(CONFIG_ENGINE); return engine_config.GetInt32Value(CONFIG_ENGINE_BLAS_THRESHOLD, std::stoi(CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT)); } int32_t -ServerConfig::GetEngineConfigOmpThreadNum() { +Config::GetEngineConfigOmpThreadNum() { ConfigNode engine_config = GetConfig(CONFIG_ENGINE); return engine_config.GetInt32Value(CONFIG_ENGINE_OMP_THREAD_NUM, std::stoi(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT)); @@ -780,14 +780,14 @@ ServerConfig::GetEngineConfigOmpThreadNum() { /* resource config */ std::string -ServerConfig::GetResourceConfigMode() { +Config::GetResourceConfigMode() { ConfigNode resource_config = GetConfig(CONFIG_RESOURCE); return resource_config.GetValue(CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); } std::vector -ServerConfig::GetResourceConfigPool() { +Config::GetResourceConfigPool() { ConfigNode resource_config = GetConfig(CONFIG_RESOURCE); return resource_config.GetSequence(CONFIG_RESOURCE_POOL); } diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/Config.h similarity index 98% rename from cpp/src/server/ServerConfig.h rename to cpp/src/server/Config.h index 1453adde49..2ad4c382cc 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/Config.h @@ -91,9 +91,9 @@ static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; static const char* CONFIG_RESOURCE_POOL = "pool"; -class ServerConfig { +class Config { public: - static ServerConfig &GetInstance(); + static Config &GetInstance(); Status LoadConfigFile(const std::string& filename); Status ValidateConfig(); diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 3d958b15a4..bc0b080625 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -17,7 +17,7 @@ #include "DBWrapper.h" -#include "ServerConfig.h" +#include "Config.h" #include "db/DBFactory.h" #include "utils/CommonUtil.h" #include "utils/Log.h" @@ -35,7 +35,7 @@ DBWrapper::DBWrapper() { } Status DBWrapper::StartService() { - ServerConfig& config = ServerConfig::GetInstance(); + Config& config = Config::GetInstance(); //db config engine::DBOptions opt; diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index d8924e9d7d..7ca0b2a7c9 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -16,14 +16,6 @@ // under the License. #include -#include "Server.h" -#include "server/grpc_impl/GrpcServer.h" -#include "utils/Log.h" -#include "utils/LogUtil.h" -#include "utils/SignalUtil.h" -#include "utils/TimeRecorder.h" -#include "metrics/Metrics.h" - #include #include #include @@ -31,10 +23,17 @@ //#include #include #include -#include -#include "src/wrapper/KnowhereResource.h" +#include "Server.h" +#include "server/grpc_impl/GrpcServer.h" +#include "server/Config.h" +#include "utils/Log.h" +#include "utils/LogUtil.h" +#include "utils/SignalUtil.h" +#include "utils/TimeRecorder.h" #include "metrics/Metrics.h" +#include "scheduler/SchedInst.h" +#include "wrapper/KnowhereResource.h" #include "DBWrapper.h" @@ -161,7 +160,7 @@ Server::Start() { } /* log path is defined in Config file, so InitLog must be called after LoadConfig */ - ServerConfig &config = ServerConfig::GetInstance(); + Config &config = Config::GetInstance(); std::string time_zone = config.GetServerConfigTimeZone(); if (time_zone.length() == 3) { time_zone = "CUT"; @@ -230,7 +229,7 @@ Server::Stop() { ErrorCode Server::LoadConfig() { - ServerConfig& server_config = ServerConfig::GetInstance(); + Config& server_config = Config::GetInstance(); server_config.LoadConfigFile(config_filename_); auto status = server_config.ValidateConfig(); if (!status.ok()) { diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index ad0abd713d..e699dd04b7 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -15,8 +15,9 @@ // specific language governing permissions and limitations // under the License. +#include + #include "GrpcRequestTask.h" -#include "../ServerConfig.h" #include "utils/CommonUtil.h" #include "utils/Log.h" #include "utils/TimeRecorder.h" @@ -28,9 +29,8 @@ #include "scheduler/SchedInst.h" //#include -#include "src/server/Server.h" +#include "server/Server.h" -#include namespace zilliz { namespace milvus { diff --git a/cpp/src/server/grpc_impl/GrpcServer.cpp b/cpp/src/server/grpc_impl/GrpcServer.cpp index e330f5483c..c10d7b5379 100644 --- a/cpp/src/server/grpc_impl/GrpcServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcServer.cpp @@ -17,7 +17,7 @@ #include "milvus.grpc.pb.h" #include "GrpcServer.h" -#include "server/ServerConfig.h" +#include "server/Config.h" #include "server/DBWrapper.h" #include "utils/Log.h" #include "GrpcRequestHandler.h" @@ -73,7 +73,7 @@ GrpcServer::Stop() { Status GrpcServer::StartService() { - ServerConfig &config = ServerConfig::GetInstance(); + Config &config = Config::GetInstance(); std::string address = config.GetServerConfigAddress(); std::string port = config.GetServerConfigPort(); diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp index cfdbbdc768..3a3fe0c7e2 100644 --- a/cpp/src/utils/LogUtil.cpp +++ b/cpp/src/utils/LogUtil.cpp @@ -15,13 +15,11 @@ // specific language governing permissions and limitations // under the License. -#include "LogUtil.h" -#include "server/ServerConfig.h" - #include #include #include +#include "LogUtil.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/wrapper/KnowhereResource.cpp b/cpp/src/wrapper/KnowhereResource.cpp index dfd75df253..1e4248f6e1 100644 --- a/cpp/src/wrapper/KnowhereResource.cpp +++ b/cpp/src/wrapper/KnowhereResource.cpp @@ -18,7 +18,7 @@ #include "KnowhereResource.h" #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" -#include "server/ServerConfig.h" +#include "server/Config.h" #include @@ -38,7 +38,7 @@ ErrorCode KnowhereResource::Initialize() { GpuResourcesArray gpu_resources; //get build index gpu resource - server::ServerConfig& config = server::ServerConfig::GetInstance(); + server::Config& config = server::Config::GetInstance(); int32_t build_index_gpu = config.GetDBConfigBuildIndexGPU(); gpu_resources.insert(std::make_pair(build_index_gpu, GpuResourceSetting())); diff --git a/cpp/unittest/db/engine_test.cpp b/cpp/unittest/db/engine_test.cpp index e06e9038c8..f55d8d9e27 100644 --- a/cpp/unittest/db/engine_test.cpp +++ b/cpp/unittest/db/engine_test.cpp @@ -21,7 +21,6 @@ #include "db/engine/EngineFactory.h" #include "db/engine/ExecutionEngineImpl.h" -#include "server/ServerConfig.h" #include "utils.h" using namespace zilliz::milvus; diff --git a/cpp/unittest/db/search_test.cpp b/cpp/unittest/db/search_test.cpp index 295650f2d2..d492256534 100644 --- a/cpp/unittest/db/search_test.cpp +++ b/cpp/unittest/db/search_test.cpp @@ -15,13 +15,12 @@ // specific language governing permissions and limitations // under the License. -#include "server/ServerConfig.h" -#include "utils/TimeRecorder.h" - #include #include #include -#include + +#include "scheduler/task/SearchTask.h" +#include "utils/TimeRecorder.h" using namespace zilliz::milvus; diff --git a/cpp/unittest/main.cpp b/cpp/unittest/main.cpp index 33a860945d..7a28bbf45f 100644 --- a/cpp/unittest/main.cpp +++ b/cpp/unittest/main.cpp @@ -15,13 +15,12 @@ // specific language governing permissions and limitations // under the License. -#include "utils/easylogging++.h" -#include "server/ServerConfig.h" -#include "utils/CommonUtil.h" - #include #include +#include "utils/easylogging++.h" +#include "utils/CommonUtil.h" + INITIALIZE_EASYLOGGINGPP using namespace zilliz::milvus; diff --git a/cpp/unittest/scheduler/schedinst_test.cpp b/cpp/unittest/scheduler/schedinst_test.cpp index 27b6a17bbb..cb56264525 100644 --- a/cpp/unittest/scheduler/schedinst_test.cpp +++ b/cpp/unittest/scheduler/schedinst_test.cpp @@ -15,12 +15,10 @@ // specific language governing permissions and limitations // under the License. - -#include "scheduler/SchedInst.h" -#include "server/ServerConfig.h" #include #include +#include "scheduler/SchedInst.h" namespace zilliz { namespace milvus { diff --git a/cpp/unittest/scheduler/scheduler_test.cpp b/cpp/unittest/scheduler/scheduler_test.cpp index 1cfbda1e14..b8899ebdc2 100644 --- a/cpp/unittest/scheduler/scheduler_test.cpp +++ b/cpp/unittest/scheduler/scheduler_test.cpp @@ -15,17 +15,16 @@ // specific language governing permissions and limitations // under the License. -#include "scheduler/Scheduler.h" #include -#include -#include +#include "scheduler/tasklabel/DefaultLabel.h" #include "cache/DataObj.h" #include "cache/GpuCacheMgr.h" +#include "scheduler/Scheduler.h" #include "scheduler/task/TestTask.h" #include "scheduler/ResourceFactory.h" #include "scheduler/resource/Resource.h" #include "utils/Error.h" -#include "src/wrapper/vec_index.h" +#include "wrapper/vec_index.h" #include "scheduler/tasklabel/SpecResLabel.h" diff --git a/cpp/unittest/server/cache_test.cpp b/cpp/unittest/server/cache_test.cpp index bf9b6edf7d..94eae5dbf0 100644 --- a/cpp/unittest/server/cache_test.cpp +++ b/cpp/unittest/server/cache_test.cpp @@ -18,10 +18,8 @@ #include #include "cache/CpuCacheMgr.h" #include "cache/GpuCacheMgr.h" -#include "server/ServerConfig.h" - #include "utils/Error.h" -#include "src/wrapper/vec_index.h" +#include "wrapper/vec_index.h" using namespace zilliz::milvus; diff --git a/cpp/unittest/server/config_test.cpp b/cpp/unittest/server/config_test.cpp index 13089e914b..65ce562722 100644 --- a/cpp/unittest/server/config_test.cpp +++ b/cpp/unittest/server/config_test.cpp @@ -19,7 +19,6 @@ #include #include "config/ConfigMgr.h" -#include "server/ServerConfig.h" #include "utils/CommonUtil.h" #include "utils/ValidationUtil.h" diff --git a/cpp/unittest/server/rpc_test.cpp b/cpp/unittest/server/rpc_test.cpp index 5ad6005a36..c196c082e5 100644 --- a/cpp/unittest/server/rpc_test.cpp +++ b/cpp/unittest/server/rpc_test.cpp @@ -29,7 +29,6 @@ #include "grpc/gen-status/status.pb.h" #include "server/DBWrapper.h" -#include "server/ServerConfig.h" #include "scheduler/SchedInst.h" #include "scheduler/ResourceFactory.h" #include "utils/CommonUtil.h" From b0fe84cd816fc2548ad776f875c485d61ef66cc6 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Sat, 21 Sep 2019 18:14:29 +0800 Subject: [PATCH 075/217] MS-574 save config in map Former-commit-id: 961dc6749c1b262e489a825cce81586bf27f2c18 --- cpp/src/server/Config.cpp | 255 ++++++++++++++++++++++++++++---------- cpp/src/server/Config.h | 20 ++- 2 files changed, 204 insertions(+), 71 deletions(-) diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 9f4f456452..9f7b9fd12e 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include "config/ConfigMgr.h" #include "utils/CommonUtil.h" @@ -32,8 +33,8 @@ namespace zilliz { namespace milvus { namespace server { -constexpr uint64_t MB = 1024 * 1024; -constexpr uint64_t GB = MB * 1024; +constexpr uint64_t MB = 1UL << 20; +constexpr uint64_t GB = 1UL << 30; Config & Config::GetInstance() { @@ -626,164 +627,282 @@ Config::GetConfig(const std::string &name) { return root_node.GetChild(name); } +Status +Config::GetConfigValueInMem(const std::string &parent_key, + const std::string &child_key, + std::string &value) { + if (config_map_.find(parent_key) != config_map_.end() && + config_map_[parent_key].find(child_key) != config_map_[parent_key].end()) { + std::lock_guard lock(mutex_); + value = config_map_[parent_key][child_key]; + return Status::OK(); + } else { + return Status(SERVER_UNEXPECTED_ERROR, "key not exist"); + } +} + +void +Config::SetConfigValueInMem(const std::string &parent_key, + const std::string &child_key, + std::string &value) { + std::lock_guard lock(mutex_); + config_map_[parent_key][child_key] = value; +} + +//////////////////////////////////////////////////////////////////////////////// /* server config */ std::string Config::GetServerConfigAddress() { - ConfigNode server_config = GetConfig(CONFIG_SERVER); - return server_config.GetValue(CONFIG_SERVER_ADDRESS, - CONFIG_SERVER_ADDRESS_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value).ok()) { + value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_ADDRESS, + CONFIG_SERVER_ADDRESS_DEFAULT); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); + } + return value; } std::string Config::GetServerConfigPort() { - ConfigNode server_config = GetConfig(CONFIG_SERVER); - return server_config.GetValue(CONFIG_SERVER_PORT, - CONFIG_SERVER_PORT_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value).ok()) { + value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_PORT, + CONFIG_SERVER_PORT_DEFAULT); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); + } + return value; } std::string Config::GetServerConfigMode() { - ConfigNode server_config = GetConfig(CONFIG_SERVER); - return server_config.GetValue(CONFIG_SERVER_MODE, - CONFIG_SERVER_MODE_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value).ok()) { + value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_MODE, + CONFIG_SERVER_MODE_DEFAULT); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); + } + return value; } std::string Config::GetServerConfigTimeZone() { - ConfigNode server_config = GetConfig(CONFIG_SERVER); - return server_config.GetValue(CONFIG_SERVER_TIME_ZONE, - CONFIG_SERVER_TIME_ZONE_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value).ok()) { + value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_TIME_ZONE, + CONFIG_SERVER_TIME_ZONE_DEFAULT); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); + } + return value; } +//////////////////////////////////////////////////////////////////////////////// /* db config */ std::string Config::GetDBConfigPath() { - ConfigNode db_config = GetConfig(CONFIG_DB); - return db_config.GetValue(CONFIG_DB_PATH, - CONFIG_DB_PATH_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value).ok()) { + value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_PATH, + CONFIG_DB_PATH_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); + } + return value; } std::string Config::GetDBConfigSlavePath() { - ConfigNode db_config = GetConfig(CONFIG_DB); - return db_config.GetValue(CONFIG_DB_SLAVE_PATH, - CONFIG_DB_SLAVE_PATH_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value).ok()) { + value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_SLAVE_PATH, + CONFIG_DB_SLAVE_PATH_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); + } + return value; } std::string Config::GetDBConfigBackendUrl() { - ConfigNode db_config = GetConfig(CONFIG_DB); - return db_config.GetValue(CONFIG_DB_BACKEND_URL, - CONFIG_DB_BACKEND_URL_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value).ok()) { + value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_BACKEND_URL, + CONFIG_DB_BACKEND_URL_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); + } + return value; } int32_t Config::GetDBConfigArchiveDiskThreshold() { - ConfigNode db_config = GetConfig(CONFIG_DB); - return db_config.GetInt32Value(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, - std::stoi(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value).ok()) { + value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, + CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); + } + return std::stoi(value); } int32_t Config::GetDBConfigArchiveDaysThreshold() { - ConfigNode db_config = GetConfig(CONFIG_DB); - return db_config.GetInt32Value(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, - std::stoi(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value).ok()) { + value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, + CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); + } + return std::stoi(value); } int32_t Config::GetDBConfigBufferSize() { - ConfigNode db_config = GetConfig(CONFIG_DB); - return db_config.GetInt32Value(CONFIG_DB_BUFFER_SIZE, - std::stoi(CONFIG_DB_BUFFER_SIZE_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value).ok()) { + value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_BUFFER_SIZE, + CONFIG_DB_BUFFER_SIZE_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); + } + return std::stoi(value); } int32_t Config::GetDBConfigBuildIndexGPU() { - ConfigNode db_config = GetConfig(CONFIG_DB); - return db_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, - std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value).ok()) { + value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_BUILD_INDEX_GPU, + CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); + } + return std::stoi(value); } +//////////////////////////////////////////////////////////////////////////////// /* metric config */ bool Config::GetMetricConfigAutoBootup() { - ConfigNode metric_config = GetConfig(CONFIG_METRIC); - return metric_config.GetBoolValue(CONFIG_METRIC_AUTO_BOOTUP, - std::stoi(CONFIG_METRIC_AUTO_BOOTUP_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value).ok()) { + value = GetConfig(CONFIG_METRIC).GetValue(CONFIG_METRIC_AUTO_BOOTUP, + CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value); + } + std::transform(value.begin(), value.end(), value.begin(), ::tolower); + return (value == "true" || value == "on" || value == "yes" || value == "1"); } std::string Config::GetMetricConfigCollector() { - ConfigNode metric_config = GetConfig(CONFIG_METRIC); - return metric_config.GetValue(CONFIG_METRIC_COLLECTOR, - CONFIG_METRIC_COLLECTOR_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value).ok()) { + value = GetConfig(CONFIG_METRIC).GetValue(CONFIG_METRIC_COLLECTOR, + CONFIG_METRIC_COLLECTOR_DEFAULT); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); + } + return value; } std::string Config::GetMetricConfigPrometheusPort() { - ConfigNode metric_config = GetConfig(CONFIG_METRIC); - return metric_config.GetValue(CONFIG_METRIC_PROMETHEUS_PORT, - CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value).ok()) { + value = GetConfig(CONFIG_METRIC).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, + CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); + } + return value; } +//////////////////////////////////////////////////////////////////////////////// /* cache config */ int32_t Config::GetCacheConfigCpuMemCapacity() { - ConfigNode cache_config = GetConfig(CONFIG_CACHE); - return cache_config.GetInt32Value(CONFIG_CACHE_CPU_MEM_CAPACITY, - std::stoi(CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value).ok()) { + value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_CAPACITY, + CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value); + } + return std::stoi(value); } float Config::GetCacheConfigCpuMemThreshold() { - ConfigNode cache_config = GetConfig(CONFIG_CACHE); - return cache_config.GetFloatValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, - std::stof(CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value).ok()) { + value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, + CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value); + } + return std::stof(value); } int32_t Config::GetCacheConfigGpuMemCapacity() { - ConfigNode cache_config = GetConfig(CONFIG_CACHE); - return cache_config.GetInt32Value(CONFIG_CACHE_GPU_MEM_CAPACITY, - std::stoi(CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value).ok()) { + value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_CAPACITY, + CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value); + } + return std::stoi(value); } float Config::GetCacheConfigGpuMemThreshold() { - ConfigNode cache_config = GetConfig(CONFIG_CACHE); - return cache_config.GetFloatValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, - std::stof(CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value).ok()) { + value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, + CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value); + } + return std::stof(value); } bool Config::GetCacheConfigCacheInsertData() { - ConfigNode cache_config = GetConfig(CONFIG_CACHE); - return cache_config.GetBoolValue(CONFIG_CACHE_CACHE_INSERT_DATA, - std::stoi(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value).ok()) { + value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, + CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); + } + std::transform(value.begin(), value.end(), value.begin(), ::tolower); + return (value == "true" || value == "on" || value == "yes" || value == "1"); } +//////////////////////////////////////////////////////////////////////////////// /* engine config */ int32_t Config::GetEngineConfigBlasThreshold() { - ConfigNode engine_config = GetConfig(CONFIG_ENGINE); - return engine_config.GetInt32Value(CONFIG_ENGINE_BLAS_THRESHOLD, - std::stoi(CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value).ok()) { + value = GetConfig(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_BLAS_THRESHOLD, + CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); + SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value); + } + return std::stoi(value); } int32_t Config::GetEngineConfigOmpThreadNum() { - ConfigNode engine_config = GetConfig(CONFIG_ENGINE); - return engine_config.GetInt32Value(CONFIG_ENGINE_OMP_THREAD_NUM, - std::stoi(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value).ok()) { + value = GetConfig(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, + CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); + SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); + } + return std::stoi(value); } +//////////////////////////////////////////////////////////////////////////////// /* resource config */ std::string Config::GetResourceConfigMode() { - ConfigNode resource_config = GetConfig(CONFIG_RESOURCE); - return resource_config.GetValue(CONFIG_RESOURCE_MODE, - CONFIG_RESOURCE_MODE_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value).ok()) { + value = GetConfig(CONFIG_RESOURCE).GetValue(CONFIG_RESOURCE_MODE, + CONFIG_RESOURCE_MODE_DEFAULT); + SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); + } + return value; } std::vector diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index 2ad4c382cc..992ccf0965 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -17,10 +17,12 @@ #pragma once +#include +#include +#include "yaml-cpp/yaml.h" #include "utils/Status.h" #include "config/ConfigNode.h" -#include "yaml-cpp/yaml.h" namespace zilliz { namespace milvus { @@ -65,12 +67,12 @@ static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT = "0.85"; static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD = "gpu_mem_threshold"; static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT = "0.85"; static const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; -static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "0"; +static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; /* metric config */ static const char* CONFIG_METRIC = "metric_config"; static const char* CONFIG_METRIC_AUTO_BOOTUP = "auto_bootup"; -static const char* CONFIG_METRIC_AUTO_BOOTUP_DEFAULT = "0"; +static const char* CONFIG_METRIC_AUTO_BOOTUP_DEFAULT = "false"; static const char* CONFIG_METRIC_COLLECTOR = "collector"; static const char* CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus"; static const char* CONFIG_METRIC_PROMETHEUS = "prometheus_config"; @@ -110,6 +112,14 @@ class Config { Status CheckEngineConfig(); Status CheckResourceConfig(); + Status GetConfigValueInMem(const std::string& parent_key, + const std::string& child_key, + std::string& value); + + void SetConfigValueInMem(const std::string& parent_key, + const std::string& child_key, + std::string& value); + public: std::string GetServerConfigAddress(); std::string GetServerConfigPort(); @@ -140,6 +150,10 @@ class Config { std::string GetResourceConfigMode(); std::vector GetResourceConfigPool(); + + private: + std::unordered_map> config_map_; + std::mutex mutex_; }; } From d470889d4e6c51a420bd7130c789679339abc4a2 Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Sat, 21 Sep 2019 19:16:15 +0800 Subject: [PATCH 076/217] MS-582 grafana displays changes frequently Former-commit-id: 9e2771a4ea84cf8b868628ce64cf0b0ad48a200d --- cpp/CHANGELOG.md | 1 + docker/docker-compose-monitor.yml | 12 +++++++----- docker/prometheus.yml | 14 +++----------- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 09929ed8d0..c8701a3fda 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -62,6 +62,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-510 - unittest out of memory and crashed - MS-507 - Dataset 10m-512, index type sq8,performance in-normal when set CPU_CACHE to 16 or 64 - MS-543 - SearchTask fail without exception +- MS-582 - grafana displays changes frequently ## Improvement - MS-327 - Clean code for milvus diff --git a/docker/docker-compose-monitor.yml b/docker/docker-compose-monitor.yml index 9cb1fb1902..81e7084af9 100644 --- a/docker/docker-compose-monitor.yml +++ b/docker/docker-compose-monitor.yml @@ -40,15 +40,17 @@ services: networks: - monitor - milvus: + milvus_server: runtime: nvidia image: registry.zilliz.com/milvus/engine:branch-0.4.0-release - container_name: milvus - hostname: milvus restart: always + links: + - prometheus + environment: + WEB_APP: host.docker.internal volumes: - - ../cpp/conf/server_config.yaml:/opt/milvus/cpp/conf/server_config.yaml - - ../cpp/conf/log_config.conf:/opt/milvus/cpp/conf/log_config.conf + - ../cpp/conf/server_config.yaml:/opt/milvus/conf/server_config.yaml + - ../cpp/conf/log_config.conf:/opt/milvus/conf/log_config.conf ports: - "8080:8080" - "19530:19530" diff --git a/docker/prometheus.yml b/docker/prometheus.yml index 7a2a07b390..91c64458a5 100644 --- a/docker/prometheus.yml +++ b/docker/prometheus.yml @@ -24,23 +24,15 @@ scrape_configs: # scheme defaults to 'http'. static_configs: - - targets: ['localhost:9090'] + - targets: ['prometheus:9090'] # scrape metrics of server - job_name: 'milvus_server' scrape_interval: 1s static_configs: - - targets: ['localhost:8080'] - - # scrape metrics of server - - job_name: 'milvus_server_1' - scrape_interval: 1s - static_configs: - - targets: ['localhost:8080'] - - + - targets: ['milvus_server:8080'] # under development - job_name: 'pushgateway' static_configs: - - targets: ['localhost:9091'] + - targets: ['pushgateway:9091'] From 3cbc8b6825ce0b88b4d01fc103077ce17f487736 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 25 Sep 2019 10:50:33 +0800 Subject: [PATCH 077/217] MS-586 Remove BUILD_FAISS_WITH_MKL option Former-commit-id: 22f4383828bc1e975c7b21e8ee6a64936039e41b --- cpp/CHANGELOG.md | 1 + cpp/build.sh | 10 ++-------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 5fb91c72d2..23acfecf2b 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -29,6 +29,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-567 - Add NOTICE.md - MS-569 - Complete the NOTICE.md - MS-575 - Add Clang-format & Clang-tidy & Cpplint +- MS-586 - Remove BUILD_FAISS_WITH_MKL option # Milvus 0.4.0 (2019-09-12) diff --git a/cpp/build.sh b/cpp/build.sh index b09e5b6313..013d4d575d 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -7,10 +7,9 @@ MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" DB_PATH="/opt/milvus" PROFILING="OFF" -BUILD_FAISS_WITH_MKL="OFF" USE_JFROG_CACHE="OFF" -while getopts "p:d:t:uhrcgmj" arg +while getopts "p:d:t:uhrcgj" arg do case $arg in t) @@ -38,9 +37,6 @@ do g) PROFILING="ON" ;; - m) - BUILD_FAISS_WITH_MKL="ON" - ;; j) USE_JFROG_CACHE="ON" ;; @@ -55,11 +51,10 @@ parameter: -r: remove previous build directory(default: OFF) -c: code coverage(default: OFF) -g: profiling(default: OFF) --m: build faiss with MKL(default: OFF) -j: use jfrog cache build directory usage: -./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c] [-k] [-m] [-j] +./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c] [-k] [-j] " exit 0 ;; @@ -87,7 +82,6 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then -DBUILD_COVERAGE=${BUILD_COVERAGE} \ -DMILVUS_DB_PATH=${DB_PATH} \ -DMILVUS_ENABLE_PROFILING=${PROFILING} \ - -DBUILD_FAISS_WITH_MKL=${BUILD_FAISS_WITH_MKL} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ ../" From f195bd7d334171f14a5638a6d00381bc4914185b Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 25 Sep 2019 11:42:25 +0800 Subject: [PATCH 078/217] remove unused option Former-commit-id: 2594078698d7e570d40f0e72a8f868e1d5a0bd2d --- cpp/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d226a03e61..e7cf83d704 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -43,7 +43,6 @@ endif() set(MILVUS_VERSION "${GIT_BRANCH_NAME}") string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}") -set(CLANG_FORMAT_VERSION "6.0") find_package(ClangTools) if("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND) # Generate a Clang compile_commands.json "compilation database" file for use From be2abf935c48e6edcf6b41c69c0204d47951f218 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 25 Sep 2019 14:15:26 +0800 Subject: [PATCH 079/217] fix build error Former-commit-id: f094a9ad816accc2b6fc96e2c515e554d9ce8f14 --- cpp/unittest/server/rpc_test.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cpp/unittest/server/rpc_test.cpp b/cpp/unittest/server/rpc_test.cpp index 69b258aa75..cf3ba8598c 100644 --- a/cpp/unittest/server/rpc_test.cpp +++ b/cpp/unittest/server/rpc_test.cpp @@ -50,19 +50,19 @@ class RpcHandlerTest : public testing::Test { void SetUp() override { - auto res_mgr = engine::ResMgrInst::GetInstance(); + auto res_mgr = scheduler::ResMgrInst::GetInstance(); res_mgr->Clear(); - res_mgr->Add(engine::ResourceFactory::Create("disk", "DISK", 0, true, false)); - res_mgr->Add(engine::ResourceFactory::Create("cpu", "CPU", 0, true, true)); - res_mgr->Add(engine::ResourceFactory::Create("gtx1660", "GPU", 0, true, true)); + res_mgr->Add(scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false)); + res_mgr->Add(scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, true)); + res_mgr->Add(scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true)); - auto default_conn = engine::Connection("IO", 500.0); - auto PCIE = engine::Connection("IO", 11000.0); + auto default_conn = scheduler::Connection("IO", 500.0); + auto PCIE = scheduler::Connection("IO", 11000.0); res_mgr->Connect("disk", "cpu", default_conn); res_mgr->Connect("cpu", "gtx1660", PCIE); res_mgr->Start(); - engine::SchedInst::GetInstance()->Start(); - engine::JobMgrInst::GetInstance()->Start(); + scheduler::SchedInst::GetInstance()->Start(); + scheduler::JobMgrInst::GetInstance()->Start(); engine::DBOptions opt; @@ -106,9 +106,9 @@ class RpcHandlerTest : public testing::Test { void TearDown() override { server::DBWrapper::GetInstance().StopService(); - engine::JobMgrInst::GetInstance()->Stop(); - engine::ResMgrInst::GetInstance()->Stop(); - engine::SchedInst::GetInstance()->Stop(); + scheduler::JobMgrInst::GetInstance()->Stop(); + scheduler::ResMgrInst::GetInstance()->Stop(); + scheduler::SchedInst::GetInstance()->Stop(); boost::filesystem::remove_all("/tmp/milvus_test"); } protected: From 66403b6591a6b877ba34f2837465e1649e9a5d2f Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Wed, 25 Sep 2019 14:15:26 +0800 Subject: [PATCH 080/217] Update README.md Former-commit-id: 8b2c69babf81423bc4f09cf67f5b8198d7551633 --- cpp/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/README.md b/cpp/README.md index d32f04f316..31a774fb3e 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -8,7 +8,7 @@ Milvus is an open source vector search engine that supports similarity search of Milvus was developed by ZILLIZ, a tech startup that intends to reinvent data science, with the purpose of providing enterprises with efficient and scalable similarity search and analysis of feature vectors and unstructured data. -Milvus provides stable Python and C++ APIs, as well as RESTful API. +Milvus provides stable Python, C++ and Java API. Keep up-to-date with newest releases and latest updates by reading [Releases](https://www.milvus-io/docs/master/releases). From 8a53d8c9f151928271ffcf161636bd9e6d13120e Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Wed, 25 Sep 2019 14:33:16 +0800 Subject: [PATCH 081/217] Update README.md Former-commit-id: 16c10a07a3d4632b4419174882b0d7cd7b75fa4e --- cpp/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpp/README.md b/cpp/README.md index 31a774fb3e..be6254e866 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -1,3 +1,9 @@ + + + +- Slack Community(https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk) +- Blog(https://www.milvus.io/blog/) + # Welcome to Milvus Firstly, welcome, and thanks for your interest in [Milvus](https://milvus.io)! No matter who you are, what you do, we greatly appreciate your contribution to help us reinvent data science with Milvus. From 1f99ab1c7a4d2863e0e996ccebfcbacab60b6b46 Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Wed, 25 Sep 2019 14:33:36 +0800 Subject: [PATCH 082/217] Update README.md Former-commit-id: 41bfed06d0b2ee0970e99873f22e008780988f30 --- cpp/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index be6254e866..2d0bef0f29 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -1,8 +1,7 @@ - -- Slack Community(https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk) -- Blog(https://www.milvus.io/blog/) +- [Slack Community](https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk) +- [Blog](https://www.milvus.io/blog/) # Welcome to Milvus From 30925f759bc3c924cd039d5719a51a12e6c8bbf8 Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Wed, 25 Sep 2019 14:37:26 +0800 Subject: [PATCH 083/217] Update README.md Former-commit-id: 0ca186c1fd21c7dadcf6f892a210e9d3b8925696 --- cpp/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index 2d0bef0f29..1943a4be7f 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -1,5 +1,4 @@ - - [Slack Community](https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk) - [Blog](https://www.milvus.io/blog/) @@ -38,7 +37,7 @@ Keep up-to-date with newest releases and latest updates by reading [Releases](ht #### Use Docker -Use Docker to install Milvus is a breeze. See the [Milvus install guide](https://www.milvus-io/docs/master/userguide/install_milvus.md) for details. +Use Docker to install Milvus is a breeze. See the [Milvus install guide](https://milvus.io/docs/en/userguide/install_milvus/) for details. #### Use source code From f1c40f44407c3a49bde28151c1fcf50cad612774 Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Wed, 25 Sep 2019 14:56:38 +0800 Subject: [PATCH 084/217] Update README.md Former-commit-id: f418eaa3b6a769e6f07552a21e2f947d845f7851 --- cpp/README.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index 1943a4be7f..fcc5a27027 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -1,4 +1,3 @@ - - [Slack Community](https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk) - [Blog](https://www.milvus.io/blog/) @@ -12,9 +11,9 @@ Milvus is an open source vector search engine that supports similarity search of Milvus was developed by ZILLIZ, a tech startup that intends to reinvent data science, with the purpose of providing enterprises with efficient and scalable similarity search and analysis of feature vectors and unstructured data. -Milvus provides stable Python, C++ and Java API. +Milvus provides stable Python, C++ and Java APIs. -Keep up-to-date with newest releases and latest updates by reading [Releases](https://www.milvus-io/docs/master/releases). +Keep up-to-date with newest releases and latest updates by reading Milvus [release notes](https://milvus.io/docs/en/Releases/v0.4.0/). - GPU-accelerated search engine @@ -184,17 +183,17 @@ $ python3 example.py ## Contribution guidelines -Contributions are welcomed and greatly appreciated. If you want to contribute to Milvus, please read the [contribution guidelines](CONTRIBUTING.md). This project adheres to the [code of conduct](CODE OF CONDUCT.md) of Milvus. By participating, you are expected to uphold this code. +Contributions are welcomed and greatly appreciated. If you want to contribute to Milvus, please read our [contribution guidelines](CONTRIBUTING.md). This project adheres to the [code of conduct](CODE OF CONDUCT.md) of Milvus. By participating, you are expected to uphold this code. -We use [GitHub issues](https://github.com/milvus-io/milvus/issues) to track issues and bugs. For general questions and discussions, please go to [Milvus Forum]. +We use [GitHub issues](https://github.com/milvus-io/milvus/issues) to track issues and bugs. For general questions and public discussions, please join our community. ## Join the Milvus community -For public discussion of Milvus, please join our [discussion group](milvusio.slack.com). +To connect with other users and contributors, welcome to join our [slack channel](https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk). ## Milvus Roadmap -Please read our [roadmap](milvus-io/milvus/docs/master/roadmap.md) to learn about upcoming features. +Please read our [roadmap](https://milvus.io/docs/en/roadmap/) to learn about upcoming features. ## Resources @@ -206,7 +205,7 @@ Please read our [roadmap](milvus-io/milvus/docs/master/roadmap.md) to learn abou [Milvus CSDN](https://mp.csdn.net/mdeditor/100041006#) -[Milvus roadmap](https://www.milvus-io/docs/master/roadmap.md) +[Milvus roadmap](https://milvus.io/docs/en/roadmap/) ## License From 5c4889dd257e1ef95d1d455712d474ef434468fa Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 25 Sep 2019 15:04:42 +0800 Subject: [PATCH 085/217] format CMakeLists.txt Former-commit-id: 447d58d1b8d0a065f2c5c5ec7b7aa3b57babe8b3 --- ci/jenkinsfile/cluster_deploy2dev.groovy | 2 +- ci/jenkinsfile/deploy2dev.groovy | 2 +- ci/jenkinsfile/deploy2staging.groovy | 2 +- ci/jenkinsfile/dev_test.groovy | 2 +- ci/jenkinsfile/staging_test.groovy | 2 +- cpp/CMakeLists.txt | 2 +- cpp/cmake/ThirdPartyPackages.cmake | 31 +++++++++++++++--------- 7 files changed, 26 insertions(+), 17 deletions(-) diff --git a/ci/jenkinsfile/cluster_deploy2dev.groovy b/ci/jenkinsfile/cluster_deploy2dev.groovy index 7c7a94999d..7f2a584256 100644 --- a/ci/jenkinsfile/cluster_deploy2dev.groovy +++ b/ci/jenkinsfile/cluster_deploy2dev.groovy @@ -5,7 +5,7 @@ try { dir ("milvus-helm") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) dir ("milvus/milvus-cluster") { - sh "helm install --wait --timeout 300 --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.4.0 . " + sh "helm install --wait --timeout 300 --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.5.0 . " } } /* diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy index 8278b7745e..b00c8fa335 100644 --- a/ci/jenkinsfile/deploy2dev.groovy +++ b/ci/jenkinsfile/deploy2dev.groovy @@ -5,7 +5,7 @@ try { dir ("milvus-helm") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.4.0 ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.5.0 ." } } } catch (exc) { diff --git a/ci/jenkinsfile/deploy2staging.groovy b/ci/jenkinsfile/deploy2staging.groovy index ee01097de8..42ccfda71a 100644 --- a/ci/jenkinsfile/deploy2staging.groovy +++ b/ci/jenkinsfile/deploy2staging.groovy @@ -5,7 +5,7 @@ try { dir ("milvus-helm") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.4.0 ." + sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.5.0 ." } } } catch (exc) { diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy index 623e4092cd..cf8d9b66b0 100644 --- a/ci/jenkinsfile/dev_test.groovy +++ b/ci/jenkinsfile/dev_test.groovy @@ -16,7 +16,7 @@ timeout(time: 30, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.5.0 ." } } dir ("${PROJECT_NAME}_test") { diff --git a/ci/jenkinsfile/staging_test.groovy b/ci/jenkinsfile/staging_test.groovy index 7fceee5dfd..dcf1787103 100644 --- a/ci/jenkinsfile/staging_test.groovy +++ b/ci/jenkinsfile/staging_test.groovy @@ -17,7 +17,7 @@ timeout(time: 40, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ." + sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.5.0 ." } } dir ("${PROJECT_NAME}_test") { diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index e7cf83d704..1b5c4fc161 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -71,7 +71,7 @@ if(MILVUS_VERSION_MAJOR STREQUAL "" OR MILVUS_VERSION_MINOR STREQUAL "" OR MILVUS_VERSION_PATCH STREQUAL "") message(WARNING "Failed to determine Milvus version from git branch name") - set(MILVUS_VERSION "0.4.0") + set(MILVUS_VERSION "0.5.0") endif() message(STATUS "Build version = ${MILVUS_VERSION}") diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index f280c48293..37e9f4c3cd 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -137,9 +137,26 @@ if(NOT DEFINED USE_JFROG_CACHE) set(USE_JFROG_CACHE "OFF") endif() if(USE_JFROG_CACHE STREQUAL "ON") - set(JFROG_ARTFACTORY_CACHE_URL "http://192.168.1.201:80/artifactory/generic-local/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") - set(JFROG_USER_NAME "test") - set(JFROG_PASSWORD "Fantast1c") + if(DEFINED ENV{JFROG_ARTFACTORY_URL}) + set(JFROG_ARTFACTORY_URL "$ENV{JFROG_ARTFACTORY_URL}") + endif() + if(NOT DEFINED JFROG_ARTFACTORY_URL) + message(FATAL_ERROR "JFROG_ARTFACTORY_URL is not set") + endif() + set(JFROG_ARTFACTORY_CACHE_URL "{JFROG_ARTFACTORY_URL}/generic-local/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") + if(DEFINED ENV{JFROG_USER_NAME}) + set(JFROG_USER_NAME "$ENV{JFROG_USER_NAME}") + endif() + if(NOT DEFINED JFROG_USER_NAME) + message(FATAL_ERROR "JFROG_USER_NAME is not set") + endif() + if(DEFINED ENV{JFROG_PASSWORD}) + set(JFROG_PASSWORD "$ENV{JFROG_PASSWORD}") + endif() + if(NOT DEFINED JFROG_PASSWORD) + message(FATAL_ERROR "JFROG_PASSWORD is not set") + endif() + set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache") endif() @@ -242,14 +259,6 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) set(${_LIB_NAME} "${_LIB_VERSION}") endforeach() -if(DEFINED ENV{MILVUS_ARROW_URL}) - set(ARROW_SOURCE_URL "$ENV{MILVUS_ARROW_URL}") -else() - set(ARROW_SOURCE_URL - "https://github.com/youny626/arrow.git" - ) -endif() - if(DEFINED ENV{MILVUS_BOOST_URL}) set(BOOST_SOURCE_URL "$ENV{MILVUS_BOOST_URL}") else() From 0339a93aed0977c851f83bd70e834a3282c4d927 Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Wed, 25 Sep 2019 15:08:29 +0800 Subject: [PATCH 086/217] Update CONTRIBUTING.md Former-commit-id: 1cd9de3f3556277bd9c798a2a6eec16ff5dbc921 --- cpp/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CONTRIBUTING.md b/cpp/CONTRIBUTING.md index 9e23d6b0ed..a10cb9cc49 100644 --- a/cpp/CONTRIBUTING.md +++ b/cpp/CONTRIBUTING.md @@ -4,7 +4,7 @@ First of all, thanks for taking the time to contribute to Milvus! It's people li The following are a set of guidelines for contributing to Milvus. Following these guidelines helps contributing to this project easy and transparent. These are mostly guideline, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request. -As for everything else in the project, the contributions to Milvus are governed by our [Code of Conduct](http://hood.ie/code-of-conduct/). +As for everything else in the project, the contributions to Milvus are governed by our [Code of Conduct](CODE_OF_CONDUCT.md). TOC From 84f77debca9dcc1c157cd0a46017b280863b908c Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Wed, 25 Sep 2019 15:09:06 +0800 Subject: [PATCH 087/217] Update CONTRIBUTING.md Former-commit-id: a8c9caa337275e27efc7f9ba38486942b1b76792 --- cpp/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CONTRIBUTING.md b/cpp/CONTRIBUTING.md index a10cb9cc49..3be9b7248b 100644 --- a/cpp/CONTRIBUTING.md +++ b/cpp/CONTRIBUTING.md @@ -4,7 +4,7 @@ First of all, thanks for taking the time to contribute to Milvus! It's people li The following are a set of guidelines for contributing to Milvus. Following these guidelines helps contributing to this project easy and transparent. These are mostly guideline, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request. -As for everything else in the project, the contributions to Milvus are governed by our [Code of Conduct](CODE_OF_CONDUCT.md). +As for everything else in the project, the contributions to Milvus are governed by our [Code of Conduct](CODE OF CONDUCT.md). TOC From 41a2cc343254045d1764b19edd231599ed97ba46 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 25 Sep 2019 15:17:06 +0800 Subject: [PATCH 088/217] format CMakeLists.txt Former-commit-id: 281e0bcb78233f6570c426fab6f9a4a067adf790 --- cpp/cmake/ThirdPartyPackages.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 37e9f4c3cd..a178468b0f 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -143,7 +143,7 @@ if(USE_JFROG_CACHE STREQUAL "ON") if(NOT DEFINED JFROG_ARTFACTORY_URL) message(FATAL_ERROR "JFROG_ARTFACTORY_URL is not set") endif() - set(JFROG_ARTFACTORY_CACHE_URL "{JFROG_ARTFACTORY_URL}/generic-local/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") + set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/generic-local/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") if(DEFINED ENV{JFROG_USER_NAME}) set(JFROG_USER_NAME "$ENV{JFROG_USER_NAME}") endif() From f10b7ed8b057c4b6b47fd065e0bb3c66bc0d81df Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Wed, 25 Sep 2019 15:18:23 +0800 Subject: [PATCH 089/217] Update CONTRIBUTING.md Former-commit-id: c00ea6cb77a7fa35c2ccc864226af5fbb18a64ea --- cpp/CONTRIBUTING.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cpp/CONTRIBUTING.md b/cpp/CONTRIBUTING.md index 3be9b7248b..0d9e30f678 100644 --- a/cpp/CONTRIBUTING.md +++ b/cpp/CONTRIBUTING.md @@ -36,8 +36,8 @@ The Milvus team members will review your pull requests, and once it is accepted, Before sending your pull requests for review, make sure your changes are consistent with the guidelines and follow the Milvus coding style. -- Include unit tests when you contribute new features, as they help to a) prove that your code works correctly, and b) guard against future breaking changes to lower the maintenance cost. -- Bug fixes also generally require unit tests, because the presence of bugs usually indicates insufficient test coverage. +- Include unit tests when you contribute new features, as they help to prove that your code works correctly, and also guard against future breaking changes to lower the maintenance cost. +- Bug fixes also require unit tests, because the presence of bugs usually indicates insufficient test coverage. - Keep API compatibility in mind when you change code in Milvus. Reviewers of your pull request will comment on any API compatibility issues. - When you contribute a new feature to Milvus, the maintenance burden is (by default) transferred to the Milvus team. This means that the benefit of the contribution must be compared against the cost of maintaining the feature. @@ -49,8 +49,6 @@ Before sending your pull requests for review, make sure your changes are consist ```shell $ ./build.sh -u -or -$ ./build.sh --unittest ``` From dc273ae4acb1be6aa8495b1b85f12fcfc787d104 Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Wed, 25 Sep 2019 16:09:18 +0800 Subject: [PATCH 090/217] Update CONTRIBUTING.md Former-commit-id: 1d9bda6fbbe40be5562dd6f6d03910c5ab5fc5f6 --- cpp/CONTRIBUTING.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cpp/CONTRIBUTING.md b/cpp/CONTRIBUTING.md index 0d9e30f678..8711d87050 100644 --- a/cpp/CONTRIBUTING.md +++ b/cpp/CONTRIBUTING.md @@ -43,11 +43,22 @@ Before sending your pull requests for review, make sure your changes are consist ## Coding Style +The coding style used in Milvus generally follow [Google C++ Style Guide](https://google.github.io/styleguide/cppguide.html). +And we made the following changes based on the guide: + +- 4 spaces for indentation +- Adopt .cpp file extension instead of .cc extension +- 120-character line length +- The file name starts with the upper case ## Run unit test +We use Google Test framework for test running. +To run unit test for Milvus under C++, please use the following command: + ```shell +# Run unit test for Milvus $ ./build.sh -u ``` From 0d0d08c256fb7ef99265c394d5bfb5dc1c41c653 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 25 Sep 2019 16:16:08 +0800 Subject: [PATCH 091/217] MS-574 let all config APIs return Status Former-commit-id: 7a843fa57779637761a80def4705a37f9805ceaf --- cpp/src/cache/CpuCacheMgr.cpp | 15 +- cpp/src/cache/GpuCacheMgr.cpp | 14 +- cpp/src/db/engine/ExecutionEngineImpl.cpp | 3 +- cpp/src/db/engine/ExecutionEngineImpl.h | 2 +- cpp/src/metrics/Metrics.cpp | 4 +- cpp/src/metrics/PrometheusMetrics.cpp | 7 +- cpp/src/scheduler/SchedInst.cpp | 7 +- cpp/src/server/Config.cpp | 588 ++++++++++++++-------- cpp/src/server/Config.h | 93 +++- cpp/src/server/DBWrapper.cpp | 45 +- cpp/src/server/Server.cpp | 8 +- cpp/src/server/grpc_impl/GrpcServer.cpp | 13 +- cpp/src/wrapper/KnowhereResource.cpp | 11 +- 13 files changed, 538 insertions(+), 272 deletions(-) diff --git a/cpp/src/cache/CpuCacheMgr.cpp b/cpp/src/cache/CpuCacheMgr.cpp index 9a69bf599f..1f93ec92bd 100644 --- a/cpp/src/cache/CpuCacheMgr.cpp +++ b/cpp/src/cache/CpuCacheMgr.cpp @@ -30,10 +30,21 @@ namespace { CpuCacheMgr::CpuCacheMgr() { server::Config& config = server::Config::GetInstance(); - int64_t cap = config.GetCacheConfigCpuMemCapacity() * unit; + Status s; + + int32_t cpu_mem_cap; + s = config.GetCacheConfigCpuMemCapacity(cpu_mem_cap); + if (!s.ok()) { + SERVER_LOG_ERROR << s.message(); + } + int64_t cap = cpu_mem_cap * unit; cache_ = std::make_shared>(cap, 1UL<<32); - float cpu_mem_threshold = config.GetCacheConfigCpuMemThreshold(); + float cpu_mem_threshold; + s = config.GetCacheConfigCpuMemThreshold(cpu_mem_threshold); + if (!s.ok()) { + SERVER_LOG_ERROR << s.message(); + } if (cpu_mem_threshold > 0.0 && cpu_mem_threshold <= 1.0) { cache_->set_freemem_percent(cpu_mem_threshold); } else { diff --git a/cpp/src/cache/GpuCacheMgr.cpp b/cpp/src/cache/GpuCacheMgr.cpp index bc6c3e13db..5c104ba0e7 100644 --- a/cpp/src/cache/GpuCacheMgr.cpp +++ b/cpp/src/cache/GpuCacheMgr.cpp @@ -34,11 +34,21 @@ namespace { GpuCacheMgr::GpuCacheMgr() { server::Config& config = server::Config::GetInstance(); + Status s; - int32_t cap = config.GetCacheConfigGpuMemCapacity() * G_BYTE; + int32_t gpu_mem_cap; + s = config.GetCacheConfigGpuMemCapacity(gpu_mem_cap); + if (!s.ok()) { + SERVER_LOG_ERROR << s.message(); + } + int32_t cap = gpu_mem_cap * G_BYTE; cache_ = std::make_shared>(cap, 1UL<<32); - float gpu_mem_threshold = config.GetCacheConfigGpuMemThreshold(); + float gpu_mem_threshold; + s = config.GetCacheConfigGpuMemThreshold(gpu_mem_threshold); + if (!s.ok()) { + SERVER_LOG_ERROR << s.message(); + } if (gpu_mem_threshold > 0.0 && gpu_mem_threshold <= 1.0) { cache_->set_freemem_percent(gpu_mem_threshold); } else { diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 742d21a0be..a2640cb527 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -336,7 +336,8 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { Status ExecutionEngineImpl::Init() { using namespace zilliz::milvus::server; server::Config &config = server::Config::GetInstance(); - gpu_num_ = config.GetDBConfigBuildIndexGPU(); + Status s = config.GetDBConfigBuildIndexGPU(gpu_num_); + if (!s.ok()) return s; return Status::OK(); } diff --git a/cpp/src/db/engine/ExecutionEngineImpl.h b/cpp/src/db/engine/ExecutionEngineImpl.h index cb08c50ad4..44d59d83e8 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.h +++ b/cpp/src/db/engine/ExecutionEngineImpl.h @@ -101,7 +101,7 @@ protected: std::string location_; int32_t nlist_ = 0; - int64_t gpu_num_ = 0; + int32_t gpu_num_ = 0; }; diff --git a/cpp/src/metrics/Metrics.cpp b/cpp/src/metrics/Metrics.cpp index 6c6df56b17..612f5e7fce 100644 --- a/cpp/src/metrics/Metrics.cpp +++ b/cpp/src/metrics/Metrics.cpp @@ -33,7 +33,9 @@ Metrics::GetInstance() { MetricsBase & Metrics::CreateMetricsCollector() { Config &config = Config::GetInstance(); - std::string collector_type_str = config.GetMetricConfigCollector(); + std::string collector_type_str; + + config.GetMetricConfigCollector(collector_type_str); if (collector_type_str == "prometheus") { return PrometheusMetrics::GetInstance(); diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index 14d2de6732..b607dbb020 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -31,11 +31,14 @@ ErrorCode PrometheusMetrics::Init() { try { Config &config = Config::GetInstance(); - startup_ = config.GetMetricConfigAutoBootup(); + Status s = config.GetMetricConfigAutoBootup(startup_); + if (!s.ok()) return s.code(); if (!startup_) return SERVER_SUCCESS; // Following should be read from config file. - const std::string bind_address = config.GetMetricConfigPrometheusPort(); + std::string bind_address; + s = config.GetMetricConfigPrometheusPort(bind_address); + if (!s.ok()) return s.code(); const std::string uri = std::string("/tmp/metrics"); const std::size_t num_threads = 2; diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index 5c4ec522da..8fe19f5b3e 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -39,8 +39,11 @@ std::mutex JobMgrInst::mutex_; void load_simple_config() { server::Config &config = server::Config::GetInstance(); - auto mode = config.GetResourceConfigMode(); - auto pool = config.GetResourceConfigPool(); + std::string mode; + config.GetResourceConfigMode(mode); + std::vector pool; + config.GetResourceConfigPool(pool); + bool cpu = false; std::set gpu_ids; for (auto &resource : pool) { diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 9f7b9fd12e..9d76fa72c6 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -105,7 +105,7 @@ Config::CheckServerConfig() { */ bool okay = true; - ConfigNode server_config = GetConfig(CONFIG_SERVER); + ConfigNode server_config = GetConfigNode(CONFIG_SERVER); std::string ip_address = server_config.GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); if (!ValidationUtil::ValidateIpAddress(ip_address).ok()) { @@ -173,7 +173,7 @@ Config::CheckDBConfig() { build_index_gpu: 0 # which gpu is used to build index, default: 0, range: 0 ~ gpu number - 1 */ bool okay = true; - ConfigNode db_config = GetConfig(CONFIG_DB); + ConfigNode db_config = GetConfigNode(CONFIG_DB); std::string db_path = db_config.GetValue(CONFIG_DB_PATH); if (db_path.empty()) { @@ -187,13 +187,15 @@ Config::CheckDBConfig() { okay = false; } - std::string archive_disk_threshold_str = db_config.GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); + std::string archive_disk_threshold_str = + db_config.GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(archive_disk_threshold_str).ok()) { std::cerr << "ERROR: archive_disk_threshold " << archive_disk_threshold_str << " is not a number" << std::endl; okay = false; } - std::string archive_days_threshold_str = db_config.GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); + std::string archive_days_threshold_str = + db_config.GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(archive_days_threshold_str).ok()) { std::cerr << "ERROR: archive_days_threshold " << archive_days_threshold_str << " is not a number" << std::endl; okay = false; @@ -203,8 +205,7 @@ Config::CheckDBConfig() { if (!ValidationUtil::ValidateStringIsNumber(buffer_size_str).ok()) { std::cerr << "ERROR: buffer_size " << buffer_size_str << " is not a number" << std::endl; okay = false; - } - else { + } else { uint64_t buffer_size = (uint64_t) std::stol(buffer_size_str); buffer_size *= GB; unsigned long total_mem = 0, free_mem = 0; @@ -242,7 +243,7 @@ Config::CheckMetricConfig() { (not used) push_gateway_port: 9091 # push method configure: push gateway port */ bool okay = true; - ConfigNode metric_config = GetConfig(CONFIG_METRIC); + ConfigNode metric_config = GetConfigNode(CONFIG_METRIC); std::string is_startup_str = metric_config.GetValue(CONFIG_METRIC_AUTO_BOOTUP, CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); @@ -251,7 +252,8 @@ Config::CheckMetricConfig() { okay = false; } - std::string port_str = metric_config.GetChild(CONFIG_METRIC_PROMETHEUS).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, "8080"); + std::string + port_str = metric_config.GetChild(CONFIG_METRIC_PROMETHEUS).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, "8080"); if (!ValidationUtil::ValidateStringIsNumber(port_str).ok()) { std::cerr << "ERROR: port specified in prometheus_config " << port_str << " is not a number" << std::endl; okay = false; @@ -272,15 +274,14 @@ Config::CheckCacheConfig() { */ bool okay = true; - ConfigNode cache_config = GetConfig(CONFIG_CACHE); + ConfigNode cache_config = GetConfigNode(CONFIG_CACHE); std::string cpu_cache_capacity_str = cache_config.GetValue(CONFIG_CACHE_CPU_MEM_CAPACITY, CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(cpu_cache_capacity_str).ok()) { std::cerr << "ERROR: cpu_cache_capacity " << cpu_cache_capacity_str << " is not a number" << std::endl; okay = false; - } - else { + } else { uint64_t cpu_cache_capacity = (uint64_t) std::stol(cpu_cache_capacity_str); cpu_cache_capacity *= GB; unsigned long total_mem = 0, free_mem = 0; @@ -288,13 +289,13 @@ Config::CheckCacheConfig() { if (cpu_cache_capacity >= total_mem) { std::cerr << "ERROR: cpu_cache_capacity exceed system memory" << std::endl; okay = false; - } - else if (cpu_cache_capacity > (double) total_mem * 0.9) { + } else if (cpu_cache_capacity > (double) total_mem * 0.9) { std::cerr << "Warning: cpu_cache_capacity value is too aggressive" << std::endl; } uint64_t buffer_size = - (uint64_t) GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_BUFFER_SIZE, std::stoi(CONFIG_DB_BUFFER_SIZE_DEFAULT)); + (uint64_t) GetConfigNode(CONFIG_DB).GetInt32Value(CONFIG_DB_BUFFER_SIZE, + std::stoi(CONFIG_DB_BUFFER_SIZE_DEFAULT)); buffer_size *= GB; if (buffer_size + cpu_cache_capacity >= total_mem) { std::cerr << "ERROR: sum of cpu_cache_capacity and insert_buffer_size exceed system memory" << std::endl; @@ -308,8 +309,7 @@ Config::CheckCacheConfig() { if (!ValidationUtil::ValidateStringIsDouble(cpu_cache_free_percent_str, cpu_cache_free_percent).ok()) { std::cerr << "ERROR: cpu_cache_free_percent " << cpu_cache_free_percent_str << " is not a double" << std::endl; okay = false; - } - else if (cpu_cache_free_percent < std::numeric_limits::epsilon() || cpu_cache_free_percent > 1.0) { + } else if (cpu_cache_free_percent < std::numeric_limits::epsilon() || cpu_cache_free_percent > 1.0) { std::cerr << "ERROR: invalid cpu_cache_free_percent " << cpu_cache_free_percent_str << std::endl; okay = false; } @@ -326,22 +326,20 @@ Config::CheckCacheConfig() { if (!ValidationUtil::ValidateStringIsNumber(gpu_cache_capacity_str).ok()) { std::cerr << "ERROR: gpu_cache_capacity " << gpu_cache_capacity_str << " is not a number" << std::endl; okay = false; - } - else { + } else { uint64_t gpu_cache_capacity = (uint64_t) std::stol(gpu_cache_capacity_str); gpu_cache_capacity *= GB; - int gpu_index = GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); + int gpu_index = GetConfigNode(CONFIG_DB).GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, + std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); size_t gpu_memory; if (!ValidationUtil::GetGpuMemory(gpu_index, gpu_memory).ok()) { std::cerr << "ERROR: could not get gpu memory for device " << gpu_index << std::endl; okay = false; - } - else if (gpu_cache_capacity >= gpu_memory) { + } else if (gpu_cache_capacity >= gpu_memory) { std::cerr << "ERROR: gpu_cache_capacity " << gpu_cache_capacity << " exceed total gpu memory " << gpu_memory << std::endl; okay = false; - } - else if (gpu_cache_capacity > (double) gpu_memory * 0.9) { + } else if (gpu_cache_capacity > (double) gpu_memory * 0.9) { std::cerr << "Warning: gpu_cache_capacity value is too aggressive" << std::endl; } } @@ -352,8 +350,7 @@ Config::CheckCacheConfig() { if (!ValidationUtil::ValidateStringIsDouble(gpu_cache_free_percent_str, gpu_cache_free_percent).ok()) { std::cerr << "ERROR: gpu_cache_free_percent " << gpu_cache_free_percent_str << " is not a double" << std::endl; okay = false; - } - else if (gpu_cache_free_percent < std::numeric_limits::epsilon() || gpu_cache_free_percent > 1.0) { + } else if (gpu_cache_free_percent < std::numeric_limits::epsilon() || gpu_cache_free_percent > 1.0) { std::cerr << "ERROR: invalid gpu_cache_free_percent " << gpu_cache_free_percent << std::endl; okay = false; } @@ -369,7 +366,7 @@ Config::CheckEngineConfig() { omp_thread_num: 0 # how many compute threads be used by engine, 0 means use all cpu core to compute */ bool okay = true; - ConfigNode engine_config = GetConfig(CONFIG_ENGINE); + ConfigNode engine_config = GetConfigNode(CONFIG_ENGINE); std::string use_blas_threshold_str = engine_config.GetValue(CONFIG_ENGINE_BLAS_THRESHOLD, CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); @@ -407,7 +404,7 @@ Config::CheckResourceConfig() { - gpu100 */ bool okay = true; - server::ConfigNode &config = GetConfig(server::CONFIG_RESOURCE); + server::ConfigNode &config = GetConfigNode(server::CONFIG_RESOURCE); auto mode = config.GetValue(CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); if (mode != "simple") { std::cerr << "ERROR: invalid resource config: mode is " << mode << std::endl; @@ -613,15 +610,8 @@ Config::PrintAll() const { } } -ConfigNode -Config::GetConfig(const std::string &name) const { - const ConfigMgr *mgr = ConfigMgr::GetInstance(); - const ConfigNode &root_node = mgr->GetRootNode(); - return root_node.GetChild(name); -} - ConfigNode & -Config::GetConfig(const std::string &name) { +Config::GetConfigNode(const std::string &name) { ConfigMgr *mgr = ConfigMgr::GetInstance(); ConfigNode &root_node = mgr->GetRootNode(); return root_node.GetChild(name); @@ -641,274 +631,436 @@ Config::GetConfigValueInMem(const std::string &parent_key, } } -void +Status Config::SetConfigValueInMem(const std::string &parent_key, const std::string &child_key, std::string &value) { std::lock_guard lock(mutex_); config_map_[parent_key][child_key] = value; + return Status::OK(); } //////////////////////////////////////////////////////////////////////////////// /* server config */ -std::string -Config::GetServerConfigAddress() { - std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value).ok()) { - value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_ADDRESS, - CONFIG_SERVER_ADDRESS_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); +Status +Config::GetServerConfigStrAddress(std::string& value) { + if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_ADDRESS, + CONFIG_SERVER_ADDRESS_DEFAULT); + return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); } - return value; } -std::string -Config::GetServerConfigPort() { - std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value).ok()) { - value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_PORT, - CONFIG_SERVER_PORT_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); +Status +Config::GetServerConfigStrPort(std::string& value) { + if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_PORT, + CONFIG_SERVER_PORT_DEFAULT); + return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); } - return value; } -std::string -Config::GetServerConfigMode() { - std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value).ok()) { - value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_MODE, - CONFIG_SERVER_MODE_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); +Status +Config::GetServerConfigStrMode(std::string& value) { + if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_MODE, + CONFIG_SERVER_MODE_DEFAULT); + return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); } - return value; } -std::string -Config::GetServerConfigTimeZone() { - std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value).ok()) { - value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_TIME_ZONE, - CONFIG_SERVER_TIME_ZONE_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); +Status +Config::GetServerConfigStrTimeZone(std::string& value) { + if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_TIME_ZONE, + CONFIG_SERVER_TIME_ZONE_DEFAULT); + return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); } - return value; } //////////////////////////////////////////////////////////////////////////////// /* db config */ -std::string -Config::GetDBConfigPath() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value).ok()) { - value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_PATH, - CONFIG_DB_PATH_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); +Status +Config::GetDBConfigStrPath(std::string& value) { + if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PATH, + CONFIG_DB_PATH_DEFAULT); + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); } - return value; } -std::string -Config::GetDBConfigSlavePath() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value).ok()) { - value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_SLAVE_PATH, - CONFIG_DB_SLAVE_PATH_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); +Status +Config::GetDBConfigStrSlavePath(std::string& value) { + if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SLAVE_PATH, + CONFIG_DB_SLAVE_PATH_DEFAULT); + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); } - return value; } -std::string -Config::GetDBConfigBackendUrl() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value).ok()) { - value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_BACKEND_URL, - CONFIG_DB_BACKEND_URL_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); +Status +Config::GetDBConfigStrBackendUrl(std::string& value) { + if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BACKEND_URL, + CONFIG_DB_BACKEND_URL_DEFAULT); + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); } - return value; } -int32_t -Config::GetDBConfigArchiveDiskThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value).ok()) { - value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, - CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); +Status +Config::GetDBConfigStrArchiveDiskThreshold(std::string& value) { + if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, + CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); } - return std::stoi(value); } -int32_t -Config::GetDBConfigArchiveDaysThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value).ok()) { - value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, - CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); +Status +Config::GetDBConfigStrArchiveDaysThreshold(std::string& value) { + if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, + CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); } - return std::stoi(value); } -int32_t -Config::GetDBConfigBufferSize() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value).ok()) { - value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_BUFFER_SIZE, - CONFIG_DB_BUFFER_SIZE_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); +Status +Config::GetDBConfigStrBufferSize(std::string& value) { + if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUFFER_SIZE, + CONFIG_DB_BUFFER_SIZE_DEFAULT); + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); } - return std::stoi(value); } -int32_t -Config::GetDBConfigBuildIndexGPU() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value).ok()) { - value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_BUILD_INDEX_GPU, - CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); +Status +Config::GetDBConfigStrBuildIndexGPU(std::string& value) { + if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUILD_INDEX_GPU, + CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); } - return std::stoi(value); } //////////////////////////////////////////////////////////////////////////////// /* metric config */ -bool -Config::GetMetricConfigAutoBootup() { - std::string value; - if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value).ok()) { - value = GetConfig(CONFIG_METRIC).GetValue(CONFIG_METRIC_AUTO_BOOTUP, - CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value); +Status +Config::GetMetricConfigStrAutoBootup(std::string& value) { + if (GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_AUTO_BOOTUP, + CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); + return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value); } - std::transform(value.begin(), value.end(), value.begin(), ::tolower); - return (value == "true" || value == "on" || value == "yes" || value == "1"); } -std::string -Config::GetMetricConfigCollector() { - std::string value; - if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value).ok()) { - value = GetConfig(CONFIG_METRIC).GetValue(CONFIG_METRIC_COLLECTOR, - CONFIG_METRIC_COLLECTOR_DEFAULT); - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); +Status +Config::GetMetricConfigStrCollector(std::string& value) { + if (GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_COLLECTOR, + CONFIG_METRIC_COLLECTOR_DEFAULT); + return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); } - return value; } -std::string -Config::GetMetricConfigPrometheusPort() { - std::string value; - if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value).ok()) { - value = GetConfig(CONFIG_METRIC).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, - CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); +Status +Config::GetMetricConfigStrPrometheusPort(std::string& value) { + if (GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, + CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); + return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); } - return value; } //////////////////////////////////////////////////////////////////////////////// /* cache config */ -int32_t -Config::GetCacheConfigCpuMemCapacity() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value).ok()) { - value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_CAPACITY, - CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value); +Status +Config::GetCacheConfigStrCpuMemCapacity(std::string& value) { + if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_CAPACITY, + CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); + return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value); } - return std::stoi(value); } -float -Config::GetCacheConfigCpuMemThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value).ok()) { - value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, - CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value); +Status +Config::GetCacheConfigStrCpuMemThreshold(std::string& value) { + if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, + CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT); + return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value); } - return std::stof(value); } -int32_t -Config::GetCacheConfigGpuMemCapacity() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value).ok()) { - value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_CAPACITY, - CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value); +Status +Config::GetCacheConfigStrGpuMemCapacity(std::string& value) { + if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_CAPACITY, + CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT); + return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value); } - return std::stoi(value); } -float -Config::GetCacheConfigGpuMemThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value).ok()) { - value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, - CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value); +Status +Config::GetCacheConfigStrGpuMemThreshold(std::string& value) { + if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, + CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT); + return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value); } - return std::stof(value); } -bool -Config::GetCacheConfigCacheInsertData() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value).ok()) { - value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, - CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); +Status +Config::GetCacheConfigStrCacheInsertData(std::string& value) { + if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, + CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); + return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); } - std::transform(value.begin(), value.end(), value.begin(), ::tolower); - return (value == "true" || value == "on" || value == "yes" || value == "1"); } //////////////////////////////////////////////////////////////////////////////// /* engine config */ -int32_t -Config::GetEngineConfigBlasThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value).ok()) { - value = GetConfig(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_BLAS_THRESHOLD, - CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value); +Status +Config::GetEngineConfigStrBlasThreshold(std::string& value) { + if (GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_BLAS_THRESHOLD, + CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); + return SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value); } - return std::stoi(value); } -int32_t -Config::GetEngineConfigOmpThreadNum() { - std::string value; - if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value).ok()) { - value = GetConfig(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, - CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); +Status +Config::GetEngineConfigStrOmpThreadNum(std::string& value) { + if (GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, + CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); + return SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); } - return std::stoi(value); } //////////////////////////////////////////////////////////////////////////////// /* resource config */ -std::string -Config::GetResourceConfigMode() { - std::string value; - if (!GetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value).ok()) { - value = GetConfig(CONFIG_RESOURCE).GetValue(CONFIG_RESOURCE_MODE, - CONFIG_RESOURCE_MODE_DEFAULT); - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); +Status +Config::GetResourceConfigStrMode(std::string& value) { + if (GetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_RESOURCE).GetValue(CONFIG_RESOURCE_MODE, + CONFIG_RESOURCE_MODE_DEFAULT); + return SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); } - return value; } -std::vector -Config::GetResourceConfigPool() { - ConfigNode resource_config = GetConfig(CONFIG_RESOURCE); - return resource_config.GetSequence(CONFIG_RESOURCE_POOL); + +//////////////////////////////////////////////////////////////////////////////// +Status +Config::GetServerConfigAddress(std::string& value) { + return GetServerConfigStrAddress(value); +} + +Status +Config::GetServerConfigPort(std::string& value) { + return GetServerConfigStrPort(value); +} + +Status +Config::GetServerConfigMode(std::string& value) { + return GetServerConfigStrMode(value); +} + +Status +Config::GetServerConfigTimeZone(std::string& value) { + return GetServerConfigStrTimeZone(value); +} + +Status +Config::GetDBConfigPath(std::string& value) { + return GetDBConfigStrPath(value); +} + +Status +Config::GetDBConfigSlavePath(std::string& value) { + return GetDBConfigStrSlavePath(value); +} + +Status +Config::GetDBConfigBackendUrl(std::string& value) { + return GetDBConfigStrBackendUrl(value); +} + +Status +Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { + std::string str; + Status s = GetDBConfigStrArchiveDiskThreshold(str); + if (!s.ok()) return s; + value = std::stoi(str); + return Status::OK(); +} + +Status +Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { + std::string str; + Status s = GetDBConfigStrArchiveDaysThreshold(str); + if (!s.ok()) return s; + value = std::stoi(str); + return Status::OK(); +} + +Status +Config::GetDBConfigBufferSize(int32_t& value) { + std::string str; + Status s = GetDBConfigStrBufferSize(str); + if (!s.ok()) return s; + value = std::stoi(str); + return Status::OK(); +} + +Status +Config::GetDBConfigBuildIndexGPU(int32_t& value) { + std::string str; + Status s = GetDBConfigStrBuildIndexGPU(str); + if (!s.ok()) return s; + value = std::stoi(str); + return Status::OK(); +} + +Status +Config::GetMetricConfigAutoBootup(bool& value) { + std::string str; + Status s = GetMetricConfigStrAutoBootup(str); + if (!s.ok()) return s; + std::transform(str.begin(), str.end(), str.begin(), ::tolower); + value = (str == "true" || str == "on" || str == "yes" || str == "1"); + return Status::OK(); +} + +Status +Config::GetMetricConfigCollector(std::string& value) { + return GetMetricConfigStrCollector(value); +} + +Status +Config::GetMetricConfigPrometheusPort(std::string& value) { + return GetMetricConfigStrPrometheusPort(value); +} + +Status +Config::GetCacheConfigCpuMemCapacity(int32_t& value) { + std::string str; + Status s = GetCacheConfigStrCpuMemCapacity(str); + if (!s.ok()) return s; + value = std::stoi(str); + return Status::OK(); +} + +Status +Config::GetCacheConfigCpuMemThreshold(float& value) { + std::string str; + Status s = GetCacheConfigStrCpuMemThreshold(str); + if (!s.ok()) return s; + value = std::stof(str); + return Status::OK(); +} + +Status +Config::GetCacheConfigGpuMemCapacity(int32_t& value) { + std::string str; + Status s = GetCacheConfigStrGpuMemCapacity(str); + if (!s.ok()) return s; + value = std::stoi(str); + return Status::OK(); +} + +Status +Config::GetCacheConfigGpuMemThreshold(float& value) { + std::string str; + Status s = GetCacheConfigStrGpuMemThreshold(str); + if (!s.ok()) return s; + value = std::stof(str); + return Status::OK(); +} + +Status +Config::GetCacheConfigCacheInsertData(bool& value) { + std::string str; + Status s = GetCacheConfigStrCacheInsertData(str); + if (!s.ok()) return s; + std::transform(str.begin(), str.end(), str.begin(), ::tolower); + value = (str == "true" || str == "on" || str == "yes" || str == "1"); + return Status::OK(); +} + +Status +Config::GetEngineConfigBlasThreshold(int32_t& value) { + std::string str; + Status s = GetEngineConfigStrBlasThreshold(str); + if (!s.ok()) return s; + value = std::stoi(str); + return Status::OK(); +} + +Status +Config::GetEngineConfigOmpThreadNum(int32_t& value) { + std::string str; + Status s = GetEngineConfigStrOmpThreadNum(str); + if (!s.ok()) return s; + value = std::stoi(str); + return Status::OK(); +} + +Status +Config::GetResourceConfigMode(std::string& value) { + return GetResourceConfigStrMode(value); +} + +Status +Config::GetResourceConfigPool(std::vector& value) { + ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE); + value = resource_config.GetSequence(CONFIG_RESOURCE_POOL); + return Status::OK(); } } diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index 992ccf0965..3eeda32546 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -102,8 +102,7 @@ class Config { void PrintAll() const; private: - ConfigNode GetConfig(const std::string& name) const; - ConfigNode& GetConfig(const std::string& name); + ConfigNode& GetConfigNode(const std::string& name); Status CheckServerConfig(); Status CheckDBConfig(); @@ -116,40 +115,80 @@ class Config { const std::string& child_key, std::string& value); - void SetConfigValueInMem(const std::string& parent_key, + Status SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value); + private: + /* server config */ + Status GetServerConfigStrAddress(std::string& value); + Status GetServerConfigStrPort(std::string& value); + Status GetServerConfigStrMode(std::string& value); + Status GetServerConfigStrTimeZone(std::string& value); + + /* db config */ + Status GetDBConfigStrPath(std::string& value); + Status GetDBConfigStrSlavePath(std::string& value); + Status GetDBConfigStrBackendUrl(std::string& value); + Status GetDBConfigStrArchiveDiskThreshold(std::string& value); + Status GetDBConfigStrArchiveDaysThreshold(std::string& value); + Status GetDBConfigStrBufferSize(std::string& value); + Status GetDBConfigStrBuildIndexGPU(std::string& value); + + /* metric config */ + Status GetMetricConfigStrAutoBootup(std::string& value); + Status GetMetricConfigStrCollector(std::string& value); + Status GetMetricConfigStrPrometheusPort(std::string& value); + + /* cache config */ + Status GetCacheConfigStrCpuMemCapacity(std::string& value); + Status GetCacheConfigStrCpuMemThreshold(std::string& value); + Status GetCacheConfigStrGpuMemCapacity(std::string& value); + Status GetCacheConfigStrGpuMemThreshold(std::string& value); + Status GetCacheConfigStrCacheInsertData(std::string& value); + + /* engine config */ + Status GetEngineConfigStrBlasThreshold(std::string& value); + Status GetEngineConfigStrOmpThreadNum(std::string& value); + + /* resource config */ + Status GetResourceConfigStrMode(std::string& value); + public: - std::string GetServerConfigAddress(); - std::string GetServerConfigPort(); - std::string GetServerConfigMode(); - std::string GetServerConfigTimeZone(); + /* server config */ + Status GetServerConfigAddress(std::string& value); + Status GetServerConfigPort(std::string& value); + Status GetServerConfigMode(std::string& value); + Status GetServerConfigTimeZone(std::string& value); - std::string GetDBConfigPath(); - std::string GetDBConfigSlavePath(); - std::string GetDBConfigBackendUrl(); - int32_t GetDBConfigArchiveDiskThreshold(); - int32_t GetDBConfigArchiveDaysThreshold(); - int32_t GetDBConfigBufferSize(); - int32_t GetDBConfigBuildIndexGPU(); + /* db config */ + Status GetDBConfigPath(std::string& value); + Status GetDBConfigSlavePath(std::string& value); + Status GetDBConfigBackendUrl(std::string& value); + Status GetDBConfigArchiveDiskThreshold(int32_t& value); + Status GetDBConfigArchiveDaysThreshold(int32_t& value); + Status GetDBConfigBufferSize(int32_t& value); + Status GetDBConfigBuildIndexGPU(int32_t& value); - bool GetMetricConfigAutoBootup(); - std::string GetMetricConfigCollector(); - std::string GetMetricConfigPrometheusPort(); + /* metric config */ + Status GetMetricConfigAutoBootup(bool& value); + Status GetMetricConfigCollector(std::string& value); + Status GetMetricConfigPrometheusPort(std::string& value); - int32_t GetCacheConfigCpuMemCapacity(); - float GetCacheConfigCpuMemThreshold(); - int32_t GetCacheConfigGpuMemCapacity(); - float GetCacheConfigGpuMemThreshold(); - bool GetCacheConfigCacheInsertData(); + /* cache config */ + Status GetCacheConfigCpuMemCapacity(int32_t& value); + Status GetCacheConfigCpuMemThreshold(float& value); + Status GetCacheConfigGpuMemCapacity(int32_t& value); + Status GetCacheConfigGpuMemThreshold(float& value); + Status GetCacheConfigCacheInsertData(bool& value); - int32_t GetEngineConfigBlasThreshold(); - int32_t GetEngineConfigOmpThreadNum(); + /* engine config */ + Status GetEngineConfigBlasThreshold(int32_t& value); + Status GetEngineConfigOmpThreadNum(int32_t& value); - std::string GetResourceConfigMode(); - std::vector - GetResourceConfigPool(); + /* resource config */ + Status GetResourceConfigMode(std::string& value); + Status GetResourceConfigPool(std::vector& value); private: std::unordered_map> config_map_; diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index bc0b080625..bb628a0c63 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -36,19 +36,33 @@ DBWrapper::DBWrapper() { Status DBWrapper::StartService() { Config& config = Config::GetInstance(); - + Status s; //db config engine::DBOptions opt; - opt.meta.backend_uri = config.GetDBConfigBackendUrl(); - opt.meta.path = config.GetDBConfigPath() + "/db"; - std::string db_slave_path = config.GetDBConfigSlavePath(); + s = config.GetDBConfigBackendUrl(opt.meta.backend_uri); + if (!s.ok()) return s; + + std::string path; + s = config.GetDBConfigPath(path); + if (!s.ok()) return s; + + opt.meta.path = path + "/db"; + + std::string db_slave_path; + s = config.GetDBConfigSlavePath(db_slave_path); + if (!s.ok()) return s; + StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta.slave_paths); // cache config - opt.insert_cache_immediately_ = config.GetCacheConfigCacheInsertData(); + s = config.GetCacheConfigCacheInsertData(opt.insert_cache_immediately_); + if (!s.ok()) return s; + + std::string mode; + s = config.GetServerConfigMode(mode); + if (!s.ok()) return s; - std::string mode = config.GetServerConfigMode(); if (mode == "single") { opt.mode = engine::DBOptions::MODE::SINGLE; } @@ -64,8 +78,10 @@ Status DBWrapper::StartService() { } // engine config - int32_t omp_thread = config.GetEngineConfigOmpThreadNum(); - if(omp_thread > 0) { + int32_t omp_thread; + s = config.GetEngineConfigOmpThreadNum(omp_thread); + if (!s.ok()) return s; + if (omp_thread > 0) { omp_set_num_threads(omp_thread); SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; } else { @@ -77,15 +93,22 @@ Status DBWrapper::StartService() { } //init faiss global variable - faiss::distance_compute_blas_threshold = config.GetEngineConfigBlasThreshold(); + int32_t blas_threshold; + s = config.GetEngineConfigBlasThreshold(blas_threshold); + if (!s.ok()) return s; + faiss::distance_compute_blas_threshold = blas_threshold; //set archive config engine::ArchiveConf::CriteriaT criterial; - int32_t disk = config.GetDBConfigArchiveDiskThreshold(); - int32_t days = config.GetDBConfigArchiveDaysThreshold(); + int32_t disk, days; + s = config.GetDBConfigArchiveDiskThreshold(disk); + if (!s.ok()) return s; if (disk > 0) { criterial[engine::ARCHIVE_CONF_DISK] = disk; } + + s = config.GetDBConfigArchiveDaysThreshold(days); + if (!s.ok()) return s; if (days > 0) { criterial[engine::ARCHIVE_CONF_DAYS] = days; } diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 7ca0b2a7c9..d926ca5dac 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -161,7 +161,13 @@ Server::Start() { /* log path is defined in Config file, so InitLog must be called after LoadConfig */ Config &config = Config::GetInstance(); - std::string time_zone = config.GetServerConfigTimeZone(); + std::string time_zone; + Status s = config.GetServerConfigTimeZone(time_zone); + if (!s.ok()) { + std::cerr << "Fail to get server config timezone" << std::endl; + return; + } + if (time_zone.length() == 3) { time_zone = "CUT"; } else { diff --git a/cpp/src/server/grpc_impl/GrpcServer.cpp b/cpp/src/server/grpc_impl/GrpcServer.cpp index c10d7b5379..7e20921549 100644 --- a/cpp/src/server/grpc_impl/GrpcServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcServer.cpp @@ -74,8 +74,17 @@ GrpcServer::Stop() { Status GrpcServer::StartService() { Config &config = Config::GetInstance(); - std::string address = config.GetServerConfigAddress(); - std::string port = config.GetServerConfigPort(); + std::string address, port; + Status s; + + s = config.GetServerConfigAddress(address); + if (!s.ok()) { + return s; + } + s = config.GetServerConfigPort(port); + if (!s.ok()) { + return s; + } std::string server_address(address + ":" + port); diff --git a/cpp/src/wrapper/KnowhereResource.cpp b/cpp/src/wrapper/KnowhereResource.cpp index 1e4248f6e1..a54ff1e84d 100644 --- a/cpp/src/wrapper/KnowhereResource.cpp +++ b/cpp/src/wrapper/KnowhereResource.cpp @@ -36,15 +36,22 @@ ErrorCode KnowhereResource::Initialize() { }; using GpuResourcesArray = std::map; GpuResourcesArray gpu_resources; + Status s; //get build index gpu resource server::Config& config = server::Config::GetInstance(); - int32_t build_index_gpu = config.GetDBConfigBuildIndexGPU(); + int32_t build_index_gpu; + s = config.GetDBConfigBuildIndexGPU(build_index_gpu); + if (!s.ok()) return s.code(); + gpu_resources.insert(std::make_pair(build_index_gpu, GpuResourceSetting())); //get search gpu resource - auto pool = config.GetResourceConfigPool(); + std::vector pool; + s = config.GetResourceConfigPool(pool); + if (!s.ok()) return s.code(); + std::set gpu_ids; for (auto &resource : pool) { if (resource.length() < 4 || resource.substr(0, 3) != "gpu") { From 0f287c39100d50c38a9db5135f5c3d7373677f5b Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 25 Sep 2019 16:31:30 +0800 Subject: [PATCH 092/217] format CMakeLists.txt Former-commit-id: b1980e2e8fda99a49d9bd005ce0caf0ef494ca02 --- cpp/src/core/cmake/ThirdPartyPackagesCore.cmake | 3 --- 1 file changed, 3 deletions(-) diff --git a/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake b/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake index fad0869021..7b0c75633d 100644 --- a/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake +++ b/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake @@ -123,9 +123,6 @@ if(NOT DEFINED USE_JFROG_CACHE) set(USE_JFROG_CACHE "OFF") endif() if(USE_JFROG_CACHE STREQUAL "ON") - set(JFROG_ARTFACTORY_CACHE_URL "http://192.168.1.201:80/artifactory/generic-local/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${KNOWHERE_BUILD_ARCH}/${BUILD_TYPE}") - set(JFROG_USER_NAME "test") - set(JFROG_PASSWORD "Fantast1c") set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache") endif() From e9307c3c240aa16610c77717bed361c8d8a81618 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 25 Sep 2019 17:30:14 +0800 Subject: [PATCH 093/217] add JFROG_USER parameter Former-commit-id: 7bb920f6efc3f33649f0766b34e1878872c75039 --- ci/jenkinsfile/milvus_build.groovy | 11 +++-------- ci/jenkinsfile/milvus_build_no_ut.groovy | 11 +++-------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/ci/jenkinsfile/milvus_build.groovy b/ci/jenkinsfile/milvus_build.groovy index 0e24c63a34..289b9a6fe8 100644 --- a/ci/jenkinsfile/milvus_build.groovy +++ b/ci/jenkinsfile/milvus_build.groovy @@ -5,17 +5,12 @@ container('milvus-build-env') { try { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - /* - dir ("cpp/thirdparty/knowhere") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/knowhere.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh "./build.sh -t ${params.BUILD_TYPE} -p ${knowhere_build_dir} -j" - } - */ - dir ("cpp") { sh "git config --global user.email \"test@zilliz.com\"" sh "git config --global user.name \"test\"" - sh "./build.sh -t ${params.BUILD_TYPE} -j -u -c" + withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { + sh "export JFROG_ARTFACTORY_URL='http://192.168.1.201:80/artifactory' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -j -u -c" + } } } catch (exc) { updateGitlabCommitStatus name: 'Build Engine', state: 'failed' diff --git a/ci/jenkinsfile/milvus_build_no_ut.groovy b/ci/jenkinsfile/milvus_build_no_ut.groovy index dd0fe4a442..4caa9f0fa7 100644 --- a/ci/jenkinsfile/milvus_build_no_ut.groovy +++ b/ci/jenkinsfile/milvus_build_no_ut.groovy @@ -5,17 +5,12 @@ container('milvus-build-env') { try { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - /* - dir ("cpp/thirdparty/knowhere") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/knowhere.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh "./build.sh -t ${params.BUILD_TYPE} -p ${knowhere_build_dir} -j" - } - */ - dir ("cpp") { sh "git config --global user.email \"test@zilliz.com\"" sh "git config --global user.name \"test\"" - sh "./build.sh -t ${params.BUILD_TYPE} -j" + withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { + sh "export JFROG_ARTFACTORY_URL='http://192.168.1.201:80/artifactory' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -j" + } } } catch (exc) { updateGitlabCommitStatus name: 'Build Engine', state: 'failed' From 964ee0013dd7da483067cf9c2cb75e9b6f57b850 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 25 Sep 2019 17:42:21 +0800 Subject: [PATCH 094/217] add JFROG parameter Former-commit-id: 13493ae141c2ba1081b6e4c4eb5f9aa49a4b03ee --- ci/jenkinsfile/milvus_build.groovy | 2 +- ci/jenkinsfile/milvus_build_no_ut.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/jenkinsfile/milvus_build.groovy b/ci/jenkinsfile/milvus_build.groovy index 289b9a6fe8..44895ed5c9 100644 --- a/ci/jenkinsfile/milvus_build.groovy +++ b/ci/jenkinsfile/milvus_build.groovy @@ -9,7 +9,7 @@ container('milvus-build-env') { sh "git config --global user.email \"test@zilliz.com\"" sh "git config --global user.name \"test\"" withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh "export JFROG_ARTFACTORY_URL='http://192.168.1.201:80/artifactory' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -j -u -c" + sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -j -u -c" } } } catch (exc) { diff --git a/ci/jenkinsfile/milvus_build_no_ut.groovy b/ci/jenkinsfile/milvus_build_no_ut.groovy index 4caa9f0fa7..a807ec7ad4 100644 --- a/ci/jenkinsfile/milvus_build_no_ut.groovy +++ b/ci/jenkinsfile/milvus_build_no_ut.groovy @@ -9,7 +9,7 @@ container('milvus-build-env') { sh "git config --global user.email \"test@zilliz.com\"" sh "git config --global user.name \"test\"" withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh "export JFROG_ARTFACTORY_URL='http://192.168.1.201:80/artifactory' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -j" + sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -j" } } } catch (exc) { From 455a31c8bc50bef28a9f3f4b577b13f8b8df3936 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 25 Sep 2019 19:05:08 +0800 Subject: [PATCH 095/217] MS-590 Refine cmake code to support cpplint Former-commit-id: 583d31a79b3b491c3c3bf7eda9fa613d1c6a58ff --- cpp/CMakeLists.txt | 19 ++--- cpp/build-support/lint_exclusions.txt | 6 ++ cpp/build.sh | 102 ++++++++++++++++---------- cpp/cmake/DefineOptions.cmake | 1 + 4 files changed, 80 insertions(+), 48 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index e7cf83d704..4fffe66220 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -44,13 +44,7 @@ set(MILVUS_VERSION "${GIT_BRANCH_NAME}") string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}") find_package(ClangTools) -if("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND) - # Generate a Clang compile_commands.json "compilation database" file for use - # with various development tools, such as Vim's YouCompleteMe plugin. - # See http://clang.llvm.org/docs/JSONCompilationDatabase.html - set(CMAKE_EXPORT_COMPILE_COMMANDS 1) -endif() - +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support") if(CMAKE_BUILD_TYPE STREQUAL "Release") @@ -71,7 +65,7 @@ if(MILVUS_VERSION_MAJOR STREQUAL "" OR MILVUS_VERSION_MINOR STREQUAL "" OR MILVUS_VERSION_PATCH STREQUAL "") message(WARNING "Failed to determine Milvus version from git branch name") - set(MILVUS_VERSION "0.4.0") + set(MILVUS_VERSION "0.5.0") endif() message(STATUS "Build version = ${MILVUS_VERSION}") @@ -182,6 +176,9 @@ endif() find_program(CPPLINT_BIN NAMES cpplint cpplint.py HINTS ${BUILD_SUPPORT_DIR}) message(STATUS "Found cpplint executable at ${CPPLINT_BIN}") +# +# "make lint" targets +# add_custom_target(lint ${PYTHON_EXECUTABLE} ${BUILD_SUPPORT_DIR}/run_cpplint.py @@ -194,11 +191,11 @@ add_custom_target(lint ${MILVUS_LINT_QUIET}) # -# "make format" and "make check-format" targets +# "make clang-format" and "make check-clang-format" targets # if(${CLANG_FORMAT_FOUND}) # runs clang format and updates files in place. - add_custom_target(format + add_custom_target(clang-format ${PYTHON_EXECUTABLE} ${BUILD_SUPPORT_DIR}/run_clang_format.py --clang_format_binary @@ -211,7 +208,7 @@ if(${CLANG_FORMAT_FOUND}) ${MILVUS_LINT_QUIET}) # runs clang format and exits with a non-zero exit code if any files need to be reformatted - add_custom_target(check-format + add_custom_target(check-clang-format ${PYTHON_EXECUTABLE} ${BUILD_SUPPORT_DIR}/run_clang_format.py --clang_format_binary diff --git a/cpp/build-support/lint_exclusions.txt b/cpp/build-support/lint_exclusions.txt index e69de29bb2..3ae6878c43 100644 --- a/cpp/build-support/lint_exclusions.txt +++ b/cpp/build-support/lint_exclusions.txt @@ -0,0 +1,6 @@ +*cmake-build-debug* +*cmake-build-release* +*cmake_build* +*src/thirdparty* +*src/core/thirdparty* +*src/grpc* \ No newline at end of file diff --git a/cpp/build.sh b/cpp/build.sh index 013d4d575d..4087db4592 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -8,10 +8,18 @@ BUILD_COVERAGE="OFF" DB_PATH="/opt/milvus" PROFILING="OFF" USE_JFROG_CACHE="OFF" +RUN_CPPLINT="OFF" +CUDA_COMPILER=/usr/local/cuda/bin/nvcc -while getopts "p:d:t:uhrcgj" arg +while getopts "p:d:t:ulrcgjh" arg do case $arg in + p) + INSTALL_PREFIX=$OPTARG + ;; + d) + DB_PATH=$OPTARG + ;; t) BUILD_TYPE=$OPTARG # BUILD_TYPE ;; @@ -19,11 +27,8 @@ do echo "Build and run unittest cases" ; BUILD_UNITTEST="ON"; ;; - p) - INSTALL_PREFIX=$OPTARG - ;; - d) - DB_PATH=$OPTARG + l) + RUN_CPPLINT="ON" ;; r) if [[ -d cmake_build ]]; then @@ -44,63 +49,86 @@ do echo " parameter: --t: build type(default: Debug) --u: building unit test options(default: OFF) -p: install prefix(default: $(pwd)/milvus) -d: db path(default: /opt/milvus) +-t: build type(default: Debug) +-u: building unit test options(default: OFF) +-l: run cpplint, clang-format and clang-tidy(default: OFF) -r: remove previous build directory(default: OFF) -c: code coverage(default: OFF) -g: profiling(default: OFF) --j: use jfrog cache build directory +-j: use jfrog cache build directory(default: OFF) +-h: help usage: -./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c] [-k] [-j] +./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-g] [-j] [-h] " exit 0 ;; ?) - echo "unknown argument" + echo "ERROR! unknown argument" exit 1 ;; esac done if [[ ! -d cmake_build ]]; then - mkdir cmake_build - MAKE_CLEAN="ON" + mkdir cmake_build fi cd cmake_build -CUDA_COMPILER=/usr/local/cuda/bin/nvcc +CMAKE_CMD="cmake \ +-DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ +-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} +-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ +-DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ +-DBUILD_COVERAGE=${BUILD_COVERAGE} \ +-DMILVUS_DB_PATH=${DB_PATH} \ +-DMILVUS_ENABLE_PROFILING=${PROFILING} \ +-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ +../" +echo ${CMAKE_CMD} +${CMAKE_CMD} if [[ ${MAKE_CLEAN} == "ON" ]]; then - CMAKE_CMD="cmake -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ - -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} - -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ - -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ - -DBUILD_COVERAGE=${BUILD_COVERAGE} \ - -DMILVUS_DB_PATH=${DB_PATH} \ - -DMILVUS_ENABLE_PROFILING=${PROFILING} \ - -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ - ../" - echo ${CMAKE_CMD} - - ${CMAKE_CMD} make clean fi -make -j 4 || exit 1 +if [[ ${RUN_CPPLINT} == "ON" ]]; then + # cpplint check + make lint + if [ $? -ne 0 ]; then + echo "ERROR! cpplint check not pass" + exit 1 + fi + # clang-format check + make check-clang-format + if [ $? -ne 0 ]; then + echo "ERROR! clang-format check failed" + exit 1 + fi + # clang-tidy check + make check-clang-tidy + if [ $? -ne 0 ]; then + echo "ERROR! clang-tidy check failed" + exit 1 + fi +else + # compile and build + make -j 4 || exit 1 -if [[ ${BUILD_TYPE} != "Debug" ]]; then - strip src/milvus_server -fi + # strip binary symbol + if [[ ${BUILD_TYPE} != "Debug" ]]; then + strip src/milvus_server + fi -make install || exit 1 + make install || exit 1 -if [[ ${BUILD_COVERAGE} == "ON" ]]; then - cd - - bash `pwd`/coverage.sh - cd - -fi + # evaluate code coverage + if [[ ${BUILD_COVERAGE} == "ON" ]]; then + cd - + bash `pwd`/coverage.sh + cd - + fi +fi \ No newline at end of file diff --git a/cpp/cmake/DefineOptions.cmake b/cpp/cmake/DefineOptions.cmake index 2b9a68e88e..1b0646c2fa 100644 --- a/cpp/cmake/DefineOptions.cmake +++ b/cpp/cmake/DefineOptions.cmake @@ -110,6 +110,7 @@ endif() #---------------------------------------------------------------------- set_option_category("Test and benchmark") +unset(MILVUS_BUILD_TESTS CACHE) if (BUILD_UNIT_TEST) define_option(MILVUS_BUILD_TESTS "Build the MILVUS googletest unit tests" ON) else() From ee7fb3684ad70e8263b92a7070b97cc5c7dd0e91 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 25 Sep 2019 19:16:12 +0800 Subject: [PATCH 096/217] MS-574 add config check Former-commit-id: ac1ed68cc7fcd08cde02bccbd574ac34fdbfdb97 --- cpp/CHANGELOG.md | 3 +- cpp/src/server/Config.cpp | 985 +++++++++++-------------------- cpp/src/server/Config.h | 98 +-- cpp/src/server/Server.cpp | 8 +- cpp/src/utils/ValidationUtil.cpp | 50 +- cpp/src/utils/ValidationUtil.h | 4 +- 6 files changed, 456 insertions(+), 692 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 667399cc5d..2281494fed 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -18,7 +18,8 @@ Please mark all change in change log and use the ticket from JIRA. - MS-562 - Add JobMgr and TaskCreator in Scheduler - MS-566 - Refactor cmake - MS-555 - Remove old scheduler -- MS-578 - Makesure milvus5.0 don't crack 0.3.1 data +- MS-574 - Milvus configuration refactor +- MS-578 - Make sure milvus5.0 don't crack 0.3.1 data ## New Feature diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 9d76fa72c6..b89f1e3e02 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -33,7 +33,6 @@ namespace zilliz { namespace milvus { namespace server { -constexpr uint64_t MB = 1UL << 20; constexpr uint64_t GB = 1UL << 30; Config & @@ -71,545 +70,272 @@ Config::LoadConfigFile(const std::string &filename) { return Status::OK(); } +void +Config::PrintConfigSection(const std::string& config_node_name) { + std::cout << std::endl; + std::cout << config_node_name << ":" << std::endl; + if (config_map_.find(config_node_name) != config_map_.end()) { + for (auto item: config_map_[config_node_name]) { + std::cout << item.first << ": " << item.second << std::endl; + } + } +} + +void +Config::PrintAll() { + PrintConfigSection(CONFIG_SERVER); + PrintConfigSection(CONFIG_DB); + PrintConfigSection(CONFIG_CACHE); + PrintConfigSection(CONFIG_METRIC); + PrintConfigSection(CONFIG_ENGINE); + PrintConfigSection(CONFIG_RESOURCE); +} + +//////////////////////////////////////////////////////////////////////////////// Status -Config::ValidateConfig() { - if (!CheckServerConfig().ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Server config validation check fail"); - } - if (!CheckDBConfig().ok()) { - return Status(SERVER_INVALID_ARGUMENT, "DB config validation check fail"); - } - if (!CheckMetricConfig().ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Metric config validation check fail"); - } - if (!CheckCacheConfig().ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Cache config validation check fail"); - } - if (!CheckEngineConfig().ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Engine config validation check fail"); - } - if (!CheckResourceConfig().ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Resource config validation check fail"); +Config::CheckServerConfigAddress(std::string &value) { + if (!ValidationUtil::ValidateIpAddress(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid server config address: " + value); } return Status::OK(); } Status -Config::CheckServerConfig() { -/* - server_config: - address: 0.0.0.0 # milvus server ip address (IPv4) - port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 - mode: single # milvus deployment type: single, cluster, read_only - time_zone: UTC+8 # Use the UTC-x or UTC+x to specify a time zone. eg. UTC+8 for China Standard Time - -*/ - bool okay = true; - ConfigNode server_config = GetConfigNode(CONFIG_SERVER); - - std::string ip_address = server_config.GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); - if (!ValidationUtil::ValidateIpAddress(ip_address).ok()) { - std::cerr << "ERROR: invalid server IP address: " << ip_address << std::endl; - okay = false; - } - - std::string port_str = server_config.GetValue(CONFIG_SERVER_PORT, CONFIG_SERVER_PORT_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(port_str).ok()) { - std::cerr << "ERROR: port " << port_str << " is not a number" << std::endl; - okay = false; +Config::CheckServerConfigPort(std::string &value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid server config port: " + value); } else { - int32_t port = std::stol(port_str); + int32_t port = std::stoi(value); if (!(port > 1024 && port < 65535)) { - std::cerr << "ERROR: port " << port_str << " out of range (1024, 65535)" << std::endl; - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Server config port out of range (1024, 65535): " + value); } } + return Status::OK(); +} - std::string mode = server_config.GetValue(CONFIG_SERVER_MODE, CONFIG_SERVER_MODE_DEFAULT); - if (mode != "single" && mode != "cluster" && mode != "read_only") { - std::cerr << "ERROR: mode " << mode << " is not one of ['single', 'cluster', 'read_only']" << std::endl; - okay = false; +Status +Config::CheckServerConfigMode(std::string &value) { + if (value != "single" && value != "cluster" && value != "read_only") { + return Status(SERVER_INVALID_ARGUMENT, "Invalid server config mode [single, cluster, read_only]: " + value); } + return Status::OK(); +} - std::string time_zone = server_config.GetValue(CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); - int flag = 0; - if (time_zone.length() <= 3) { - flag = 1; +Status +Config::CheckServerConfigTimeZone(std::string &value) { + if (value.length() <= 3) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid server config time_zone: " + value); } else { - if (time_zone.substr(0, 3) != "UTC") { - flag = 1; + if (value.substr(0, 3) != "UTC") { + return Status(SERVER_INVALID_ARGUMENT, "Invalid server config time_zone: " + value); } else { try { - stoi(time_zone.substr(3)); + stoi(value.substr(3)); } catch (...) { - flag = 1; + return Status(SERVER_INVALID_ARGUMENT, "Invalid server config time_zone: " + value); } } } - if (flag == 1) { - std::cerr << "ERROR: time_zone " << time_zone << " format wrong" << std::endl; - okay = false; - } - - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Illegal server config")); + return Status::OK(); } Status -Config::CheckDBConfig() { -/* - db_config: - db_path: @MILVUS_DB_PATH@ # milvus data storage path - db_slave_path: # secondry data storage path, split by semicolon - - # URI format: dialect://username:password@host:port/database - # All parts except dialect are optional, but you MUST include the delimiters - # Currently dialect supports mysql or sqlite - db_backend_url: sqlite://:@:/ - - archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB - archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day - insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB. - # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB - build_index_gpu: 0 # which gpu is used to build index, default: 0, range: 0 ~ gpu number - 1 -*/ - bool okay = true; - ConfigNode db_config = GetConfigNode(CONFIG_DB); - - std::string db_path = db_config.GetValue(CONFIG_DB_PATH); - if (db_path.empty()) { - std::cerr << "ERROR: db_path is empty" << std::endl; - okay = false; +Config::CheckDBConfigPath(const std::string &value) { + if (value.empty()) { + return Status(SERVER_INVALID_ARGUMENT, "DB config path empty"); } + return Status::OK(); +} - std::string db_backend_url = db_config.GetValue(CONFIG_DB_BACKEND_URL); - if (!ValidationUtil::ValidateDbURI(db_backend_url).ok()) { - std::cerr << "ERROR: invalid db_backend_url: " << db_backend_url << std::endl; - okay = false; +Status +Config::CheckDBConfigBackendUrl(const std::string &value) { + if (!ValidationUtil::ValidateDbURI(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config backend_url: " + value); } + return Status::OK(); +} - std::string archive_disk_threshold_str = - db_config.GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(archive_disk_threshold_str).ok()) { - std::cerr << "ERROR: archive_disk_threshold " << archive_disk_threshold_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckDBConfigArchiveDiskThreshold(const std::string &value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config archive_disk_threshold: " + value); } + return Status::OK(); +} - std::string archive_days_threshold_str = - db_config.GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(archive_days_threshold_str).ok()) { - std::cerr << "ERROR: archive_days_threshold " << archive_days_threshold_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckDBConfigArchiveDaysThreshold(const std::string &value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config archive_days_threshold: " + value); } + return Status::OK(); +} - std::string buffer_size_str = db_config.GetValue(CONFIG_DB_BUFFER_SIZE, CONFIG_DB_BUFFER_SIZE_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(buffer_size_str).ok()) { - std::cerr << "ERROR: buffer_size " << buffer_size_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckDBConfigBufferSize(const std::string &value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config buffer_size: " + value); } else { - uint64_t buffer_size = (uint64_t) std::stol(buffer_size_str); - buffer_size *= GB; + int64_t buffer_size = std::stoi(value) * GB; unsigned long total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (buffer_size >= total_mem) { - std::cerr << "ERROR: buffer_size exceed system memory" << std::endl; - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "DB config buffer_size exceed system memory: " + value); } } + return Status::OK(); +} - std::string gpu_index_str = db_config.GetValue(CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(gpu_index_str).ok()) { - std::cerr << "ERROR: gpu_index " << gpu_index_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckDBConfigBuildIndexGPU(const std::string &value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config build_index_gpu: " + value); } else { - int32_t gpu_index = std::stol(gpu_index_str); + int32_t gpu_index = std::stoi(value); if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { - std::cerr << "ERROR: invalid gpu_index " << gpu_index_str << std::endl; - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config build_index_gpu: " + value); } } - - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "DB config is illegal")); + return Status::OK(); } Status -Config::CheckMetricConfig() { -/* - metric_config: - is_startup: off # if monitoring start: on, off - collector: prometheus # metrics collector: prometheus - prometheus_config: # following are prometheus configure - port: 8080 # the port prometheus use to fetch metrics - (not used) push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address - (not used) push_gateway_port: 9091 # push method configure: push gateway port -*/ - bool okay = true; - ConfigNode metric_config = GetConfigNode(CONFIG_METRIC); - - std::string is_startup_str = - metric_config.GetValue(CONFIG_METRIC_AUTO_BOOTUP, CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); - if (!ValidationUtil::ValidateStringIsBool(is_startup_str).ok()) { - std::cerr << "ERROR: invalid is_startup config: " << is_startup_str << std::endl; - okay = false; +Config::CheckMetricConfigAutoBootup(const std::string& value) { + if (!ValidationUtil::ValidateStringIsBool(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config auto_bootup: " + value); } - - std::string - port_str = metric_config.GetChild(CONFIG_METRIC_PROMETHEUS).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, "8080"); - if (!ValidationUtil::ValidateStringIsNumber(port_str).ok()) { - std::cerr << "ERROR: port specified in prometheus_config " << port_str << " is not a number" << std::endl; - okay = false; - } - - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Metric config is illegal")); + return Status::OK(); } Status -Config::CheckCacheConfig() { -/* - cache_config: - cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory - cpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 - insert_cache_immediately: false # insert data will be load into cache immediately for hot query - gpu_cache_capacity: 5 # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory - gpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 +Config::CheckMetricConfigPrometheusPort(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config prometheus_port: " + value); + } + return Status::OK(); +} -*/ - bool okay = true; - ConfigNode cache_config = GetConfigNode(CONFIG_CACHE); - - std::string cpu_cache_capacity_str = - cache_config.GetValue(CONFIG_CACHE_CPU_MEM_CAPACITY, CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(cpu_cache_capacity_str).ok()) { - std::cerr << "ERROR: cpu_cache_capacity " << cpu_cache_capacity_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckCacheConfigCpuMemCapacity(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_mem_capacity: " + value); } else { - uint64_t cpu_cache_capacity = (uint64_t) std::stol(cpu_cache_capacity_str); - cpu_cache_capacity *= GB; + uint64_t cpu_cache_capacity = std::stoi(value) * GB; unsigned long total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (cpu_cache_capacity >= total_mem) { - std::cerr << "ERROR: cpu_cache_capacity exceed system memory" << std::endl; - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Cache config cpu_mem_capacity exceed system memory: " + value); } else if (cpu_cache_capacity > (double) total_mem * 0.9) { - std::cerr << "Warning: cpu_cache_capacity value is too aggressive" << std::endl; + std::cerr << "Warning: cpu_mem_capacity value is too big" << std::endl; } - uint64_t buffer_size = - (uint64_t) GetConfigNode(CONFIG_DB).GetInt32Value(CONFIG_DB_BUFFER_SIZE, - std::stoi(CONFIG_DB_BUFFER_SIZE_DEFAULT)); - buffer_size *= GB; - if (buffer_size + cpu_cache_capacity >= total_mem) { - std::cerr << "ERROR: sum of cpu_cache_capacity and insert_buffer_size exceed system memory" << std::endl; - okay = false; + int32_t buffer_size; + Status s = GetDBConfigBufferSize(buffer_size); + if (!s.ok()) return s; + int64_t insert_buffer_size = buffer_size * GB; + if (insert_buffer_size + cpu_cache_capacity >= total_mem) { + return Status(SERVER_INVALID_ARGUMENT, "Sum of cpu_mem_capacity and buffer_size exceed system memory"); } } + return Status::OK(); +} - std::string cpu_cache_free_percent_str = - cache_config.GetValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT); - double cpu_cache_free_percent; - if (!ValidationUtil::ValidateStringIsDouble(cpu_cache_free_percent_str, cpu_cache_free_percent).ok()) { - std::cerr << "ERROR: cpu_cache_free_percent " << cpu_cache_free_percent_str << " is not a double" << std::endl; - okay = false; - } else if (cpu_cache_free_percent < std::numeric_limits::epsilon() || cpu_cache_free_percent > 1.0) { - std::cerr << "ERROR: invalid cpu_cache_free_percent " << cpu_cache_free_percent_str << std::endl; - okay = false; - } - - std::string insert_cache_immediately_str = - cache_config.GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); - if (!ValidationUtil::ValidateStringIsBool(insert_cache_immediately_str).ok()) { - std::cerr << "ERROR: invalid insert_cache_immediately config: " << insert_cache_immediately_str << std::endl; - okay = false; - } - - std::string gpu_cache_capacity_str = - cache_config.GetValue(CONFIG_CACHE_GPU_MEM_CAPACITY, CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(gpu_cache_capacity_str).ok()) { - std::cerr << "ERROR: gpu_cache_capacity " << gpu_cache_capacity_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckCacheConfigCpuMemThreshold(const std::string& value) { + if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_mem_threshold: " + value); } else { - uint64_t gpu_cache_capacity = (uint64_t) std::stol(gpu_cache_capacity_str); - gpu_cache_capacity *= GB; - int gpu_index = GetConfigNode(CONFIG_DB).GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, - std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); + float cpu_mem_threshold = std::stof(value); + if (cpu_mem_threshold <= 0.0 || cpu_mem_threshold >= 1.0) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_mem_threshold: " + value); + } + } + return Status::OK(); +} + +Status +Config::CheckCacheConfigGpuMemCapacity(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + std::cerr << "ERROR: gpu_cache_capacity " << value << " is not a number" << std::endl; + } else { + uint64_t gpu_cache_capacity = std::stoi(value) * GB; + int gpu_index; + Status s = GetDBConfigBuildIndexGPU(gpu_index); + if (!s.ok()) return s; size_t gpu_memory; if (!ValidationUtil::GetGpuMemory(gpu_index, gpu_memory).ok()) { - std::cerr << "ERROR: could not get gpu memory for device " << gpu_index << std::endl; - okay = false; + return Status(SERVER_UNEXPECTED_ERROR, + "Fail to get GPU memory for GPU device: " + std::to_string(gpu_index)); } else if (gpu_cache_capacity >= gpu_memory) { - std::cerr << "ERROR: gpu_cache_capacity " << gpu_cache_capacity - << " exceed total gpu memory " << gpu_memory << std::endl; - okay = false; + return Status(SERVER_INVALID_ARGUMENT, + "Cache config gpu_mem_capacity exceed GPU memory: " + std::to_string(gpu_memory)); } else if (gpu_cache_capacity > (double) gpu_memory * 0.9) { - std::cerr << "Warning: gpu_cache_capacity value is too aggressive" << std::endl; + std::cerr << "Warning: gpu_mem_capacity value is too big" << std::endl; } } - - std::string gpu_cache_free_percent_str = - cache_config.GetValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT); - double gpu_cache_free_percent; - if (!ValidationUtil::ValidateStringIsDouble(gpu_cache_free_percent_str, gpu_cache_free_percent).ok()) { - std::cerr << "ERROR: gpu_cache_free_percent " << gpu_cache_free_percent_str << " is not a double" << std::endl; - okay = false; - } else if (gpu_cache_free_percent < std::numeric_limits::epsilon() || gpu_cache_free_percent > 1.0) { - std::cerr << "ERROR: invalid gpu_cache_free_percent " << gpu_cache_free_percent << std::endl; - okay = false; - } - - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Cache config is illegal")); + return Status::OK(); } Status -Config::CheckEngineConfig() { -/* - engine_config: - use_blas_threshold: 20 - omp_thread_num: 0 # how many compute threads be used by engine, 0 means use all cpu core to compute -*/ - bool okay = true; - ConfigNode engine_config = GetConfigNode(CONFIG_ENGINE); - - std::string use_blas_threshold_str = - engine_config.GetValue(CONFIG_ENGINE_BLAS_THRESHOLD, CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(use_blas_threshold_str).ok()) { - std::cerr << "ERROR: use_blas_threshold " << use_blas_threshold_str << " is not a number" << std::endl; - okay = false; - } - - std::string omp_thread_num_str = - engine_config.GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(omp_thread_num_str).ok()) { - std::cerr << "ERROR: omp_thread_num " << omp_thread_num_str << " is not a number" << std::endl; - okay = false; +Config::CheckCacheConfigGpuMemThreshold(const std::string& value) { + if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_mem_threshold: " + value); } else { - int32_t omp_thread = std::stol(omp_thread_num_str); + float gpu_mem_threshold = std::stof(value); + if (gpu_mem_threshold <= 0.0 || gpu_mem_threshold >= 1.0) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_mem_threshold: " + value); + } + } + return Status::OK(); +} + +Status +Config::CheckCacheConfigCacheInsertData(const std::string& value) { + if (!ValidationUtil::ValidateStringIsBool(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cache_insert_data: " + value); + } + return Status::OK(); +} + +Status +Config::CheckEngineConfigBlasThreshold(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config blas threshold: " + value); + } + return Status::OK(); +} + +Status +Config::CheckEngineConfigOmpThreadNum(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config omp_thread_num: " + value); + } else { + int32_t omp_thread = std::stoi(value); uint32_t sys_thread_cnt = 8; if (omp_thread > CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { - std::cerr << "ERROR: omp_thread_num " << omp_thread_num_str << " > system available thread " - << sys_thread_cnt << std::endl; - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config omp_thread_num: " + value); } } - - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Engine config is illegal")); + return Status::OK(); } Status -Config::CheckResourceConfig() { - /* - resource_config: - mode: simple - pool: - - cpu - - gpu0 - - gpu100 - */ - bool okay = true; - server::ConfigNode &config = GetConfigNode(server::CONFIG_RESOURCE); - auto mode = config.GetValue(CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); - if (mode != "simple") { - std::cerr << "ERROR: invalid resource config: mode is " << mode << std::endl; - okay = false; +Config::CheckResourceConfigMode(const std::string& value) { + if (value != "simple") { + return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config mode: " + value); } - auto pool = config.GetSequence(CONFIG_RESOURCE_POOL); - if (pool.empty()) { - std::cerr << "ERROR: invalid resource config: resources empty" << std::endl; - okay = false; - } - - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Resource config is illegal")); + return Status::OK(); } -//Status -//Config::CheckResourceConfig() { -/* - resource_config: - # resource list, length: 0~N - # please set a DISK resource and a CPU resource least, or system will not return query result. - # - # example: - # resource_name: # resource name, just using in connections below - # type: DISK # resource type, optional: DISK/CPU/GPU - # device_id: 0 - # enable_executor: false # if is enable executor, optional: true, false - - resources: - ssda: - type: DISK - device_id: 0 - enable_executor: false - - cpu: - type: CPU - device_id: 0 - enable_executor: true - - gpu0: - type: GPU - device_id: 0 - enable_executor: false - gpu_resource_num: 2 - pinned_memory: 300 - temp_memory: 300 - - # connection list, length: 0~N - # example: - # connection_name: - # speed: 100 # unit: MS/s - # endpoint: ${resource_name}===${resource_name} - connections: - io: - speed: 500 - endpoint: ssda===cpu - pcie0: - speed: 11000 - endpoint: cpu===gpu0 -*/ -// bool okay = true; -// server::ConfigNode resource_config = GetConfig(CONFIG_RESOURCE); -// if (resource_config.GetChildren().empty()) { -// std::cerr << "ERROR: no context under resource" << std::endl; -// okay = false; -// } -// -// auto resources = resource_config.GetChild(CONFIG_RESOURCES).GetChildren(); -// -// if (resources.empty()) { -// std::cerr << "no resources specified" << std::endl; -// okay = false; -// } -// -// bool resource_valid_flag = false; -// bool hasDisk = false; -// bool hasCPU = false; -// bool hasExecutor = false; -// std::set resource_list; -// for (auto &resource : resources) { -// resource_list.emplace(resource.first); -// auto &resource_conf = resource.second; -// auto type = resource_conf.GetValue(CONFIG_RESOURCE_TYPE); -// -// std::string device_id_str = resource_conf.GetValue(CONFIG_RESOURCE_DEVICE_ID, "0"); -// int32_t device_id = -1; -// if (!ValidationUtil::ValidateStringIsNumber(device_id_str).ok()) { -// std::cerr << "ERROR: device_id " << device_id_str << " is not a number" << std::endl; -// okay = false; -// } else { -// device_id = std::stol(device_id_str); -// } -// -// std::string enable_executor_str = resource_conf.GetValue(CONFIG_RESOURCE_ENABLE_EXECUTOR, "off"); -// if (!ValidationUtil::ValidateStringIsBool(enable_executor_str).ok()) { -// std::cerr << "ERROR: invalid enable_executor config: " << enable_executor_str << std::endl; -// okay = false; -// } -// -// if (type == "DISK") { -// hasDisk = true; -// } else if (type == "CPU") { -// hasCPU = true; -// if (resource_conf.GetBoolValue(CONFIG_RESOURCE_ENABLE_EXECUTOR, false)) { -// hasExecutor = true; -// } -// } -// else if (type == "GPU") { -// int build_index_gpu_index = GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, 0); -// if (device_id == build_index_gpu_index) { -// resource_valid_flag = true; -// } -// if (resource_conf.GetBoolValue(CONFIG_RESOURCE_ENABLE_EXECUTOR, false)) { -// hasExecutor = true; -// } -// std::string gpu_resource_num_str = resource_conf.GetValue(CONFIG_RESOURCE_NUM, "2"); -// if (!ValidationUtil::ValidateStringIsNumber(gpu_resource_num_str).ok()) { -// std::cerr << "ERROR: gpu_resource_num " << gpu_resource_num_str << " is not a number" << std::endl; -// okay = false; -// } -// bool mem_valid = true; -// std::string pinned_memory_str = resource_conf.GetValue(CONFIG_RESOURCE_PIN_MEMORY, "300"); -// if (!ValidationUtil::ValidateStringIsNumber(pinned_memory_str).ok()) { -// std::cerr << "ERROR: pinned_memory " << pinned_memory_str << " is not a number" << std::endl; -// okay = false; -// mem_valid = false; -// } -// std::string temp_memory_str = resource_conf.GetValue(CONFIG_RESOURCE_TEMP_MEMORY, "300"); -// if (!ValidationUtil::ValidateStringIsNumber(temp_memory_str).ok()) { -// std::cerr << "ERROR: temp_memory " << temp_memory_str << " is not a number" << std::endl; -// okay = false; -// mem_valid = false; -// } -// if (mem_valid) { -// size_t gpu_memory; -// if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) { -// std::cerr << "ERROR: could not get gpu memory for device " << device_id << std::endl; -// okay = false; -// } -// else { -// size_t prealoc_mem = std::stol(pinned_memory_str) + std::stol(temp_memory_str); -// if (prealoc_mem >= gpu_memory) { -// std::cerr << "ERROR: sum of pinned_memory and temp_memory " << prealoc_mem -// << " exceeds total gpu memory " << gpu_memory << " for device " << device_id << std::endl; -// okay = false; -// } -// } -// } -// } -// } -// -// if (!resource_valid_flag) { -// std::cerr << "Building index GPU can't be found in resource config." << std::endl; -// okay = false; -// } -// if (!hasDisk || !hasCPU) { -// std::cerr << "No DISK or CPU resource" << std::endl; -// okay = false; -// } -// if (!hasExecutor) { -// std::cerr << "No CPU or GPU resource has executor enabled" << std::endl; -// okay = false; -// } -// -// auto connections = resource_config.GetChild(CONFIG_RESOURCE_CONNECTIONS).GetChildren(); -// for (auto &connection : connections) { -// auto &connection_conf = connection.second; -// -// std::string speed_str = connection_conf.GetValue(CONFIG_SPEED_CONNECTIONS); -// if (ValidationUtil::ValidateStringIsNumber(speed_str) != SERVER_SUCCESS) { -// std::cerr << "ERROR: speed " << speed_str << " is not a number" << std::endl; -// okay = false; -// } -// -// std::string endpoint_str = connection_conf.GetValue(CONFIG_ENDPOINT_CONNECTIONS); -// std::string delimiter = "==="; -// auto delimiter_pos = endpoint_str.find(delimiter); -// if (delimiter_pos == std::string::npos) { -// std::cerr << "ERROR: invalid endpoint format: " << endpoint_str << std::endl; -// okay = false; -// } else { -// std::string left_resource = endpoint_str.substr(0, delimiter_pos); -// if (resource_list.find(left_resource) == resource_list.end()) { -// std::cerr << "ERROR: left resource " << left_resource << " does not exist" << std::endl; -// okay = false; -// } -// std::string right_resource = endpoint_str.substr(delimiter_pos + delimiter.length(), endpoint_str.length()); -// if (resource_list.find(right_resource) == resource_list.end()) { -// std::cerr << "ERROR: right resource " << right_resource << " does not exist" << std::endl; -// okay = false; -// } -// } -// } -// -// return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Resource config is illegal")); -//} - -void -Config::PrintAll() const { - if (const ConfigMgr *mgr = ConfigMgr::GetInstance()) { - std::string str = mgr->DumpString(); -// SERVER_LOG_INFO << "\n" << str; - std::cout << "\n" << str << std::endl; +Status +Config::CheckResourceConfigPool(const std::vector& value) { + if (value.empty()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config pool"); } + return Status::OK(); } +//////////////////////////////////////////////////////////////////////////////// ConfigNode & Config::GetConfigNode(const std::string &name) { ConfigMgr *mgr = ConfigMgr::GetInstance(); @@ -631,310 +357,316 @@ Config::GetConfigValueInMem(const std::string &parent_key, } } -Status +void Config::SetConfigValueInMem(const std::string &parent_key, const std::string &child_key, std::string &value) { std::lock_guard lock(mutex_); config_map_[parent_key][child_key] = value; - return Status::OK(); } //////////////////////////////////////////////////////////////////////////////// /* server config */ -Status -Config::GetServerConfigStrAddress(std::string& value) { - if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetServerConfigStrAddress() { + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value).ok()) { value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); - return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); } + return value; } -Status -Config::GetServerConfigStrPort(std::string& value) { - if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetServerConfigStrPort() { + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value).ok()) { value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_PORT, CONFIG_SERVER_PORT_DEFAULT); - return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); } + return value; } -Status -Config::GetServerConfigStrMode(std::string& value) { - if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetServerConfigStrMode() { + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value).ok()) { value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_MODE, CONFIG_SERVER_MODE_DEFAULT); - return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); } + return value; } -Status -Config::GetServerConfigStrTimeZone(std::string& value) { - if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetServerConfigStrTimeZone() { + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value).ok()) { value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); - return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); } + return value; } //////////////////////////////////////////////////////////////////////////////// /* db config */ -Status -Config::GetDBConfigStrPath(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrPath() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PATH, CONFIG_DB_PATH_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); } + return value; } -Status -Config::GetDBConfigStrSlavePath(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrSlavePath() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SLAVE_PATH, CONFIG_DB_SLAVE_PATH_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); } + return value; } -Status -Config::GetDBConfigStrBackendUrl(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrBackendUrl() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BACKEND_URL, CONFIG_DB_BACKEND_URL_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); } + return value; } -Status -Config::GetDBConfigStrArchiveDiskThreshold(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrArchiveDiskThreshold() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); } + return value; } -Status -Config::GetDBConfigStrArchiveDaysThreshold(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrArchiveDaysThreshold() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); } + return value; } -Status -Config::GetDBConfigStrBufferSize(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrBufferSize() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUFFER_SIZE, CONFIG_DB_BUFFER_SIZE_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); } + return value; } -Status -Config::GetDBConfigStrBuildIndexGPU(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrBuildIndexGPU() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); } + return value; } //////////////////////////////////////////////////////////////////////////////// /* metric config */ -Status -Config::GetMetricConfigStrAutoBootup(std::string& value) { - if (GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetMetricConfigStrAutoBootup() { + std::string value; + if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value).ok()) { value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_AUTO_BOOTUP, CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); - return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value); } + return value; } -Status -Config::GetMetricConfigStrCollector(std::string& value) { - if (GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetMetricConfigStrCollector() { + std::string value; + if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value).ok()) { value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_COLLECTOR, CONFIG_METRIC_COLLECTOR_DEFAULT); - return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); } + return value; } -Status -Config::GetMetricConfigStrPrometheusPort(std::string& value) { - if (GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetMetricConfigStrPrometheusPort() { + std::string value; + if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value).ok()) { value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); - return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); } + return value; } //////////////////////////////////////////////////////////////////////////////// /* cache config */ -Status -Config::GetCacheConfigStrCpuMemCapacity(std::string& value) { - if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetCacheConfigStrCpuMemCapacity() { + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value).ok()) { value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_CAPACITY, CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); - return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value); } + return value; } -Status -Config::GetCacheConfigStrCpuMemThreshold(std::string& value) { - if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetCacheConfigStrCpuMemThreshold() { + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value).ok()) { value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT); - return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value); } + return value; } -Status -Config::GetCacheConfigStrGpuMemCapacity(std::string& value) { - if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetCacheConfigStrGpuMemCapacity() { + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value).ok()) { value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_CAPACITY, CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT); - return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value); } + return value; } -Status -Config::GetCacheConfigStrGpuMemThreshold(std::string& value) { - if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetCacheConfigStrGpuMemThreshold() { + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value).ok()) { value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT); - return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value); } + return value; } -Status -Config::GetCacheConfigStrCacheInsertData(std::string& value) { - if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetCacheConfigStrCacheInsertData() { + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value).ok()) { value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); - return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); } + return value; } //////////////////////////////////////////////////////////////////////////////// /* engine config */ -Status -Config::GetEngineConfigStrBlasThreshold(std::string& value) { - if (GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetEngineConfigStrBlasThreshold() { + std::string value; + if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value).ok()) { value = GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_BLAS_THRESHOLD, CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); - return SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value); + SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value); } + return value; } -Status -Config::GetEngineConfigStrOmpThreadNum(std::string& value) { - if (GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetEngineConfigStrOmpThreadNum() { + std::string value; + if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value).ok()) { value = GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); - return SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); + SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); } + return value; } //////////////////////////////////////////////////////////////////////////////// /* resource config */ -Status -Config::GetResourceConfigStrMode(std::string& value) { - if (GetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetResourceConfigStrMode() { + std::string value; + if (!GetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value).ok()) { value = GetConfigNode(CONFIG_RESOURCE).GetValue(CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); - return SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); + SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); } + return value; } //////////////////////////////////////////////////////////////////////////////// Status Config::GetServerConfigAddress(std::string& value) { - return GetServerConfigStrAddress(value); + value = GetServerConfigStrAddress(); + return CheckServerConfigAddress(value); } Status Config::GetServerConfigPort(std::string& value) { - return GetServerConfigStrPort(value); + value = GetServerConfigStrPort(); + return CheckServerConfigPort(value); } Status Config::GetServerConfigMode(std::string& value) { - return GetServerConfigStrMode(value); + value = GetServerConfigStrMode(); + return CheckServerConfigMode(value); } Status Config::GetServerConfigTimeZone(std::string& value) { - return GetServerConfigStrTimeZone(value); + value = GetServerConfigStrTimeZone(); + return CheckServerConfigTimeZone(value); } Status Config::GetDBConfigPath(std::string& value) { - return GetDBConfigStrPath(value); + value = GetDBConfigStrPath(); + return CheckDBConfigPath(value); } Status Config::GetDBConfigSlavePath(std::string& value) { - return GetDBConfigStrSlavePath(value); + value = GetDBConfigStrSlavePath(); + return Status::OK(); } Status Config::GetDBConfigBackendUrl(std::string& value) { - return GetDBConfigStrBackendUrl(value); + value = GetDBConfigStrBackendUrl(); + return CheckDBConfigBackendUrl(value); } Status Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { - std::string str; - Status s = GetDBConfigStrArchiveDiskThreshold(str); + std::string str = GetDBConfigStrArchiveDiskThreshold(); + Status s = CheckDBConfigArchiveDiskThreshold(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -942,8 +674,8 @@ Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { Status Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { - std::string str; - Status s = GetDBConfigStrArchiveDaysThreshold(str); + std::string str = GetDBConfigStrArchiveDaysThreshold(); + Status s = CheckDBConfigArchiveDaysThreshold(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -951,8 +683,8 @@ Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { Status Config::GetDBConfigBufferSize(int32_t& value) { - std::string str; - Status s = GetDBConfigStrBufferSize(str); + std::string str = GetDBConfigStrBufferSize(); + Status s = CheckDBConfigBufferSize(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -960,8 +692,8 @@ Config::GetDBConfigBufferSize(int32_t& value) { Status Config::GetDBConfigBuildIndexGPU(int32_t& value) { - std::string str; - Status s = GetDBConfigStrBuildIndexGPU(str); + std::string str = GetDBConfigStrBuildIndexGPU(); + Status s = CheckDBConfigBuildIndexGPU(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -969,8 +701,8 @@ Config::GetDBConfigBuildIndexGPU(int32_t& value) { Status Config::GetMetricConfigAutoBootup(bool& value) { - std::string str; - Status s = GetMetricConfigStrAutoBootup(str); + std::string str = GetMetricConfigStrAutoBootup(); + Status s = CheckMetricConfigPrometheusPort(str); if (!s.ok()) return s; std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); @@ -979,18 +711,20 @@ Config::GetMetricConfigAutoBootup(bool& value) { Status Config::GetMetricConfigCollector(std::string& value) { - return GetMetricConfigStrCollector(value); + value = GetMetricConfigStrCollector(); + return Status::OK(); } Status Config::GetMetricConfigPrometheusPort(std::string& value) { - return GetMetricConfigStrPrometheusPort(value); + value = GetMetricConfigStrPrometheusPort(); + return CheckMetricConfigPrometheusPort(value); } Status Config::GetCacheConfigCpuMemCapacity(int32_t& value) { - std::string str; - Status s = GetCacheConfigStrCpuMemCapacity(str); + std::string str = GetCacheConfigStrCpuMemCapacity(); + Status s = CheckCacheConfigCpuMemCapacity(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -998,8 +732,8 @@ Config::GetCacheConfigCpuMemCapacity(int32_t& value) { Status Config::GetCacheConfigCpuMemThreshold(float& value) { - std::string str; - Status s = GetCacheConfigStrCpuMemThreshold(str); + std::string str = GetCacheConfigStrCpuMemThreshold(); + Status s = CheckCacheConfigCpuMemThreshold(str); if (!s.ok()) return s; value = std::stof(str); return Status::OK(); @@ -1007,8 +741,8 @@ Config::GetCacheConfigCpuMemThreshold(float& value) { Status Config::GetCacheConfigGpuMemCapacity(int32_t& value) { - std::string str; - Status s = GetCacheConfigStrGpuMemCapacity(str); + std::string str = GetCacheConfigStrGpuMemCapacity(); + Status s = CheckCacheConfigGpuMemCapacity(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -1016,8 +750,8 @@ Config::GetCacheConfigGpuMemCapacity(int32_t& value) { Status Config::GetCacheConfigGpuMemThreshold(float& value) { - std::string str; - Status s = GetCacheConfigStrGpuMemThreshold(str); + std::string str = GetCacheConfigStrGpuMemThreshold(); + Status s = CheckCacheConfigGpuMemThreshold(str); if (!s.ok()) return s; value = std::stof(str); return Status::OK(); @@ -1025,8 +759,8 @@ Config::GetCacheConfigGpuMemThreshold(float& value) { Status Config::GetCacheConfigCacheInsertData(bool& value) { - std::string str; - Status s = GetCacheConfigStrCacheInsertData(str); + std::string str = GetCacheConfigStrCacheInsertData(); + Status s = CheckCacheConfigCacheInsertData(str); if (!s.ok()) return s; std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); @@ -1035,8 +769,8 @@ Config::GetCacheConfigCacheInsertData(bool& value) { Status Config::GetEngineConfigBlasThreshold(int32_t& value) { - std::string str; - Status s = GetEngineConfigStrBlasThreshold(str); + std::string str = GetEngineConfigStrBlasThreshold(); + Status s = CheckEngineConfigBlasThreshold(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -1044,8 +778,8 @@ Config::GetEngineConfigBlasThreshold(int32_t& value) { Status Config::GetEngineConfigOmpThreadNum(int32_t& value) { - std::string str; - Status s = GetEngineConfigStrOmpThreadNum(str); + std::string str = GetEngineConfigStrOmpThreadNum(); + Status s = CheckEngineConfigOmpThreadNum(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -1053,14 +787,15 @@ Config::GetEngineConfigOmpThreadNum(int32_t& value) { Status Config::GetResourceConfigMode(std::string& value) { - return GetResourceConfigStrMode(value); + value = GetResourceConfigStrMode(); + return CheckResourceConfigMode(value); } Status Config::GetResourceConfigPool(std::vector& value) { ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE); value = resource_config.GetSequence(CONFIG_RESOURCE_POOL); - return Status::OK(); + return CheckResourceConfigPool(value); } } diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index 3eeda32546..911ec67462 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -95,64 +95,92 @@ static const char* CONFIG_RESOURCE_POOL = "pool"; class Config { public: - static Config &GetInstance(); - + static Config& GetInstance(); Status LoadConfigFile(const std::string& filename); - Status ValidateConfig(); - void PrintAll() const; + void PrintAll(); private: ConfigNode& GetConfigNode(const std::string& name); - Status CheckServerConfig(); - Status CheckDBConfig(); - Status CheckMetricConfig(); - Status CheckCacheConfig(); - Status CheckEngineConfig(); - Status CheckResourceConfig(); - Status GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value); - Status SetConfigValueInMem(const std::string& parent_key, + void SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value); - private: + void PrintConfigSection(const std::string& config_node_name); + + /////////////////////////////////////////////////////////////////////////// /* server config */ - Status GetServerConfigStrAddress(std::string& value); - Status GetServerConfigStrPort(std::string& value); - Status GetServerConfigStrMode(std::string& value); - Status GetServerConfigStrTimeZone(std::string& value); + Status CheckServerConfigAddress(std::string& value); + Status CheckServerConfigPort(std::string& value); + Status CheckServerConfigMode(std::string& value); + Status CheckServerConfigTimeZone(std::string& value); /* db config */ - Status GetDBConfigStrPath(std::string& value); - Status GetDBConfigStrSlavePath(std::string& value); - Status GetDBConfigStrBackendUrl(std::string& value); - Status GetDBConfigStrArchiveDiskThreshold(std::string& value); - Status GetDBConfigStrArchiveDaysThreshold(std::string& value); - Status GetDBConfigStrBufferSize(std::string& value); - Status GetDBConfigStrBuildIndexGPU(std::string& value); + Status CheckDBConfigPath(const std::string& value); + Status CheckDBConfigSlavePath(const std::string& value); + Status CheckDBConfigBackendUrl(const std::string& value); + Status CheckDBConfigArchiveDiskThreshold(const std::string& value); + Status CheckDBConfigArchiveDaysThreshold(const std::string& value); + Status CheckDBConfigBufferSize(const std::string& value); + Status CheckDBConfigBuildIndexGPU(const std::string& value); /* metric config */ - Status GetMetricConfigStrAutoBootup(std::string& value); - Status GetMetricConfigStrCollector(std::string& value); - Status GetMetricConfigStrPrometheusPort(std::string& value); + Status CheckMetricConfigAutoBootup(const std::string& value); + Status CheckMetricConfigPrometheusPort(const std::string& value); /* cache config */ - Status GetCacheConfigStrCpuMemCapacity(std::string& value); - Status GetCacheConfigStrCpuMemThreshold(std::string& value); - Status GetCacheConfigStrGpuMemCapacity(std::string& value); - Status GetCacheConfigStrGpuMemThreshold(std::string& value); - Status GetCacheConfigStrCacheInsertData(std::string& value); + Status CheckCacheConfigCpuMemCapacity(const std::string& value); + Status CheckCacheConfigCpuMemThreshold(const std::string& value); + Status CheckCacheConfigGpuMemCapacity(const std::string& value); + Status CheckCacheConfigGpuMemThreshold(const std::string& value); + Status CheckCacheConfigCacheInsertData(const std::string& value); /* engine config */ - Status GetEngineConfigStrBlasThreshold(std::string& value); - Status GetEngineConfigStrOmpThreadNum(std::string& value); + Status CheckEngineConfigBlasThreshold(const std::string& value); + Status CheckEngineConfigOmpThreadNum(const std::string& value); /* resource config */ - Status GetResourceConfigStrMode(std::string& value); + Status CheckResourceConfigMode(const std::string& value); + Status CheckResourceConfigPool(const std::vector& value); + + /////////////////////////////////////////////////////////////////////////// + /* server config */ + std::string GetServerConfigStrAddress(); + std::string GetServerConfigStrPort(); + std::string GetServerConfigStrMode(); + std::string GetServerConfigStrTimeZone(); + + /* db config */ + std::string GetDBConfigStrPath(); + std::string GetDBConfigStrSlavePath(); + std::string GetDBConfigStrBackendUrl(); + std::string GetDBConfigStrArchiveDiskThreshold(); + std::string GetDBConfigStrArchiveDaysThreshold(); + std::string GetDBConfigStrBufferSize(); + std::string GetDBConfigStrBuildIndexGPU(); + + /* metric config */ + std::string GetMetricConfigStrAutoBootup(); + std::string GetMetricConfigStrCollector(); + std::string GetMetricConfigStrPrometheusPort(); + + /* cache config */ + std::string GetCacheConfigStrCpuMemCapacity(); + std::string GetCacheConfigStrCpuMemThreshold(); + std::string GetCacheConfigStrGpuMemCapacity(); + std::string GetCacheConfigStrGpuMemThreshold(); + std::string GetCacheConfigStrCacheInsertData(); + + /* engine config */ + std::string GetEngineConfigStrBlasThreshold(); + std::string GetEngineConfigStrOmpThreadNum(); + + /* resource config */ + std::string GetResourceConfigStrMode(); public: /* server config */ diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index d926ca5dac..355f7903f2 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -235,14 +235,12 @@ Server::Stop() { ErrorCode Server::LoadConfig() { - Config& server_config = Config::GetInstance(); - server_config.LoadConfigFile(config_filename_); - auto status = server_config.ValidateConfig(); - if (!status.ok()) { + Config& config = Config::GetInstance(); + Status s = config.LoadConfigFile(config_filename_); + if (!s.ok()) { std::cerr << "Failed to load config file: " << config_filename_ << std::endl; exit(0); } - return SERVER_SUCCESS; } diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index c993a4ae14..c9ec2d046b 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -206,38 +206,40 @@ ValidationUtil::ValidateIpAddress(const std::string &ip_address) { } Status -ValidationUtil::ValidateStringIsNumber(const std::string &string) { - if (!string.empty() && std::all_of(string.begin(), string.end(), ::isdigit)) { +ValidationUtil::ValidateStringIsNumber(const std::string &str) { + if (str.empty() || !std::all_of(str.begin(), str.end(), ::isdigit)) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid number"); + } + try { + int32_t value = std::stoi(str); + } catch(...) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid number"); + } + return Status::OK(); +} + +Status +ValidationUtil::ValidateStringIsBool(const std::string &str) { + std::string s = str; + std::transform(s.begin(), s.end(), s.begin(), ::tolower); + if (s == "true" || s == "on" || s == "yes" || s == "1" || + s == "false" || s == "off" || s == "no" || s == "0" || + s.empty()) { return Status::OK(); } else { - return Status(SERVER_INVALID_ARGUMENT, "Not a number"); + return Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str); } } Status -ValidationUtil::ValidateStringIsBool(std::string &str) { - std::transform(str.begin(), str.end(), str.begin(), ::tolower); - if (str == "true" || str == "on" || str == "yes" || str == "1" || - str == "false" || str == "off" || str == "no" || str == "0" || - str.empty()) { - return Status::OK(); - } - else { - return Status(SERVER_INVALID_ARGUMENT, "Not a boolean: " + str); - } -} - -Status -ValidationUtil::ValidateStringIsDouble(const std::string &str, double &val) { - char *end = nullptr; - val = std::strtod(str.c_str(), &end); - if (end != str.c_str() && *end == '\0' && val != HUGE_VAL) { - return Status::OK(); - } - else { - return Status(SERVER_INVALID_ARGUMENT, "Not a double value: " + str); +ValidationUtil::ValidateStringIsFloat(const std::string &str) { + try { + float val = std::stof(str); + } catch(...) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid float: " + str); } + return Status::OK(); } Status diff --git a/cpp/src/utils/ValidationUtil.h b/cpp/src/utils/ValidationUtil.h index ad8ba5fee9..789da0de40 100644 --- a/cpp/src/utils/ValidationUtil.h +++ b/cpp/src/utils/ValidationUtil.h @@ -67,10 +67,10 @@ public: ValidateStringIsNumber(const std::string &str); static Status - ValidateStringIsBool(std::string &str); + ValidateStringIsBool(const std::string &str); static Status - ValidateStringIsDouble(const std::string &str, double &val); + ValidateStringIsFloat(const std::string &str); static Status ValidateDbURI(const std::string &uri); From 2e26d28a12e7bdfb8be84403e890d789ca32724f Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 25 Sep 2019 19:24:37 +0800 Subject: [PATCH 097/217] MS-590 Refine cmake code to support cpplint Former-commit-id: 352431bc44645574c96d28b5c46966ed85a69288 --- cpp/CHANGELOG.md | 1 + cpp/README.md | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index d2a25d18f2..76e6ec2f3b 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -30,6 +30,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-569 - Complete the NOTICE.md - MS-575 - Add Clang-format & Clang-tidy & Cpplint - MS-586 - Remove BUILD_FAISS_WITH_MKL option +- MS-590 - Refine cmake code to support cpplint # Milvus 0.4.0 (2019-09-12) diff --git a/cpp/README.md b/cpp/README.md index 5c7a1d280c..7c3d28e580 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -25,7 +25,7 @@ Keep up-to-date with newest releases and latest updates by reading [Releases](ht The data is stored and computed on a distributed architecture. This lets you scale data sizes up and down without redesigning the system. ## Architecture -![Milvus_arch](https://www.milvus-io/docs/master/assets/milvus_arch.png) +![Milvus_arch](https://milvus.io/docs/assets/milvus_arch.png) ## Get started @@ -44,10 +44,12 @@ Use Docker to install Milvus is a breeze. See the [Milvus install guide](https:/ ```shell # Install tools Centos7 : -$ yum install gfortran qt4 flex bison mysql-devel mysql +$ yum install gfortran qt4 flex bison +$ yum install mysql-devel mysql Ubuntu16.04 : -$ sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev mysql-client +$ sudo apt-get install gfortran qt4-qmake flex bison +$ sudo apt-get install libmysqlclient-dev mysql-client ``` @@ -96,12 +98,21 @@ please reinstall CMake with curl: $ sudo make install ``` +##### code format and linting + +```shell +CentOS 7: +$ yum install clang +Ubuntu 16.04: +$ sudo apt-get install clang-format clang-tidy + +$ ./build.sh -l +``` + ##### Run unit test ```shell $ ./build.sh -u -or -$ ./build.sh --unittest ``` ##### Run code coverage From 91daaf7e7e2faed9af9aa574c91fda838eaf545c Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 25 Sep 2019 19:49:22 +0800 Subject: [PATCH 098/217] MS-574 add config set APIs Former-commit-id: e5465a07270b4ffbdf9dcacfd9912ecfb88c528c --- cpp/src/server/Config.cpp | 208 ++++++++++++++++++++++++++++++++++++-- cpp/src/server/Config.h | 46 ++++++++- 2 files changed, 243 insertions(+), 11 deletions(-) diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index b89f1e3e02..8ee2f7c62b 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -93,7 +93,7 @@ Config::PrintAll() { //////////////////////////////////////////////////////////////////////////////// Status -Config::CheckServerConfigAddress(std::string &value) { +Config::CheckServerConfigAddress(const std::string &value) { if (!ValidationUtil::ValidateIpAddress(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid server config address: " + value); } @@ -101,7 +101,7 @@ Config::CheckServerConfigAddress(std::string &value) { } Status -Config::CheckServerConfigPort(std::string &value) { +Config::CheckServerConfigPort(const std::string &value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid server config port: " + value); } else { @@ -114,7 +114,7 @@ Config::CheckServerConfigPort(std::string &value) { } Status -Config::CheckServerConfigMode(std::string &value) { +Config::CheckServerConfigMode(const std::string &value) { if (value != "single" && value != "cluster" && value != "read_only") { return Status(SERVER_INVALID_ARGUMENT, "Invalid server config mode [single, cluster, read_only]: " + value); } @@ -122,7 +122,7 @@ Config::CheckServerConfigMode(std::string &value) { } Status -Config::CheckServerConfigTimeZone(std::string &value) { +Config::CheckServerConfigTimeZone(const std::string &value) { if (value.length() <= 3) { return Status(SERVER_INVALID_ARGUMENT, "Invalid server config time_zone: " + value); } else { @@ -147,6 +147,11 @@ Config::CheckDBConfigPath(const std::string &value) { return Status::OK(); } +Status +Config::CheckDBConfigSlavePath(const std::string &value) { + return Status::OK(); +} + Status Config::CheckDBConfigBackendUrl(const std::string &value) { if (!ValidationUtil::ValidateDbURI(value).ok()) { @@ -207,6 +212,14 @@ Config::CheckMetricConfigAutoBootup(const std::string& value) { return Status::OK(); } +Status +Config::CheckMetricConfigCollector(const std::string& value) { + if (value != "prometheus") { + return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config collector: " + value); + } + return Status::OK(); +} + Status Config::CheckMetricConfigPrometheusPort(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { @@ -347,9 +360,9 @@ Status Config::GetConfigValueInMem(const std::string &parent_key, const std::string &child_key, std::string &value) { + std::lock_guard lock(mutex_); if (config_map_.find(parent_key) != config_map_.end() && config_map_[parent_key].find(child_key) != config_map_[parent_key].end()) { - std::lock_guard lock(mutex_); value = config_map_[parent_key][child_key]; return Status::OK(); } else { @@ -360,7 +373,7 @@ Config::GetConfigValueInMem(const std::string &parent_key, void Config::SetConfigValueInMem(const std::string &parent_key, const std::string &child_key, - std::string &value) { + const std::string &value) { std::lock_guard lock(mutex_); config_map_[parent_key][child_key] = value; } @@ -798,6 +811,189 @@ Config::GetResourceConfigPool(std::vector& value) { return CheckResourceConfigPool(value); } +/////////////////////////////////////////////////////////////////////////////// +/* server config */ +Status +Config::SetServerConfigAddress(const std::string& value) { + Status s = CheckServerConfigAddress(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); + return Status::OK(); +} + +Status +Config::SetServerConfigPort(const std::string& value) { + Status s = CheckServerConfigPort(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); + return Status::OK(); +} + +Status +Config::SetServerConfigMode(const std::string& value) { + Status s = CheckServerConfigMode(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); + return Status::OK(); +} + +Status +Config::SetServerConfigTimeZone(const std::string& value) { + Status s = CheckServerConfigTimeZone(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); + return Status::OK(); +} + +/* db config */ +Status +Config::SetDBConfigPath(const std::string& value) { + Status s = CheckDBConfigPath(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); + return Status::OK(); +} + +Status +Config::SetDBConfigSlavePath(const std::string& value) { + Status s = CheckDBConfigSlavePath(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); + return Status::OK(); +} + +Status +Config::SetDBConfigBackendUrl(const std::string& value) { + Status s = CheckDBConfigBackendUrl(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); + return Status::OK(); +} + +Status +Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { + Status s = CheckDBConfigArchiveDiskThreshold(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); + return Status::OK(); +} + +Status +Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { + Status s = CheckDBConfigArchiveDaysThreshold(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); + return Status::OK(); +} + +Status +Config::SetDBConfigBufferSize(const std::string& value) { + Status s = CheckDBConfigBufferSize(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); + return Status::OK(); +} + +Status +Config::SetDBConfigBuildIndexGPU(const std::string& value) { + Status s = CheckDBConfigBuildIndexGPU(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); + return Status::OK(); +} + +/* metric config */ +Status +Config::SetMetricConfigAutoBootup(const std::string& value) { + Status s = CheckMetricConfigAutoBootup(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_AUTO_BOOTUP, value); + return Status::OK(); +} + +Status +Config::SetMetricConfigCollector(const std::string& value) { + Status s = CheckMetricConfigCollector(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_COLLECTOR, value); + return Status::OK(); +} + +Status +Config::SetMetricConfigPrometheusPort(const std::string& value) { + Status s = CheckMetricConfigPrometheusPort(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_PROMETHEUS_PORT, value); + return Status::OK(); +} + +/* cache config */ +Status +Config::SetCacheConfigCpuMemCapacity(const std::string& value) { + Status s = CheckCacheConfigCpuMemCapacity(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_MEM_CAPACITY, value); + return Status::OK(); +} + +Status +Config::SetCacheConfigCpuMemThreshold(const std::string& value) { + Status s = CheckCacheConfigCpuMemThreshold(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_MEM_THRESHOLD, value); + return Status::OK(); +} + +Status +Config::SetCacheConfigGpuMemCapacity(const std::string& value) { + Status s = CheckCacheConfigGpuMemCapacity(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_MEM_CAPACITY, value); + return Status::OK(); +} + +Status +Config::SetCacheConfigGpuMemThreshold(const std::string& value) { + Status s = CheckCacheConfigGpuMemThreshold(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_MEM_THRESHOLD, value); + return Status::OK(); +} + +Status +Config::SetCacheConfigCacheInsertData(const std::string& value) { + Status s = CheckCacheConfigCacheInsertData(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CACHE_INSERT_DATA, value); + return Status::OK(); +} + +/* engine config */ +Status +Config::SetEngineConfigBlasThreshold(const std::string& value) { + Status s = CheckEngineConfigBlasThreshold(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_BLAS_THRESHOLD, value); + return Status::OK(); +} + +Status +Config::SetEngineConfigOmpThreadNum(const std::string& value) { + Status s = CheckEngineConfigOmpThreadNum(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_OMP_THREAD_NUM, value); + return Status::OK(); +} + +/* resource config */ +Status +Config::SetResourceConfigMode(const std::string& value) { + Status s = CheckResourceConfigMode(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_MODE, value); + return Status::OK(); +} + } } } diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index 911ec67462..956c08100f 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -108,16 +108,16 @@ class Config { void SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, - std::string& value); + const std::string& value); void PrintConfigSection(const std::string& config_node_name); /////////////////////////////////////////////////////////////////////////// /* server config */ - Status CheckServerConfigAddress(std::string& value); - Status CheckServerConfigPort(std::string& value); - Status CheckServerConfigMode(std::string& value); - Status CheckServerConfigTimeZone(std::string& value); + Status CheckServerConfigAddress(const std::string& value); + Status CheckServerConfigPort(const std::string& value); + Status CheckServerConfigMode(const std::string& value); + Status CheckServerConfigTimeZone(const std::string& value); /* db config */ Status CheckDBConfigPath(const std::string& value); @@ -130,6 +130,7 @@ class Config { /* metric config */ Status CheckMetricConfigAutoBootup(const std::string& value); + Status CheckMetricConfigCollector(const std::string& value); Status CheckMetricConfigPrometheusPort(const std::string& value); /* cache config */ @@ -218,6 +219,41 @@ class Config { Status GetResourceConfigMode(std::string& value); Status GetResourceConfigPool(std::vector& value); + public: + /* server config */ + Status SetServerConfigAddress(const std::string& value); + Status SetServerConfigPort(const std::string& value); + Status SetServerConfigMode(const std::string& value); + Status SetServerConfigTimeZone(const std::string& value); + + /* db config */ + Status SetDBConfigPath(const std::string& value); + Status SetDBConfigSlavePath(const std::string& value); + Status SetDBConfigBackendUrl(const std::string& value); + Status SetDBConfigArchiveDiskThreshold(const std::string& value); + Status SetDBConfigArchiveDaysThreshold(const std::string& value); + Status SetDBConfigBufferSize(const std::string& value); + Status SetDBConfigBuildIndexGPU(const std::string& value); + + /* metric config */ + Status SetMetricConfigAutoBootup(const std::string& value); + Status SetMetricConfigCollector(const std::string& value); + Status SetMetricConfigPrometheusPort(const std::string& value); + + /* cache config */ + Status SetCacheConfigCpuMemCapacity(const std::string& value); + Status SetCacheConfigCpuMemThreshold(const std::string& value); + Status SetCacheConfigGpuMemCapacity(const std::string& value); + Status SetCacheConfigGpuMemThreshold(const std::string& value); + Status SetCacheConfigCacheInsertData(const std::string& value); + + /* engine config */ + Status SetEngineConfigBlasThreshold(const std::string& value); + Status SetEngineConfigOmpThreadNum(const std::string& value); + + /* resource config */ + Status SetResourceConfigMode(const std::string& value); + private: std::unordered_map> config_map_; std::mutex mutex_; From e7a6cff566e677d6ba3f1a2f56e939170ab458eb Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 25 Sep 2019 19:54:30 +0800 Subject: [PATCH 099/217] add JFROG_ARTFACTORY_URL parameter in CMakeLists.txt Former-commit-id: 20150e9cabff924b2fc83757a0faee14e26998d4 --- cpp/src/core/cmake/ThirdPartyPackagesCore.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake b/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake index 7b0c75633d..63d991a1e5 100644 --- a/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake +++ b/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake @@ -123,6 +123,7 @@ if(NOT DEFINED USE_JFROG_CACHE) set(USE_JFROG_CACHE "OFF") endif() if(USE_JFROG_CACHE STREQUAL "ON") + set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/generic-local/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${KNOWHERE_BUILD_ARCH}/${BUILD_TYPE}") set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache") endif() From 681fbe50a28cd1aa399c19fe087feff02afb0e54 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 25 Sep 2019 20:14:55 +0800 Subject: [PATCH 100/217] excluse some files Former-commit-id: 4c8381a632dbcbbe3fe7888e3ecef8caad402477 --- cpp/build-support/lint_exclusions.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/build-support/lint_exclusions.txt b/cpp/build-support/lint_exclusions.txt index 3ae6878c43..f08664a1a9 100644 --- a/cpp/build-support/lint_exclusions.txt +++ b/cpp/build-support/lint_exclusions.txt @@ -3,4 +3,5 @@ *cmake_build* *src/thirdparty* *src/core/thirdparty* -*src/grpc* \ No newline at end of file +*src/grpc* +*easylogging++* \ No newline at end of file From 6a9643f8055f361b63bde5101872d4ff99a320d8 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 25 Sep 2019 20:27:53 +0800 Subject: [PATCH 101/217] add .clang-format Former-commit-id: 30309723f212f9eca5b37146db24d7bde117b3af --- .clang-format | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .clang-format diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000000..06453dfbb2 --- /dev/null +++ b/.clang-format @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +BasedOnStyle: Google +DerivePointerAlignment: false +ColumnLimit: 90 From f2f7919123d98a22f033fcc933d9cbd81e81538d Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Thu, 26 Sep 2019 10:05:16 +0800 Subject: [PATCH 102/217] Update CONTRIBUTING.md Former-commit-id: 1b8f814d6d8d9d512333ef51235e2d5b9726d1e5 --- cpp/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CONTRIBUTING.md b/cpp/CONTRIBUTING.md index 8711d87050..665b8cb985 100644 --- a/cpp/CONTRIBUTING.md +++ b/cpp/CONTRIBUTING.md @@ -49,7 +49,7 @@ And we made the following changes based on the guide: - 4 spaces for indentation - Adopt .cpp file extension instead of .cc extension - 120-character line length -- The file name starts with the upper case +- Camel-Cased file names ## Run unit test From 607bd33b083910a47df456a24a3904c039064164 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 25 Sep 2019 20:14:55 +0800 Subject: [PATCH 103/217] excluse some files Former-commit-id: 5b1b5b4c5d31d2915573c26d8438f7382cabbeca --- cpp/build-support/lint_exclusions.txt | 3 ++- cpp/build-support/run_cpplint.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp/build-support/lint_exclusions.txt b/cpp/build-support/lint_exclusions.txt index 3ae6878c43..f08664a1a9 100644 --- a/cpp/build-support/lint_exclusions.txt +++ b/cpp/build-support/lint_exclusions.txt @@ -3,4 +3,5 @@ *cmake_build* *src/thirdparty* *src/core/thirdparty* -*src/grpc* \ No newline at end of file +*src/grpc* +*easylogging++* \ No newline at end of file diff --git a/cpp/build-support/run_cpplint.py b/cpp/build-support/run_cpplint.py index 2a33748279..98c4170858 100755 --- a/cpp/build-support/run_cpplint.py +++ b/cpp/build-support/run_cpplint.py @@ -84,7 +84,7 @@ if __name__ == "__main__": cmd = [ arguments.cpplint_binary, '--verbose=2', - '--linelength=90', + '--linelength=120', '--filter=' + ','.join(_filters) ] if (arguments.cpplint_binary.endswith('.py') and From a392e03e853d5cdff574657e4d58583cb1be78d3 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 26 Sep 2019 10:39:10 +0800 Subject: [PATCH 104/217] add .clang-tidy Former-commit-id: 16e15b04d447624f1ca140be6f9b9d066aaa64cf --- .clang-tidy | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .clang-tidy diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 0000000000..1f7850efe7 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +Checks: 'clang-diagnostic-*,clang-analyzer-*,-clang-analyzer-alpha*,google-*,modernize-*,readability-*' +# produce HeaderFilterRegex from cpp/build-support/lint_exclusions.txt with: +# echo -n '^('; sed -e 's/*/\.*/g' cpp/build-support/lint_exclusions.txt | tr '\n' '|'; echo ')$' +HeaderFilterRegex: '^(.*cmake-build-debug.*|.*cmake-build-release.*|.*cmake_build.*|.*src/thirdparty.*|.*src/core/thirdparty.*|.*src/grpc.*|)$' +AnalyzeTemporaryDtors: true +CheckOptions: + - key: google-readability-braces-around-statements.ShortStatementLines + value: '1' + - key: google-readability-function-size.StatementThreshold + value: '800' + - key: google-readability-namespace-comments.ShortNamespaceLines + value: '10' + - key: google-readability-namespace-comments.SpacesBeforeComments + value: '2' From e3929b0d409d2261204e44ff1081903ce5d3cd1f Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 26 Sep 2019 10:42:26 +0800 Subject: [PATCH 105/217] add .clang-tidy-ignore Former-commit-id: a641dba1d299c58e28dd61ac6920ead28c2d5ed6 --- .clang-tidy-ignore | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .clang-tidy-ignore diff --git a/.clang-tidy-ignore b/.clang-tidy-ignore new file mode 100644 index 0000000000..0afeab3fa7 --- /dev/null +++ b/.clang-tidy-ignore @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ipc-adapter-test.cc +# memory-pool-test.cc From 731d5402b22c9db4a6302eae894b006a1fa983bc Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 26 Sep 2019 10:50:27 +0800 Subject: [PATCH 106/217] add cmake format python file Former-commit-id: 073fb9af2e0d2e4ea64de620cbcc2852e9fe83ee --- cmake-format.py | 59 ++++++++++++++++++++++++++++ run-cmake-format.py | 96 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 cmake-format.py create mode 100755 run-cmake-format.py diff --git a/cmake-format.py b/cmake-format.py new file mode 100644 index 0000000000..0976642031 --- /dev/null +++ b/cmake-format.py @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# cmake-format configuration file +# Use run-cmake-format.py to reformat all cmake files in the source tree + +# How wide to allow formatted cmake files +line_width = 90 + +# How many spaces to tab for indent +tab_size = 2 + +# If arglists are longer than this, break them always +max_subargs_per_line = 4 + +# If true, separate flow control names from their parentheses with a space +separate_ctrl_name_with_space = False + +# If true, separate function names from parentheses with a space +separate_fn_name_with_space = False + +# If a statement is wrapped to more than one line, than dangle the closing +# parenthesis on it's own line +dangle_parens = False + +# What style line endings to use in the output. +line_ending = 'unix' + +# Format command names consistently as 'lower' or 'upper' case +command_case = 'lower' + +# Format keywords consistently as 'lower' or 'upper' case +keyword_case = 'unchanged' + +# enable comment markup parsing and reflow +enable_markup = False + +# If comment markup is enabled, don't reflow the first comment block in +# eachlistfile. Use this to preserve formatting of your +# copyright/licensestatements. +first_comment_is_literal = False + +# If comment markup is enabled, don't reflow any comment block which matchesthis +# (regex) pattern. Default is `None` (disabled). +literal_comment_pattern = None diff --git a/run-cmake-format.py b/run-cmake-format.py new file mode 100755 index 0000000000..07c615996d --- /dev/null +++ b/run-cmake-format.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import hashlib +import pathlib +import subprocess +import sys + + +patterns = [ + 'cpp/CMakeLists.txt', + # Keep an explicit list of files to format as we don't want to reformat + # files we imported from other location. + 'cpp/cmake/BuildUtils.cmake', + 'cpp/cmake/DefineOptions.cmake', + 'cpp/cmake/FindClangTools.cmake', + 'cpp/cmake/ThirdPartyPackages.cmake', + 'cpp/src/core/cmake/BuildUtilsCore.cmake', + 'cpp/src/core/cmake/DefineOptionsCore.cmake', + 'cpp/src/core/cmake/ThirdPartyPackagesCore.cmake + 'cpp/src/**/CMakeLists.txt', + 'cpp/unittest/**/CMakeLists.txt' +] + +here = pathlib.Path(__file__).parent + + +def find_cmake_files(): + for pat in patterns: + yield from here.glob(pat) + + +def run_cmake_format(paths): + # cmake-format is fast enough that running in parallel doesn't seem + # necessary + # autosort is off because it breaks in cmake_format 5.1 + # See: https://github.com/cheshirekow/cmake_format/issues/111 + cmd = ['cmake-format', '--in-place', '--autosort=false'] + paths + try: + subprocess.run(cmd, check=True) + except FileNotFoundError: + try: + import cmake_format + except ImportError: + raise ImportError( + "Please install cmake-format: `pip install cmake_format`") + else: + # Other error, re-raise + raise + + +def check_cmake_format(paths): + hashes = {} + for p in paths: + contents = p.read_bytes() + hashes[p] = hashlib.sha256(contents).digest() + + run_cmake_format(paths) + + # Check contents didn't change + changed = [] + for p in paths: + contents = p.read_bytes() + if hashes[p] != hashlib.sha256(contents).digest(): + changed.append(p) + + if changed: + items = "\n".join("- %s" % p for p in sorted(changed)) + print("The following cmake files need re-formatting:\n%s" % (items,)) + print() + print("Consider running `run-cmake-format.py`") + sys.exit(1) + + +if __name__ == "__main__": + paths = list(find_cmake_files()) + if "--check" in sys.argv: + check_cmake_format(paths) + else: + run_cmake_format(paths) From 4685869bd3f538f29ed0b21900a41a9179fe4b3b Mon Sep 17 00:00:00 2001 From: "jielin.xu" Date: Thu, 26 Sep 2019 11:05:59 +0800 Subject: [PATCH 107/217] Update CODE OF CONDUCT.md Former-commit-id: fec111fd18e6c31e8583b9f8af383a281f729bc3 --- cpp/{CODE OF CONDUCT.md => CODE_OF_CONDUCT.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cpp/{CODE OF CONDUCT.md => CODE_OF_CONDUCT.md} (100%) diff --git a/cpp/CODE OF CONDUCT.md b/cpp/CODE_OF_CONDUCT.md similarity index 100% rename from cpp/CODE OF CONDUCT.md rename to cpp/CODE_OF_CONDUCT.md From cf3798dd5586e6daceaee1140260b72557975573 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 26 Sep 2019 11:19:39 +0800 Subject: [PATCH 108/217] add cmake format python file Former-commit-id: f64a840c533e0d19c2ef912175adaecd140298d1 --- run-cmake-format.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/run-cmake-format.py b/run-cmake-format.py index 07c615996d..b09100757a 100755 --- a/run-cmake-format.py +++ b/run-cmake-format.py @@ -33,7 +33,7 @@ patterns = [ 'cpp/cmake/ThirdPartyPackages.cmake', 'cpp/src/core/cmake/BuildUtilsCore.cmake', 'cpp/src/core/cmake/DefineOptionsCore.cmake', - 'cpp/src/core/cmake/ThirdPartyPackagesCore.cmake + 'cpp/src/core/cmake/ThirdPartyPackagesCore.cmake', 'cpp/src/**/CMakeLists.txt', 'cpp/unittest/**/CMakeLists.txt' ] @@ -51,7 +51,9 @@ def run_cmake_format(paths): # necessary # autosort is off because it breaks in cmake_format 5.1 # See: https://github.com/cheshirekow/cmake_format/issues/111 - cmd = ['cmake-format', '--in-place', '--autosort=false'] + paths + _paths = [str(path) for path in paths] + + cmd = ['cmake-format', '--in-place', '--autosort=false'] + _paths try: subprocess.run(cmd, check=True) except FileNotFoundError: From e1cfb6cc0650b5f08c8bc6e748730bd6c7273570 Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Thu, 26 Sep 2019 11:27:27 +0800 Subject: [PATCH 109/217] MS-591 Rename source files in wrapper Former-commit-id: 1b864ef00ccdc0e524c2cf7cceb97428171077f8 --- cpp/src/cache/DataObj.h | 2 +- cpp/src/db/engine/ExecutionEngineImpl.cpp | 4 ++-- cpp/src/db/engine/ExecutionEngineImpl.h | 2 +- cpp/src/wrapper/{data_transfer.cpp => DataTransfer.cpp} | 2 +- cpp/src/wrapper/{data_transfer.h => DataTransfer.h} | 0 cpp/src/wrapper/{vec_impl.cpp => VecImpl.cpp} | 4 ++-- cpp/src/wrapper/{vec_impl.h => VecImpl.h} | 2 +- cpp/src/wrapper/{vec_index.cpp => VecIndex.cpp} | 4 ++-- cpp/src/wrapper/{vec_index.h => VecIndex.h} | 0 cpp/unittest/scheduler/scheduler_test.cpp | 2 +- cpp/unittest/server/cache_test.cpp | 2 +- cpp/unittest/wrapper/CMakeLists.txt | 6 +++--- cpp/unittest/wrapper/wrapper_test.cpp | 2 +- 13 files changed, 16 insertions(+), 16 deletions(-) rename cpp/src/wrapper/{data_transfer.cpp => DataTransfer.cpp} (98%) rename cpp/src/wrapper/{data_transfer.h => DataTransfer.h} (100%) rename cpp/src/wrapper/{vec_impl.cpp => VecImpl.cpp} (99%) rename cpp/src/wrapper/{vec_impl.h => VecImpl.h} (99%) rename cpp/src/wrapper/{vec_index.cpp => VecIndex.cpp} (99%) rename cpp/src/wrapper/{vec_index.h => VecIndex.h} (100%) diff --git a/cpp/src/cache/DataObj.h b/cpp/src/cache/DataObj.h index 12406301cb..f9215bc152 100644 --- a/cpp/src/cache/DataObj.h +++ b/cpp/src/cache/DataObj.h @@ -18,7 +18,7 @@ #pragma once -#include "src/wrapper/vec_index.h" +#include "src/wrapper/VecIndex.h" #include diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 45a7699c10..b9d18b4a2f 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -23,8 +23,8 @@ #include "utils/CommonUtil.h" #include "utils/Exception.h" -#include "src/wrapper/vec_index.h" -#include "src/wrapper/vec_impl.h" +#include "src/wrapper/VecIndex.h" +#include "src/wrapper/VecImpl.h" #include "knowhere/common/Exception.h" #include "server/Config.h" diff --git a/cpp/src/db/engine/ExecutionEngineImpl.h b/cpp/src/db/engine/ExecutionEngineImpl.h index 44d59d83e8..1a3c6c83be 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.h +++ b/cpp/src/db/engine/ExecutionEngineImpl.h @@ -18,7 +18,7 @@ #pragma once #include "ExecutionEngine.h" -#include "src/wrapper/vec_index.h" +#include "src/wrapper/VecIndex.h" #include #include diff --git a/cpp/src/wrapper/data_transfer.cpp b/cpp/src/wrapper/DataTransfer.cpp similarity index 98% rename from cpp/src/wrapper/data_transfer.cpp rename to cpp/src/wrapper/DataTransfer.cpp index ad29e6cd7d..294a3c20ab 100644 --- a/cpp/src/wrapper/data_transfer.cpp +++ b/cpp/src/wrapper/DataTransfer.cpp @@ -16,7 +16,7 @@ // under the License. -#include "data_transfer.h" +#include "DataTransfer.h" namespace zilliz { diff --git a/cpp/src/wrapper/data_transfer.h b/cpp/src/wrapper/DataTransfer.h similarity index 100% rename from cpp/src/wrapper/data_transfer.h rename to cpp/src/wrapper/DataTransfer.h diff --git a/cpp/src/wrapper/vec_impl.cpp b/cpp/src/wrapper/VecImpl.cpp similarity index 99% rename from cpp/src/wrapper/vec_impl.cpp rename to cpp/src/wrapper/VecImpl.cpp index e909e611b2..24f6a2bb0e 100644 --- a/cpp/src/wrapper/vec_impl.cpp +++ b/cpp/src/wrapper/VecImpl.cpp @@ -21,8 +21,8 @@ #include "knowhere/index/vector_index/IndexGPUIVF.h" #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/helpers/Cloner.h" -#include "vec_impl.h" -#include "data_transfer.h" +#include "VecImpl.h" +#include "DataTransfer.h" namespace zilliz { diff --git a/cpp/src/wrapper/vec_impl.h b/cpp/src/wrapper/VecImpl.h similarity index 99% rename from cpp/src/wrapper/vec_impl.h rename to cpp/src/wrapper/VecImpl.h index b02ffd690b..c111eb75b4 100644 --- a/cpp/src/wrapper/vec_impl.h +++ b/cpp/src/wrapper/VecImpl.h @@ -19,7 +19,7 @@ #pragma once #include "knowhere/index/vector_index/VectorIndex.h" -#include "vec_index.h" +#include "VecIndex.h" namespace zilliz { diff --git a/cpp/src/wrapper/vec_index.cpp b/cpp/src/wrapper/VecIndex.cpp similarity index 99% rename from cpp/src/wrapper/vec_index.cpp rename to cpp/src/wrapper/VecIndex.cpp index 17a27caa74..cd8843e5ac 100644 --- a/cpp/src/wrapper/vec_index.cpp +++ b/cpp/src/wrapper/VecIndex.cpp @@ -25,8 +25,8 @@ #include "knowhere/index/vector_index/IndexKDT.h" #include "knowhere/index/vector_index/IndexNSG.h" #include "knowhere/common/Exception.h" -#include "vec_index.h" -#include "vec_impl.h" +#include "VecIndex.h" +#include "VecImpl.h" #include "utils/Log.h" #include diff --git a/cpp/src/wrapper/vec_index.h b/cpp/src/wrapper/VecIndex.h similarity index 100% rename from cpp/src/wrapper/vec_index.h rename to cpp/src/wrapper/VecIndex.h diff --git a/cpp/unittest/scheduler/scheduler_test.cpp b/cpp/unittest/scheduler/scheduler_test.cpp index dd2c8e8cb9..8fc65f7d3b 100644 --- a/cpp/unittest/scheduler/scheduler_test.cpp +++ b/cpp/unittest/scheduler/scheduler_test.cpp @@ -26,7 +26,7 @@ #include "cache/DataObj.h" #include "cache/GpuCacheMgr.h" #include "utils/Error.h" -#include "wrapper/vec_index.h" +#include "wrapper/VecIndex.h" namespace zilliz { diff --git a/cpp/unittest/server/cache_test.cpp b/cpp/unittest/server/cache_test.cpp index 3587846085..6c2160c9ab 100644 --- a/cpp/unittest/server/cache_test.cpp +++ b/cpp/unittest/server/cache_test.cpp @@ -20,7 +20,7 @@ #include "cache/GpuCacheMgr.h" #include "server/ServerConfig.h" #include "utils/Error.h" -#include "wrapper/vec_index.h" +#include "wrapper/VecIndex.h" using namespace zilliz::milvus; diff --git a/cpp/unittest/wrapper/CMakeLists.txt b/cpp/unittest/wrapper/CMakeLists.txt index d0eb3bfcfb..633cbaf053 100644 --- a/cpp/unittest/wrapper/CMakeLists.txt +++ b/cpp/unittest/wrapper/CMakeLists.txt @@ -21,9 +21,9 @@ include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") set(wrapper_files - ${MILVUS_ENGINE_SRC}/wrapper/data_transfer.cpp - ${MILVUS_ENGINE_SRC}/wrapper/vec_impl.cpp - ${MILVUS_ENGINE_SRC}/wrapper/vec_index.cpp) + ${MILVUS_ENGINE_SRC}/wrapper/DataTransfer.cpp + ${MILVUS_ENGINE_SRC}/wrapper/VecImpl.cpp + ${MILVUS_ENGINE_SRC}/wrapper/VecIndex.cpp) set(util_files utils.cpp diff --git a/cpp/unittest/wrapper/wrapper_test.cpp b/cpp/unittest/wrapper/wrapper_test.cpp index b052758b6c..4c50201fb3 100644 --- a/cpp/unittest/wrapper/wrapper_test.cpp +++ b/cpp/unittest/wrapper/wrapper_test.cpp @@ -16,7 +16,7 @@ // under the License. #include "utils/easylogging++.h" -#include "src/wrapper/vec_index.h" +#include "src/wrapper/VecIndex.h" #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #include "utils.h" From 786e9c5adc30e01a4cfaf01dcc5cdbcf4d76b6b6 Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Thu, 26 Sep 2019 11:43:45 +0800 Subject: [PATCH 110/217] MS-592 Change showtables stream transport to unary Former-commit-id: c094aa407179fc9b5aea81fdda3fe849d7564bda --- cpp/CHANGELOG.md | 1 + cpp/src/grpc/gen-milvus/milvus.grpc.pb.cc | 38 +- cpp/src/grpc/gen-milvus/milvus.grpc.pb.h | 142 +-- cpp/src/grpc/gen-milvus/milvus.pb.cc | 965 +++++++++++++----- cpp/src/grpc/gen-milvus/milvus.pb.h | 650 +++++++++--- cpp/src/grpc/milvus.proto | 25 +- .../examples/grpcsimple/src/ClientTest.cpp | 10 +- cpp/src/sdk/grpc/ClientProxy.cpp | 16 +- cpp/src/sdk/grpc/GrpcClient.cpp | 29 +- cpp/src/sdk/grpc/GrpcClient.h | 2 +- .../server/grpc_impl/GrpcRequestHandler.cpp | 21 +- cpp/src/server/grpc_impl/GrpcRequestHandler.h | 2 +- cpp/src/server/grpc_impl/GrpcRequestTask.cpp | 24 +- cpp/src/server/grpc_impl/GrpcRequestTask.h | 6 +- cpp/unittest/CMakeLists.txt | 2 +- 15 files changed, 1366 insertions(+), 567 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 2612062263..3b4efaccc3 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -153,6 +153,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-546 - Add simple mode resource_config - MS-570 - Add prometheus docker-compose file - MS-576 - Scheduler refactor +- MS-592 - Change showtables stream transport to unary ## New Feature - MS-343 - Implement ResourceMgr diff --git a/cpp/src/grpc/gen-milvus/milvus.grpc.pb.cc b/cpp/src/grpc/gen-milvus/milvus.grpc.pb.cc index 0a41753828..82a1b99162 100644 --- a/cpp/src/grpc/gen-milvus/milvus.grpc.pb.cc +++ b/cpp/src/grpc/gen-milvus/milvus.grpc.pb.cc @@ -53,7 +53,7 @@ MilvusService::Stub::Stub(const std::shared_ptr< ::grpc::ChannelInterface>& chan , rpcmethod_SearchInFiles_(MilvusService_method_names[6], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) , rpcmethod_DescribeTable_(MilvusService_method_names[7], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) , rpcmethod_CountTable_(MilvusService_method_names[8], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) - , rpcmethod_ShowTables_(MilvusService_method_names[9], ::grpc::internal::RpcMethod::SERVER_STREAMING, channel) + , rpcmethod_ShowTables_(MilvusService_method_names[9], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) , rpcmethod_Cmd_(MilvusService_method_names[10], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) , rpcmethod_DeleteByRange_(MilvusService_method_names[11], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) , rpcmethod_PreloadTable_(MilvusService_method_names[12], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) @@ -313,20 +313,32 @@ void MilvusService::Stub::experimental_async::CountTable(::grpc::ClientContext* return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TableRowCount>::Create(channel_.get(), cq, rpcmethod_CountTable_, context, request, false); } -::grpc::ClientReader< ::milvus::grpc::TableName>* MilvusService::Stub::ShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request) { - return ::grpc_impl::internal::ClientReaderFactory< ::milvus::grpc::TableName>::Create(channel_.get(), rpcmethod_ShowTables_, context, request); +::grpc::Status MilvusService::Stub::ShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::milvus::grpc::TableNameList* response) { + return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_ShowTables_, context, request, response); } -void MilvusService::Stub::experimental_async::ShowTables(::grpc::ClientContext* context, ::milvus::grpc::Command* request, ::grpc::experimental::ClientReadReactor< ::milvus::grpc::TableName>* reactor) { - ::grpc_impl::internal::ClientCallbackReaderFactory< ::milvus::grpc::TableName>::Create(stub_->channel_.get(), stub_->rpcmethod_ShowTables_, context, request, reactor); +void MilvusService::Stub::experimental_async::ShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_ShowTables_, context, request, response, std::move(f)); } -::grpc::ClientAsyncReader< ::milvus::grpc::TableName>* MilvusService::Stub::AsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq, void* tag) { - return ::grpc_impl::internal::ClientAsyncReaderFactory< ::milvus::grpc::TableName>::Create(channel_.get(), cq, rpcmethod_ShowTables_, context, request, true, tag); +void MilvusService::Stub::experimental_async::ShowTables(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableNameList* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_ShowTables_, context, request, response, std::move(f)); } -::grpc::ClientAsyncReader< ::milvus::grpc::TableName>* MilvusService::Stub::PrepareAsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { - return ::grpc_impl::internal::ClientAsyncReaderFactory< ::milvus::grpc::TableName>::Create(channel_.get(), cq, rpcmethod_ShowTables_, context, request, false, nullptr); +void MilvusService::Stub::experimental_async::ShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_ShowTables_, context, request, response, reactor); +} + +void MilvusService::Stub::experimental_async::ShowTables(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableNameList* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_ShowTables_, context, request, response, reactor); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableNameList>* MilvusService::Stub::AsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TableNameList>::Create(channel_.get(), cq, rpcmethod_ShowTables_, context, request, true); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableNameList>* MilvusService::Stub::PrepareAsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TableNameList>::Create(channel_.get(), cq, rpcmethod_ShowTables_, context, request, false); } ::grpc::Status MilvusService::Stub::Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::milvus::grpc::StringReply* response) { @@ -517,8 +529,8 @@ MilvusService::Service::Service() { std::mem_fn(&MilvusService::Service::CountTable), this))); AddMethod(new ::grpc::internal::RpcServiceMethod( MilvusService_method_names[9], - ::grpc::internal::RpcMethod::SERVER_STREAMING, - new ::grpc::internal::ServerStreamingHandler< MilvusService::Service, ::milvus::grpc::Command, ::milvus::grpc::TableName>( + ::grpc::internal::RpcMethod::NORMAL_RPC, + new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::Command, ::milvus::grpc::TableNameList>( std::mem_fn(&MilvusService::Service::ShowTables), this))); AddMethod(new ::grpc::internal::RpcServiceMethod( MilvusService_method_names[10], @@ -613,10 +625,10 @@ MilvusService::Service::~Service() { return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } -::grpc::Status MilvusService::Service::ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::grpc::ServerWriter< ::milvus::grpc::TableName>* writer) { +::grpc::Status MilvusService::Service::ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response) { (void) context; (void) request; - (void) writer; + (void) response; return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } diff --git a/cpp/src/grpc/gen-milvus/milvus.grpc.pb.h b/cpp/src/grpc/gen-milvus/milvus.grpc.pb.h index 0ab25afb71..8ea2d13c80 100644 --- a/cpp/src/grpc/gen-milvus/milvus.grpc.pb.h +++ b/cpp/src/grpc/gen-milvus/milvus.grpc.pb.h @@ -192,14 +192,12 @@ class MilvusService final { // // // @return table names. - std::unique_ptr< ::grpc::ClientReaderInterface< ::milvus::grpc::TableName>> ShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command& request) { - return std::unique_ptr< ::grpc::ClientReaderInterface< ::milvus::grpc::TableName>>(ShowTablesRaw(context, request)); + virtual ::grpc::Status ShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::milvus::grpc::TableNameList* response) = 0; + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableNameList>> AsyncShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableNameList>>(AsyncShowTablesRaw(context, request, cq)); } - std::unique_ptr< ::grpc::ClientAsyncReaderInterface< ::milvus::grpc::TableName>> AsyncShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq, void* tag) { - return std::unique_ptr< ::grpc::ClientAsyncReaderInterface< ::milvus::grpc::TableName>>(AsyncShowTablesRaw(context, request, cq, tag)); - } - std::unique_ptr< ::grpc::ClientAsyncReaderInterface< ::milvus::grpc::TableName>> PrepareAsyncShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncReaderInterface< ::milvus::grpc::TableName>>(PrepareAsyncShowTablesRaw(context, request, cq)); + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableNameList>> PrepareAsyncShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableNameList>>(PrepareAsyncShowTablesRaw(context, request, cq)); } // * // @brief Give the server status @@ -387,7 +385,10 @@ class MilvusService final { // // // @return table names. - virtual void ShowTables(::grpc::ClientContext* context, ::milvus::grpc::Command* request, ::grpc::experimental::ClientReadReactor< ::milvus::grpc::TableName>* reactor) = 0; + virtual void ShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response, std::function) = 0; + virtual void ShowTables(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableNameList* response, std::function) = 0; + virtual void ShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + virtual void ShowTables(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableNameList* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; // * // @brief Give the server status // @@ -459,9 +460,8 @@ class MilvusService final { virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableSchema>* PrepareAsyncDescribeTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableRowCount>* AsyncCountTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableRowCount>* PrepareAsyncCountTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; - virtual ::grpc::ClientReaderInterface< ::milvus::grpc::TableName>* ShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request) = 0; - virtual ::grpc::ClientAsyncReaderInterface< ::milvus::grpc::TableName>* AsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq, void* tag) = 0; - virtual ::grpc::ClientAsyncReaderInterface< ::milvus::grpc::TableName>* PrepareAsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableNameList>* AsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableNameList>* PrepareAsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::StringReply>* AsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::StringReply>* PrepareAsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* AsyncDeleteByRangeRaw(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam& request, ::grpc::CompletionQueue* cq) = 0; @@ -539,14 +539,12 @@ class MilvusService final { std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableRowCount>> PrepareAsyncCountTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableRowCount>>(PrepareAsyncCountTableRaw(context, request, cq)); } - std::unique_ptr< ::grpc::ClientReader< ::milvus::grpc::TableName>> ShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command& request) { - return std::unique_ptr< ::grpc::ClientReader< ::milvus::grpc::TableName>>(ShowTablesRaw(context, request)); + ::grpc::Status ShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::milvus::grpc::TableNameList* response) override; + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableNameList>> AsyncShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableNameList>>(AsyncShowTablesRaw(context, request, cq)); } - std::unique_ptr< ::grpc::ClientAsyncReader< ::milvus::grpc::TableName>> AsyncShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq, void* tag) { - return std::unique_ptr< ::grpc::ClientAsyncReader< ::milvus::grpc::TableName>>(AsyncShowTablesRaw(context, request, cq, tag)); - } - std::unique_ptr< ::grpc::ClientAsyncReader< ::milvus::grpc::TableName>> PrepareAsyncShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncReader< ::milvus::grpc::TableName>>(PrepareAsyncShowTablesRaw(context, request, cq)); + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableNameList>> PrepareAsyncShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableNameList>>(PrepareAsyncShowTablesRaw(context, request, cq)); } ::grpc::Status Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::milvus::grpc::StringReply* response) override; std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>> AsyncCmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { @@ -622,7 +620,10 @@ class MilvusService final { void CountTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableRowCount* response, std::function) override; void CountTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableRowCount* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; void CountTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableRowCount* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; - void ShowTables(::grpc::ClientContext* context, ::milvus::grpc::Command* request, ::grpc::experimental::ClientReadReactor< ::milvus::grpc::TableName>* reactor) override; + void ShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response, std::function) override; + void ShowTables(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableNameList* response, std::function) override; + void ShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void ShowTables(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableNameList* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; void Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response, std::function) override; void Cmd(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::StringReply* response, std::function) override; void Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; @@ -672,9 +673,8 @@ class MilvusService final { ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableSchema>* PrepareAsyncDescribeTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableRowCount>* AsyncCountTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableRowCount>* PrepareAsyncCountTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; - ::grpc::ClientReader< ::milvus::grpc::TableName>* ShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request) override; - ::grpc::ClientAsyncReader< ::milvus::grpc::TableName>* AsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq, void* tag) override; - ::grpc::ClientAsyncReader< ::milvus::grpc::TableName>* PrepareAsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableNameList>* AsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableNameList>* PrepareAsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>* AsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>* PrepareAsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* AsyncDeleteByRangeRaw(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam& request, ::grpc::CompletionQueue* cq) override; @@ -798,7 +798,7 @@ class MilvusService final { // // // @return table names. - virtual ::grpc::Status ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::grpc::ServerWriter< ::milvus::grpc::TableName>* writer); + virtual ::grpc::Status ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response); // * // @brief Give the server status // @@ -1027,12 +1027,12 @@ class MilvusService final { BaseClassMustBeDerivedFromService(this); } // disable synchronous version of this method - ::grpc::Status ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::grpc::ServerWriter< ::milvus::grpc::TableName>* writer) override { + ::grpc::Status ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response) override { abort(); return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } - void RequestShowTables(::grpc::ServerContext* context, ::milvus::grpc::Command* request, ::grpc::ServerAsyncWriter< ::milvus::grpc::TableName>* writer, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncServerStreaming(9, context, request, writer, new_call_cq, notification_cq, tag); + void RequestShowTables(::grpc::ServerContext* context, ::milvus::grpc::Command* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::TableNameList>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(9, context, request, response, new_call_cq, notification_cq, tag); } }; template @@ -1422,20 +1422,29 @@ class MilvusService final { public: ExperimentalWithCallbackMethod_ShowTables() { ::grpc::Service::experimental().MarkMethodCallback(9, - new ::grpc_impl::internal::CallbackServerStreamingHandler< ::milvus::grpc::Command, ::milvus::grpc::TableName>( - [this] { return this->ShowTables(); })); + new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::Command, ::milvus::grpc::TableNameList>( + [this](::grpc::ServerContext* context, + const ::milvus::grpc::Command* request, + ::milvus::grpc::TableNameList* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + return this->ShowTables(context, request, response, controller); + })); + } + void SetMessageAllocatorFor_ShowTables( + ::grpc::experimental::MessageAllocator< ::milvus::grpc::Command, ::milvus::grpc::TableNameList>* allocator) { + static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::Command, ::milvus::grpc::TableNameList>*>( + ::grpc::Service::experimental().GetHandler(9)) + ->SetMessageAllocator(allocator); } ~ExperimentalWithCallbackMethod_ShowTables() override { BaseClassMustBeDerivedFromService(this); } // disable synchronous version of this method - ::grpc::Status ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::grpc::ServerWriter< ::milvus::grpc::TableName>* writer) override { + ::grpc::Status ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response) override { abort(); return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } - virtual ::grpc::experimental::ServerWriteReactor< ::milvus::grpc::Command, ::milvus::grpc::TableName>* ShowTables() { - return new ::grpc_impl::internal::UnimplementedWriteReactor< - ::milvus::grpc::Command, ::milvus::grpc::TableName>;} + virtual void ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } }; template class ExperimentalWithCallbackMethod_Cmd : public BaseClass { @@ -1758,7 +1767,7 @@ class MilvusService final { BaseClassMustBeDerivedFromService(this); } // disable synchronous version of this method - ::grpc::Status ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::grpc::ServerWriter< ::milvus::grpc::TableName>* writer) override { + ::grpc::Status ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response) override { abort(); return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } @@ -2040,12 +2049,12 @@ class MilvusService final { BaseClassMustBeDerivedFromService(this); } // disable synchronous version of this method - ::grpc::Status ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::grpc::ServerWriter< ::milvus::grpc::TableName>* writer) override { + ::grpc::Status ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response) override { abort(); return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } - void RequestShowTables(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncWriter< ::grpc::ByteBuffer>* writer, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncServerStreaming(9, context, request, writer, new_call_cq, notification_cq, tag); + void RequestShowTables(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(9, context, request, response, new_call_cq, notification_cq, tag); } }; template @@ -2380,20 +2389,23 @@ class MilvusService final { public: ExperimentalWithRawCallbackMethod_ShowTables() { ::grpc::Service::experimental().MarkMethodRawCallback(9, - new ::grpc_impl::internal::CallbackServerStreamingHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( - [this] { return this->ShowTables(); })); + new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( + [this](::grpc::ServerContext* context, + const ::grpc::ByteBuffer* request, + ::grpc::ByteBuffer* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + this->ShowTables(context, request, response, controller); + })); } ~ExperimentalWithRawCallbackMethod_ShowTables() override { BaseClassMustBeDerivedFromService(this); } // disable synchronous version of this method - ::grpc::Status ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::grpc::ServerWriter< ::milvus::grpc::TableName>* writer) override { + ::grpc::Status ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response) override { abort(); return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } - virtual ::grpc::experimental::ServerWriteReactor< ::grpc::ByteBuffer, ::grpc::ByteBuffer>* ShowTables() { - return new ::grpc_impl::internal::UnimplementedWriteReactor< - ::grpc::ByteBuffer, ::grpc::ByteBuffer>;} + virtual void ShowTables(::grpc::ServerContext* context, const ::grpc::ByteBuffer* request, ::grpc::ByteBuffer* response, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } }; template class ExperimentalWithRawCallbackMethod_Cmd : public BaseClass { @@ -2701,6 +2713,26 @@ class MilvusService final { virtual ::grpc::Status StreamedCountTable(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::TableName,::milvus::grpc::TableRowCount>* server_unary_streamer) = 0; }; template + class WithStreamedUnaryMethod_ShowTables : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service *service) {} + public: + WithStreamedUnaryMethod_ShowTables() { + ::grpc::Service::MarkMethodStreamed(9, + new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::Command, ::milvus::grpc::TableNameList>(std::bind(&WithStreamedUnaryMethod_ShowTables::StreamedShowTables, this, std::placeholders::_1, std::placeholders::_2))); + } + ~WithStreamedUnaryMethod_ShowTables() override { + BaseClassMustBeDerivedFromService(this); + } + // disable regular version of this method + ::grpc::Status ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + // replace default version of method with streamed unary + virtual ::grpc::Status StreamedShowTables(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::Command,::milvus::grpc::TableNameList>* server_unary_streamer) = 0; + }; + template class WithStreamedUnaryMethod_Cmd : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service *service) {} @@ -2800,29 +2832,9 @@ class MilvusService final { // replace default version of method with streamed unary virtual ::grpc::Status StreamedDropIndex(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::TableName,::milvus::grpc::Status>* server_unary_streamer) = 0; }; - typedef WithStreamedUnaryMethod_CreateTable > > > > > > > > > > > > > StreamedUnaryService; - template - class WithSplitStreamingMethod_ShowTables : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service *service) {} - public: - WithSplitStreamingMethod_ShowTables() { - ::grpc::Service::MarkMethodStreamed(9, - new ::grpc::internal::SplitServerStreamingHandler< ::milvus::grpc::Command, ::milvus::grpc::TableName>(std::bind(&WithSplitStreamingMethod_ShowTables::StreamedShowTables, this, std::placeholders::_1, std::placeholders::_2))); - } - ~WithSplitStreamingMethod_ShowTables() override { - BaseClassMustBeDerivedFromService(this); - } - // disable regular version of this method - ::grpc::Status ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::grpc::ServerWriter< ::milvus::grpc::TableName>* writer) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - // replace default version of method with split streamed - virtual ::grpc::Status StreamedShowTables(::grpc::ServerContext* context, ::grpc::ServerSplitStreamer< ::milvus::grpc::Command,::milvus::grpc::TableName>* server_split_streamer) = 0; - }; - typedef WithSplitStreamingMethod_ShowTables SplitStreamedService; - typedef WithStreamedUnaryMethod_CreateTable > > > > > > > > > > > > > > StreamedService; + typedef WithStreamedUnaryMethod_CreateTable > > > > > > > > > > > > > > StreamedUnaryService; + typedef Service SplitStreamedService; + typedef WithStreamedUnaryMethod_CreateTable > > > > > > > > > > > > > > StreamedService; }; } // namespace grpc diff --git a/cpp/src/grpc/gen-milvus/milvus.pb.cc b/cpp/src/grpc/gen-milvus/milvus.pb.cc index 0a1ab2014c..fe416a4773 100644 --- a/cpp/src/grpc/gen-milvus/milvus.pb.cc +++ b/cpp/src/grpc/gen-milvus/milvus.pb.cc @@ -21,7 +21,6 @@ extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_RowRecord_milvus_2eproto; extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_SearchParam_milvus_2eproto; extern PROTOBUF_INTERNAL_EXPORT_status_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Status_status_2eproto; -extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_TableName_milvus_2eproto; extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_TopKQueryResult_milvus_2eproto; namespace milvus { namespace grpc { @@ -29,6 +28,10 @@ class TableNameDefaultTypeInternal { public: ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; } _TableName_default_instance_; +class TableNameListDefaultTypeInternal { + public: + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; +} _TableNameList_default_instance_; class TableSchemaDefaultTypeInternal { public: ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; @@ -170,7 +173,7 @@ static void InitDefaultsscc_info_IndexParam_milvus_2eproto() { ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_IndexParam_milvus_2eproto = {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 2, InitDefaultsscc_info_IndexParam_milvus_2eproto}, { - &scc_info_TableName_milvus_2eproto.base, + &scc_info_Status_status_2eproto.base, &scc_info_Index_milvus_2eproto.base,}}; static void InitDefaultsscc_info_InsertParam_milvus_2eproto() { @@ -287,8 +290,22 @@ static void InitDefaultsscc_info_TableName_milvus_2eproto() { ::milvus::grpc::TableName::InitAsDefaultInstance(); } -::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_TableName_milvus_2eproto = - {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_TableName_milvus_2eproto}, { +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_TableName_milvus_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_TableName_milvus_2eproto}, {}}; + +static void InitDefaultsscc_info_TableNameList_milvus_2eproto() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + { + void* ptr = &::milvus::grpc::_TableNameList_default_instance_; + new (ptr) ::milvus::grpc::TableNameList(); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); + } + ::milvus::grpc::TableNameList::InitAsDefaultInstance(); +} + +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_TableNameList_milvus_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_TableNameList_milvus_2eproto}, { &scc_info_Status_status_2eproto.base,}}; static void InitDefaultsscc_info_TableRowCount_milvus_2eproto() { @@ -319,7 +336,7 @@ static void InitDefaultsscc_info_TableSchema_milvus_2eproto() { ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_TableSchema_milvus_2eproto = {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_TableSchema_milvus_2eproto}, { - &scc_info_TableName_milvus_2eproto.base,}}; + &scc_info_Status_status_2eproto.base,}}; static void InitDefaultsscc_info_TopKQueryResult_milvus_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -367,7 +384,7 @@ static void InitDefaultsscc_info_VectorIds_milvus_2eproto() { {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_VectorIds_milvus_2eproto}, { &scc_info_Status_status_2eproto.base,}}; -static ::PROTOBUF_NAMESPACE_ID::Metadata file_level_metadata_milvus_2eproto[18]; +static ::PROTOBUF_NAMESPACE_ID::Metadata file_level_metadata_milvus_2eproto[19]; static constexpr ::PROTOBUF_NAMESPACE_ID::EnumDescriptor const** file_level_enum_descriptors_milvus_2eproto = nullptr; static constexpr ::PROTOBUF_NAMESPACE_ID::ServiceDescriptor const** file_level_service_descriptors_milvus_2eproto = nullptr; @@ -377,13 +394,20 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ - PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableName, status_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableName, table_name_), ~0u, // no _has_bits_ + PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableNameList, _internal_metadata_), + ~0u, // no _extensions_ + ~0u, // no _oneof_case_ + ~0u, // no _weak_field_map_ + PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableNameList, status_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableNameList, table_names_), + ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ + PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, status_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, table_name_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, dimension_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, index_file_size_), @@ -492,6 +516,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ + PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, status_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, table_name_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, index_), ~0u, // no _has_bits_ @@ -504,27 +529,29 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT }; static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { { 0, -1, sizeof(::milvus::grpc::TableName)}, - { 7, -1, sizeof(::milvus::grpc::TableSchema)}, - { 16, -1, sizeof(::milvus::grpc::Range)}, - { 23, -1, sizeof(::milvus::grpc::RowRecord)}, - { 29, -1, sizeof(::milvus::grpc::InsertParam)}, - { 37, -1, sizeof(::milvus::grpc::VectorIds)}, - { 44, -1, sizeof(::milvus::grpc::SearchParam)}, - { 54, -1, sizeof(::milvus::grpc::SearchInFilesParam)}, - { 61, -1, sizeof(::milvus::grpc::QueryResult)}, - { 68, -1, sizeof(::milvus::grpc::TopKQueryResult)}, - { 74, -1, sizeof(::milvus::grpc::TopKQueryResultList)}, - { 81, -1, sizeof(::milvus::grpc::StringReply)}, - { 88, -1, sizeof(::milvus::grpc::BoolReply)}, - { 95, -1, sizeof(::milvus::grpc::TableRowCount)}, - { 102, -1, sizeof(::milvus::grpc::Command)}, - { 108, -1, sizeof(::milvus::grpc::Index)}, - { 115, -1, sizeof(::milvus::grpc::IndexParam)}, - { 122, -1, sizeof(::milvus::grpc::DeleteByRangeParam)}, + { 6, -1, sizeof(::milvus::grpc::TableNameList)}, + { 13, -1, sizeof(::milvus::grpc::TableSchema)}, + { 23, -1, sizeof(::milvus::grpc::Range)}, + { 30, -1, sizeof(::milvus::grpc::RowRecord)}, + { 36, -1, sizeof(::milvus::grpc::InsertParam)}, + { 44, -1, sizeof(::milvus::grpc::VectorIds)}, + { 51, -1, sizeof(::milvus::grpc::SearchParam)}, + { 61, -1, sizeof(::milvus::grpc::SearchInFilesParam)}, + { 68, -1, sizeof(::milvus::grpc::QueryResult)}, + { 75, -1, sizeof(::milvus::grpc::TopKQueryResult)}, + { 81, -1, sizeof(::milvus::grpc::TopKQueryResultList)}, + { 88, -1, sizeof(::milvus::grpc::StringReply)}, + { 95, -1, sizeof(::milvus::grpc::BoolReply)}, + { 102, -1, sizeof(::milvus::grpc::TableRowCount)}, + { 109, -1, sizeof(::milvus::grpc::Command)}, + { 115, -1, sizeof(::milvus::grpc::Index)}, + { 122, -1, sizeof(::milvus::grpc::IndexParam)}, + { 130, -1, sizeof(::milvus::grpc::DeleteByRangeParam)}, }; static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = { reinterpret_cast(&::milvus::grpc::_TableName_default_instance_), + reinterpret_cast(&::milvus::grpc::_TableNameList_default_instance_), reinterpret_cast(&::milvus::grpc::_TableSchema_default_instance_), reinterpret_cast(&::milvus::grpc::_Range_default_instance_), reinterpret_cast(&::milvus::grpc::_RowRecord_default_instance_), @@ -546,72 +573,74 @@ static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = const char descriptor_table_protodef_milvus_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = "\n\014milvus.proto\022\013milvus.grpc\032\014status.prot" - "o\"D\n\tTableName\022#\n\006status\030\001 \001(\0132\023.milvus." - "grpc.Status\022\022\n\ntable_name\030\002 \001(\t\"z\n\013Table" - "Schema\022*\n\ntable_name\030\001 \001(\0132\026.milvus.grpc" - ".TableName\022\021\n\tdimension\030\002 \001(\003\022\027\n\017index_f" - "ile_size\030\003 \001(\003\022\023\n\013metric_type\030\004 \001(\005\"/\n\005R" - "ange\022\023\n\013start_value\030\001 \001(\t\022\021\n\tend_value\030\002" - " \001(\t\" \n\tRowRecord\022\023\n\013vector_data\030\001 \003(\002\"i" - "\n\013InsertParam\022\022\n\ntable_name\030\001 \001(\t\0220\n\020row" - "_record_array\030\002 \003(\0132\026.milvus.grpc.RowRec" - "ord\022\024\n\014row_id_array\030\003 \003(\003\"I\n\tVectorIds\022#" - "\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017v" - "ector_id_array\030\002 \003(\003\"\242\001\n\013SearchParam\022\022\n\n" - "table_name\030\001 \001(\t\0222\n\022query_record_array\030\002" - " \003(\0132\026.milvus.grpc.RowRecord\022-\n\021query_ra" - "nge_array\030\003 \003(\0132\022.milvus.grpc.Range\022\014\n\004t" - "opk\030\004 \001(\003\022\016\n\006nprobe\030\005 \001(\003\"[\n\022SearchInFil" - "esParam\022\025\n\rfile_id_array\030\001 \003(\t\022.\n\014search" - "_param\030\002 \001(\0132\030.milvus.grpc.SearchParam\"+" - "\n\013QueryResult\022\n\n\002id\030\001 \001(\003\022\020\n\010distance\030\002 " - "\001(\001\"H\n\017TopKQueryResult\0225\n\023query_result_a" - "rrays\030\001 \003(\0132\030.milvus.grpc.QueryResult\"s\n" - "\023TopKQueryResultList\022#\n\006status\030\001 \001(\0132\023.m" - "ilvus.grpc.Status\0227\n\021topk_query_result\030\002" - " \003(\0132\034.milvus.grpc.TopKQueryResult\"H\n\013St" - "ringReply\022#\n\006status\030\001 \001(\0132\023.milvus.grpc." - "Status\022\024\n\014string_reply\030\002 \001(\t\"D\n\tBoolRepl" - "y\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\022\022" - "\n\nbool_reply\030\002 \001(\010\"M\n\rTableRowCount\022#\n\006s" - "tatus\030\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017tabl" - "e_row_count\030\002 \001(\003\"\026\n\007Command\022\013\n\003cmd\030\001 \001(" - "\t\"*\n\005Index\022\022\n\nindex_type\030\001 \001(\005\022\r\n\005nlist\030" - "\002 \001(\005\"[\n\nIndexParam\022*\n\ntable_name\030\001 \001(\0132" - "\026.milvus.grpc.TableName\022!\n\005index\030\002 \001(\0132\022" - ".milvus.grpc.Index\"K\n\022DeleteByRangeParam" - "\022!\n\005range\030\001 \001(\0132\022.milvus.grpc.Range\022\022\n\nt" - "able_name\030\002 \001(\t2\356\007\n\rMilvusService\022>\n\013Cre" - "ateTable\022\030.milvus.grpc.TableSchema\032\023.mil" - "vus.grpc.Status\"\000\022<\n\010HasTable\022\026.milvus.g" - "rpc.TableName\032\026.milvus.grpc.BoolReply\"\000\022" - ":\n\tDropTable\022\026.milvus.grpc.TableName\032\023.m" - "ilvus.grpc.Status\"\000\022=\n\013CreateIndex\022\027.mil" - "vus.grpc.IndexParam\032\023.milvus.grpc.Status" - "\"\000\022<\n\006Insert\022\030.milvus.grpc.InsertParam\032\026" - ".milvus.grpc.VectorIds\"\000\022F\n\006Search\022\030.mil" - "vus.grpc.SearchParam\032 .milvus.grpc.TopKQ" - "ueryResultList\"\000\022T\n\rSearchInFiles\022\037.milv" - "us.grpc.SearchInFilesParam\032 .milvus.grpc" - ".TopKQueryResultList\"\000\022C\n\rDescribeTable\022" - "\026.milvus.grpc.TableName\032\030.milvus.grpc.Ta" - "bleSchema\"\000\022B\n\nCountTable\022\026.milvus.grpc." - "TableName\032\032.milvus.grpc.TableRowCount\"\000\022" - ">\n\nShowTables\022\024.milvus.grpc.Command\032\026.mi" - "lvus.grpc.TableName\"\0000\001\0227\n\003Cmd\022\024.milvus." - "grpc.Command\032\030.milvus.grpc.StringReply\"\000" - "\022G\n\rDeleteByRange\022\037.milvus.grpc.DeleteBy" - "RangeParam\032\023.milvus.grpc.Status\"\000\022=\n\014Pre" - "loadTable\022\026.milvus.grpc.TableName\032\023.milv" - "us.grpc.Status\"\000\022B\n\rDescribeIndex\022\026.milv" - "us.grpc.TableName\032\027.milvus.grpc.IndexPar" - "am\"\000\022:\n\tDropIndex\022\026.milvus.grpc.TableNam" - "e\032\023.milvus.grpc.Status\"\000b\006proto3" + "o\"\037\n\tTableName\022\022\n\ntable_name\030\001 \001(\t\"I\n\rTa" + "bleNameList\022#\n\006status\030\001 \001(\0132\023.milvus.grp" + "c.Status\022\023\n\013table_names\030\002 \003(\t\"\207\001\n\013TableS" + "chema\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Stat" + "us\022\022\n\ntable_name\030\002 \001(\t\022\021\n\tdimension\030\003 \001(" + "\003\022\027\n\017index_file_size\030\004 \001(\003\022\023\n\013metric_typ" + "e\030\005 \001(\005\"/\n\005Range\022\023\n\013start_value\030\001 \001(\t\022\021\n" + "\tend_value\030\002 \001(\t\" \n\tRowRecord\022\023\n\013vector_" + "data\030\001 \003(\002\"i\n\013InsertParam\022\022\n\ntable_name\030" + "\001 \001(\t\0220\n\020row_record_array\030\002 \003(\0132\026.milvus" + ".grpc.RowRecord\022\024\n\014row_id_array\030\003 \003(\003\"I\n" + "\tVectorIds\022#\n\006status\030\001 \001(\0132\023.milvus.grpc" + ".Status\022\027\n\017vector_id_array\030\002 \003(\003\"\242\001\n\013Sea" + "rchParam\022\022\n\ntable_name\030\001 \001(\t\0222\n\022query_re" + "cord_array\030\002 \003(\0132\026.milvus.grpc.RowRecord" + "\022-\n\021query_range_array\030\003 \003(\0132\022.milvus.grp" + "c.Range\022\014\n\004topk\030\004 \001(\003\022\016\n\006nprobe\030\005 \001(\003\"[\n" + "\022SearchInFilesParam\022\025\n\rfile_id_array\030\001 \003" + "(\t\022.\n\014search_param\030\002 \001(\0132\030.milvus.grpc.S" + "earchParam\"+\n\013QueryResult\022\n\n\002id\030\001 \001(\003\022\020\n" + "\010distance\030\002 \001(\001\"H\n\017TopKQueryResult\0225\n\023qu" + "ery_result_arrays\030\001 \003(\0132\030.milvus.grpc.Qu" + "eryResult\"s\n\023TopKQueryResultList\022#\n\006stat" + "us\030\001 \001(\0132\023.milvus.grpc.Status\0227\n\021topk_qu" + "ery_result\030\002 \003(\0132\034.milvus.grpc.TopKQuery" + "Result\"H\n\013StringReply\022#\n\006status\030\001 \001(\0132\023." + "milvus.grpc.Status\022\024\n\014string_reply\030\002 \001(\t" + "\"D\n\tBoolReply\022#\n\006status\030\001 \001(\0132\023.milvus.g" + "rpc.Status\022\022\n\nbool_reply\030\002 \001(\010\"M\n\rTableR" + "owCount\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.St" + "atus\022\027\n\017table_row_count\030\002 \001(\003\"\026\n\007Command" + "\022\013\n\003cmd\030\001 \001(\t\"*\n\005Index\022\022\n\nindex_type\030\001 \001" + "(\005\022\r\n\005nlist\030\002 \001(\005\"h\n\nIndexParam\022#\n\006statu" + "s\030\001 \001(\0132\023.milvus.grpc.Status\022\022\n\ntable_na" + "me\030\002 \001(\t\022!\n\005index\030\003 \001(\0132\022.milvus.grpc.In" + "dex\"K\n\022DeleteByRangeParam\022!\n\005range\030\001 \001(\013" + "2\022.milvus.grpc.Range\022\022\n\ntable_name\030\002 \001(\t" + "2\360\007\n\rMilvusService\022>\n\013CreateTable\022\030.milv" + "us.grpc.TableSchema\032\023.milvus.grpc.Status" + "\"\000\022<\n\010HasTable\022\026.milvus.grpc.TableName\032\026" + ".milvus.grpc.BoolReply\"\000\022:\n\tDropTable\022\026." + "milvus.grpc.TableName\032\023.milvus.grpc.Stat" + "us\"\000\022=\n\013CreateIndex\022\027.milvus.grpc.IndexP" + "aram\032\023.milvus.grpc.Status\"\000\022<\n\006Insert\022\030." + "milvus.grpc.InsertParam\032\026.milvus.grpc.Ve" + "ctorIds\"\000\022F\n\006Search\022\030.milvus.grpc.Search" + "Param\032 .milvus.grpc.TopKQueryResultList\"" + "\000\022T\n\rSearchInFiles\022\037.milvus.grpc.SearchI" + "nFilesParam\032 .milvus.grpc.TopKQueryResul" + "tList\"\000\022C\n\rDescribeTable\022\026.milvus.grpc.T" + "ableName\032\030.milvus.grpc.TableSchema\"\000\022B\n\n" + "CountTable\022\026.milvus.grpc.TableName\032\032.mil" + "vus.grpc.TableRowCount\"\000\022@\n\nShowTables\022\024" + ".milvus.grpc.Command\032\032.milvus.grpc.Table" + "NameList\"\000\0227\n\003Cmd\022\024.milvus.grpc.Command\032" + "\030.milvus.grpc.StringReply\"\000\022G\n\rDeleteByR" + "ange\022\037.milvus.grpc.DeleteByRangeParam\032\023." + "milvus.grpc.Status\"\000\022=\n\014PreloadTable\022\026.m" + "ilvus.grpc.TableName\032\023.milvus.grpc.Statu" + "s\"\000\022B\n\rDescribeIndex\022\026.milvus.grpc.Table" + "Name\032\027.milvus.grpc.IndexParam\"\000\022:\n\tDropI" + "ndex\022\026.milvus.grpc.TableName\032\023.milvus.gr" + "pc.Status\"\000b\006proto3" ; static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_milvus_2eproto_deps[1] = { &::descriptor_table_status_2eproto, }; -static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_milvus_2eproto_sccs[18] = { +static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_milvus_2eproto_sccs[19] = { &scc_info_BoolReply_milvus_2eproto.base, &scc_info_Command_milvus_2eproto.base, &scc_info_DeleteByRangeParam_milvus_2eproto.base, @@ -625,6 +654,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_mil &scc_info_SearchParam_milvus_2eproto.base, &scc_info_StringReply_milvus_2eproto.base, &scc_info_TableName_milvus_2eproto.base, + &scc_info_TableNameList_milvus_2eproto.base, &scc_info_TableRowCount_milvus_2eproto.base, &scc_info_TableSchema_milvus_2eproto.base, &scc_info_TopKQueryResult_milvus_2eproto.base, @@ -634,10 +664,10 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_mil static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_milvus_2eproto_once; static bool descriptor_table_milvus_2eproto_initialized = false; const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_milvus_2eproto = { - &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 2472, - &descriptor_table_milvus_2eproto_once, descriptor_table_milvus_2eproto_sccs, descriptor_table_milvus_2eproto_deps, 18, 1, + &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 2539, + &descriptor_table_milvus_2eproto_once, descriptor_table_milvus_2eproto_sccs, descriptor_table_milvus_2eproto_deps, 19, 1, schemas, file_default_instances, TableStruct_milvus_2eproto::offsets, - file_level_metadata_milvus_2eproto, 18, file_level_enum_descriptors_milvus_2eproto, file_level_service_descriptors_milvus_2eproto, + file_level_metadata_milvus_2eproto, 19, file_level_enum_descriptors_milvus_2eproto, file_level_service_descriptors_milvus_2eproto, }; // Force running AddDescriptors() at dynamic initialization time. @@ -648,24 +678,11 @@ namespace grpc { // =================================================================== void TableName::InitAsDefaultInstance() { - ::milvus::grpc::_TableName_default_instance_._instance.get_mutable()->status_ = const_cast< ::milvus::grpc::Status*>( - ::milvus::grpc::Status::internal_default_instance()); } class TableName::_Internal { public: - static const ::milvus::grpc::Status& status(const TableName* msg); }; -const ::milvus::grpc::Status& -TableName::_Internal::status(const TableName* msg) { - return *msg->status_; -} -void TableName::clear_status() { - if (GetArenaNoVirtual() == nullptr && status_ != nullptr) { - delete status_; - } - status_ = nullptr; -} TableName::TableName() : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) { SharedCtor(); @@ -679,18 +696,12 @@ TableName::TableName(const TableName& from) if (!from.table_name().empty()) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); } - if (from.has_status()) { - status_ = new ::milvus::grpc::Status(*from.status_); - } else { - status_ = nullptr; - } // @@protoc_insertion_point(copy_constructor:milvus.grpc.TableName) } void TableName::SharedCtor() { ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_TableName_milvus_2eproto.base); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); - status_ = nullptr; } TableName::~TableName() { @@ -700,7 +711,6 @@ TableName::~TableName() { void TableName::SharedDtor() { table_name_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); - if (this != internal_default_instance()) delete status_; } void TableName::SetCachedSize(int size) const { @@ -719,10 +729,6 @@ void TableName::Clear() { (void) cached_has_bits; table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); - if (GetArenaNoVirtual() == nullptr && status_ != nullptr) { - delete status_; - } - status_ = nullptr; _internal_metadata_.Clear(); } @@ -734,16 +740,9 @@ const char* TableName::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID:: ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); CHK_(ptr); switch (tag >> 3) { - // .milvus.grpc.Status status = 1; + // string table_name = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { - ptr = ctx->ParseMessage(mutable_status(), ptr); - CHK_(ptr); - } else goto handle_unusual; - continue; - // string table_name = 2; - case 2: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_table_name(), ptr, ctx, "milvus.grpc.TableName.table_name"); CHK_(ptr); } else goto handle_unusual; @@ -778,20 +777,9 @@ bool TableName::MergePartialFromCodedStream( tag = p.first; if (!p.second) goto handle_unusual; switch (::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // .milvus.grpc.Status status = 1; + // string table_name = 1; case 1: { if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (10 & 0xFF)) { - DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( - input, mutable_status())); - } else { - goto handle_unusual; - } - break; - } - - // string table_name = 2; - case 2: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) { DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( input, this->mutable_table_name())); DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( @@ -831,20 +819,14 @@ void TableName::SerializeWithCachedSizes( ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; - // .milvus.grpc.Status status = 1; - if (this->has_status()) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( - 1, _Internal::status(this), output); - } - - // string table_name = 2; + // string table_name = 1; if (this->table_name().size() > 0) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->table_name().data(), static_cast(this->table_name().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, "milvus.grpc.TableName.table_name"); ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased( - 2, this->table_name(), output); + 1, this->table_name(), output); } if (_internal_metadata_.have_unknown_fields()) { @@ -860,14 +842,7 @@ void TableName::SerializeWithCachedSizes( ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; - // .milvus.grpc.Status status = 1; - if (this->has_status()) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: - InternalWriteMessageToArray( - 1, _Internal::status(this), target); - } - - // string table_name = 2; + // string table_name = 1; if (this->table_name().size() > 0) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->table_name().data(), static_cast(this->table_name().length()), @@ -875,7 +850,7 @@ void TableName::SerializeWithCachedSizes( "milvus.grpc.TableName.table_name"); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray( - 2, this->table_name(), target); + 1, this->table_name(), target); } if (_internal_metadata_.have_unknown_fields()) { @@ -899,20 +874,13 @@ size_t TableName::ByteSizeLong() const { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; - // string table_name = 2; + // string table_name = 1; if (this->table_name().size() > 0) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( this->table_name()); } - // .milvus.grpc.Status status = 1; - if (this->has_status()) { - total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( - *status_); - } - int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; @@ -944,9 +912,6 @@ void TableName::MergeFrom(const TableName& from) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); } - if (from.has_status()) { - mutable_status()->::milvus::grpc::Status::MergeFrom(from.status()); - } } void TableName::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { @@ -972,7 +937,6 @@ void TableName::InternalSwap(TableName* other) { _internal_metadata_.Swap(&other->_internal_metadata_); table_name_.Swap(&other->table_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); - swap(status_, other->status_); } ::PROTOBUF_NAMESPACE_ID::Metadata TableName::GetMetadata() const { @@ -980,20 +944,358 @@ void TableName::InternalSwap(TableName* other) { } +// =================================================================== + +void TableNameList::InitAsDefaultInstance() { + ::milvus::grpc::_TableNameList_default_instance_._instance.get_mutable()->status_ = const_cast< ::milvus::grpc::Status*>( + ::milvus::grpc::Status::internal_default_instance()); +} +class TableNameList::_Internal { + public: + static const ::milvus::grpc::Status& status(const TableNameList* msg); +}; + +const ::milvus::grpc::Status& +TableNameList::_Internal::status(const TableNameList* msg) { + return *msg->status_; +} +void TableNameList::clear_status() { + if (GetArenaNoVirtual() == nullptr && status_ != nullptr) { + delete status_; + } + status_ = nullptr; +} +TableNameList::TableNameList() + : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) { + SharedCtor(); + // @@protoc_insertion_point(constructor:milvus.grpc.TableNameList) +} +TableNameList::TableNameList(const TableNameList& from) + : ::PROTOBUF_NAMESPACE_ID::Message(), + _internal_metadata_(nullptr), + table_names_(from.table_names_) { + _internal_metadata_.MergeFrom(from._internal_metadata_); + if (from.has_status()) { + status_ = new ::milvus::grpc::Status(*from.status_); + } else { + status_ = nullptr; + } + // @@protoc_insertion_point(copy_constructor:milvus.grpc.TableNameList) +} + +void TableNameList::SharedCtor() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_TableNameList_milvus_2eproto.base); + status_ = nullptr; +} + +TableNameList::~TableNameList() { + // @@protoc_insertion_point(destructor:milvus.grpc.TableNameList) + SharedDtor(); +} + +void TableNameList::SharedDtor() { + if (this != internal_default_instance()) delete status_; +} + +void TableNameList::SetCachedSize(int size) const { + _cached_size_.Set(size); +} +const TableNameList& TableNameList::default_instance() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_TableNameList_milvus_2eproto.base); + return *internal_default_instance(); +} + + +void TableNameList::Clear() { +// @@protoc_insertion_point(message_clear_start:milvus.grpc.TableNameList) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + table_names_.Clear(); + if (GetArenaNoVirtual() == nullptr && status_ != nullptr) { + delete status_; + } + status_ = nullptr; + _internal_metadata_.Clear(); +} + +#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER +const char* TableNameList::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { + // .milvus.grpc.Status status = 1; + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + ptr = ctx->ParseMessage(mutable_status(), ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; + // repeated string table_names = 2; + case 2: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { + ptr -= 1; + do { + ptr += 1; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(add_table_names(), ptr, ctx, "milvus.grpc.TableNameList.table_names"); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 18); + } else goto handle_unusual; + continue; + default: { + handle_unusual: + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); + goto success; + } + ptr = UnknownFieldParse(tag, &_internal_metadata_, ptr, ctx); + CHK_(ptr != nullptr); + continue; + } + } // switch + } // while +success: + return ptr; +failure: + ptr = nullptr; + goto success; +#undef CHK_ +} +#else // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER +bool TableNameList::MergePartialFromCodedStream( + ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!PROTOBUF_PREDICT_TRUE(EXPRESSION)) goto failure + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + // @@protoc_insertion_point(parse_start:milvus.grpc.TableNameList) + for (;;) { + ::std::pair<::PROTOBUF_NAMESPACE_ID::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // .milvus.grpc.Status status = 1; + case 1: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (10 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( + input, mutable_status())); + } else { + goto handle_unusual; + } + break; + } + + // repeated string table_names = 2; + case 2: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( + input, this->add_table_names())); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->table_names(this->table_names_size() - 1).data(), + static_cast(this->table_names(this->table_names_size() - 1).length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.grpc.TableNameList.table_names")); + } else { + goto handle_unusual; + } + break; + } + + default: { + handle_unusual: + if (tag == 0) { + goto success; + } + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SkipField( + input, tag, _internal_metadata_.mutable_unknown_fields())); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:milvus.grpc.TableNameList) + return true; +failure: + // @@protoc_insertion_point(parse_failure:milvus.grpc.TableNameList) + return false; +#undef DO_ +} +#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + +void TableNameList::SerializeWithCachedSizes( + ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:milvus.grpc.TableNameList) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // .milvus.grpc.Status status = 1; + if (this->has_status()) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( + 1, _Internal::status(this), output); + } + + // repeated string table_names = 2; + for (int i = 0, n = this->table_names_size(); i < n; i++) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->table_names(i).data(), static_cast(this->table_names(i).length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.TableNameList.table_names"); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteString( + 2, this->table_names(i), output); + } + + if (_internal_metadata_.have_unknown_fields()) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( + _internal_metadata_.unknown_fields(), output); + } + // @@protoc_insertion_point(serialize_end:milvus.grpc.TableNameList) +} + +::PROTOBUF_NAMESPACE_ID::uint8* TableNameList::InternalSerializeWithCachedSizesToArray( + ::PROTOBUF_NAMESPACE_ID::uint8* target) const { + // @@protoc_insertion_point(serialize_to_array_start:milvus.grpc.TableNameList) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // .milvus.grpc.Status status = 1; + if (this->has_status()) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessageToArray( + 1, _Internal::status(this), target); + } + + // repeated string table_names = 2; + for (int i = 0, n = this->table_names_size(); i < n; i++) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->table_names(i).data(), static_cast(this->table_names(i).length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.TableNameList.table_names"); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + WriteStringToArray(2, this->table_names(i), target); + } + + if (_internal_metadata_.have_unknown_fields()) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( + _internal_metadata_.unknown_fields(), target); + } + // @@protoc_insertion_point(serialize_to_array_end:milvus.grpc.TableNameList) + return target; +} + +size_t TableNameList::ByteSizeLong() const { +// @@protoc_insertion_point(message_byte_size_start:milvus.grpc.TableNameList) + size_t total_size = 0; + + if (_internal_metadata_.have_unknown_fields()) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::ComputeUnknownFieldsSize( + _internal_metadata_.unknown_fields()); + } + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + // repeated string table_names = 2; + total_size += 1 * + ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->table_names_size()); + for (int i = 0, n = this->table_names_size(); i < n; i++) { + total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->table_names(i)); + } + + // .milvus.grpc.Status status = 1; + if (this->has_status()) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( + *status_); + } + + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); + SetCachedSize(cached_size); + return total_size; +} + +void TableNameList::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_merge_from_start:milvus.grpc.TableNameList) + GOOGLE_DCHECK_NE(&from, this); + const TableNameList* source = + ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated( + &from); + if (source == nullptr) { + // @@protoc_insertion_point(generalized_merge_from_cast_fail:milvus.grpc.TableNameList) + ::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this); + } else { + // @@protoc_insertion_point(generalized_merge_from_cast_success:milvus.grpc.TableNameList) + MergeFrom(*source); + } +} + +void TableNameList::MergeFrom(const TableNameList& from) { +// @@protoc_insertion_point(class_specific_merge_from_start:milvus.grpc.TableNameList) + GOOGLE_DCHECK_NE(&from, this); + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + table_names_.MergeFrom(from.table_names_); + if (from.has_status()) { + mutable_status()->::milvus::grpc::Status::MergeFrom(from.status()); + } +} + +void TableNameList::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_copy_from_start:milvus.grpc.TableNameList) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +void TableNameList::CopyFrom(const TableNameList& from) { +// @@protoc_insertion_point(class_specific_copy_from_start:milvus.grpc.TableNameList) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool TableNameList::IsInitialized() const { + return true; +} + +void TableNameList::InternalSwap(TableNameList* other) { + using std::swap; + _internal_metadata_.Swap(&other->_internal_metadata_); + table_names_.InternalSwap(CastToBase(&other->table_names_)); + swap(status_, other->status_); +} + +::PROTOBUF_NAMESPACE_ID::Metadata TableNameList::GetMetadata() const { + return GetMetadataStatic(); +} + + // =================================================================== void TableSchema::InitAsDefaultInstance() { - ::milvus::grpc::_TableSchema_default_instance_._instance.get_mutable()->table_name_ = const_cast< ::milvus::grpc::TableName*>( - ::milvus::grpc::TableName::internal_default_instance()); + ::milvus::grpc::_TableSchema_default_instance_._instance.get_mutable()->status_ = const_cast< ::milvus::grpc::Status*>( + ::milvus::grpc::Status::internal_default_instance()); } class TableSchema::_Internal { public: - static const ::milvus::grpc::TableName& table_name(const TableSchema* msg); + static const ::milvus::grpc::Status& status(const TableSchema* msg); }; -const ::milvus::grpc::TableName& -TableSchema::_Internal::table_name(const TableSchema* msg) { - return *msg->table_name_; +const ::milvus::grpc::Status& +TableSchema::_Internal::status(const TableSchema* msg) { + return *msg->status_; +} +void TableSchema::clear_status() { + if (GetArenaNoVirtual() == nullptr && status_ != nullptr) { + delete status_; + } + status_ = nullptr; } TableSchema::TableSchema() : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) { @@ -1004,10 +1306,14 @@ TableSchema::TableSchema(const TableSchema& from) : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) { _internal_metadata_.MergeFrom(from._internal_metadata_); - if (from.has_table_name()) { - table_name_ = new ::milvus::grpc::TableName(*from.table_name_); + table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (!from.table_name().empty()) { + table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); + } + if (from.has_status()) { + status_ = new ::milvus::grpc::Status(*from.status_); } else { - table_name_ = nullptr; + status_ = nullptr; } ::memcpy(&dimension_, &from.dimension_, static_cast(reinterpret_cast(&metric_type_) - @@ -1017,9 +1323,10 @@ TableSchema::TableSchema(const TableSchema& from) void TableSchema::SharedCtor() { ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_TableSchema_milvus_2eproto.base); - ::memset(&table_name_, 0, static_cast( + table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + ::memset(&status_, 0, static_cast( reinterpret_cast(&metric_type_) - - reinterpret_cast(&table_name_)) + sizeof(metric_type_)); + reinterpret_cast(&status_)) + sizeof(metric_type_)); } TableSchema::~TableSchema() { @@ -1028,7 +1335,8 @@ TableSchema::~TableSchema() { } void TableSchema::SharedDtor() { - if (this != internal_default_instance()) delete table_name_; + table_name_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (this != internal_default_instance()) delete status_; } void TableSchema::SetCachedSize(int size) const { @@ -1046,10 +1354,11 @@ void TableSchema::Clear() { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; - if (GetArenaNoVirtual() == nullptr && table_name_ != nullptr) { - delete table_name_; + table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (GetArenaNoVirtual() == nullptr && status_ != nullptr) { + delete status_; } - table_name_ = nullptr; + status_ = nullptr; ::memset(&dimension_, 0, static_cast( reinterpret_cast(&metric_type_) - reinterpret_cast(&dimension_)) + sizeof(metric_type_)); @@ -1064,30 +1373,37 @@ const char* TableSchema::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); CHK_(ptr); switch (tag >> 3) { - // .milvus.grpc.TableName table_name = 1; + // .milvus.grpc.Status status = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { - ptr = ctx->ParseMessage(mutable_table_name(), ptr); + ptr = ctx->ParseMessage(mutable_status(), ptr); CHK_(ptr); } else goto handle_unusual; continue; - // int64 dimension = 2; + // string table_name = 2; case 2: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 16)) { + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_table_name(), ptr, ctx, "milvus.grpc.TableSchema.table_name"); + CHK_(ptr); + } else goto handle_unusual; + continue; + // int64 dimension = 3; + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { dimension_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); CHK_(ptr); } else goto handle_unusual; continue; - // int64 index_file_size = 3; - case 3: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { + // int64 index_file_size = 4; + case 4: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) { index_file_size_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); CHK_(ptr); } else goto handle_unusual; continue; - // int32 metric_type = 4; - case 4: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) { + // int32 metric_type = 5; + case 5: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 40)) { metric_type_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); CHK_(ptr); } else goto handle_unusual; @@ -1122,20 +1438,35 @@ bool TableSchema::MergePartialFromCodedStream( tag = p.first; if (!p.second) goto handle_unusual; switch (::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // .milvus.grpc.TableName table_name = 1; + // .milvus.grpc.Status status = 1; case 1: { if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (10 & 0xFF)) { DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( - input, mutable_table_name())); + input, mutable_status())); } else { goto handle_unusual; } break; } - // int64 dimension = 2; + // string table_name = 2; case 2: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (16 & 0xFF)) { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( + input, this->mutable_table_name())); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->table_name().data(), static_cast(this->table_name().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.grpc.TableSchema.table_name")); + } else { + goto handle_unusual; + } + break; + } + + // int64 dimension = 3; + case 3: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) { DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( @@ -1146,9 +1477,9 @@ bool TableSchema::MergePartialFromCodedStream( break; } - // int64 index_file_size = 3; - case 3: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) { + // int64 index_file_size = 4; + case 4: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) { DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< ::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>( @@ -1159,9 +1490,9 @@ bool TableSchema::MergePartialFromCodedStream( break; } - // int32 metric_type = 4; - case 4: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) { + // int32 metric_type = 5; + case 5: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (40 & 0xFF)) { DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< ::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>( @@ -1199,25 +1530,35 @@ void TableSchema::SerializeWithCachedSizes( ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; - // .milvus.grpc.TableName table_name = 1; - if (this->has_table_name()) { + // .milvus.grpc.Status status = 1; + if (this->has_status()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( - 1, _Internal::table_name(this), output); + 1, _Internal::status(this), output); } - // int64 dimension = 2; + // string table_name = 2; + if (this->table_name().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->table_name().data(), static_cast(this->table_name().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.TableSchema.table_name"); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased( + 2, this->table_name(), output); + } + + // int64 dimension = 3; if (this->dimension() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(2, this->dimension(), output); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(3, this->dimension(), output); } - // int64 index_file_size = 3; + // int64 index_file_size = 4; if (this->index_file_size() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(3, this->index_file_size(), output); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(4, this->index_file_size(), output); } - // int32 metric_type = 4; + // int32 metric_type = 5; if (this->metric_type() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(4, this->metric_type(), output); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(5, this->metric_type(), output); } if (_internal_metadata_.have_unknown_fields()) { @@ -1233,26 +1574,37 @@ void TableSchema::SerializeWithCachedSizes( ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; - // .milvus.grpc.TableName table_name = 1; - if (this->has_table_name()) { + // .milvus.grpc.Status status = 1; + if (this->has_status()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessageToArray( - 1, _Internal::table_name(this), target); + 1, _Internal::status(this), target); } - // int64 dimension = 2; + // string table_name = 2; + if (this->table_name().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->table_name().data(), static_cast(this->table_name().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.TableSchema.table_name"); + target = + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray( + 2, this->table_name(), target); + } + + // int64 dimension = 3; if (this->dimension() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(2, this->dimension(), target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(3, this->dimension(), target); } - // int64 index_file_size = 3; + // int64 index_file_size = 4; if (this->index_file_size() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(3, this->index_file_size(), target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(4, this->index_file_size(), target); } - // int32 metric_type = 4; + // int32 metric_type = 5; if (this->metric_type() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(4, this->metric_type(), target); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(5, this->metric_type(), target); } if (_internal_metadata_.have_unknown_fields()) { @@ -1276,28 +1628,35 @@ size_t TableSchema::ByteSizeLong() const { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; - // .milvus.grpc.TableName table_name = 1; - if (this->has_table_name()) { + // string table_name = 2; + if (this->table_name().size() > 0) { total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( - *table_name_); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->table_name()); } - // int64 dimension = 2; + // .milvus.grpc.Status status = 1; + if (this->has_status()) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( + *status_); + } + + // int64 dimension = 3; if (this->dimension() != 0) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size( this->dimension()); } - // int64 index_file_size = 3; + // int64 index_file_size = 4; if (this->index_file_size() != 0) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size( this->index_file_size()); } - // int32 metric_type = 4; + // int32 metric_type = 5; if (this->metric_type() != 0) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( @@ -1331,8 +1690,12 @@ void TableSchema::MergeFrom(const TableSchema& from) { ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; - if (from.has_table_name()) { - mutable_table_name()->::milvus::grpc::TableName::MergeFrom(from.table_name()); + if (from.table_name().size() > 0) { + + table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); + } + if (from.has_status()) { + mutable_status()->::milvus::grpc::Status::MergeFrom(from.status()); } if (from.dimension() != 0) { set_dimension(from.dimension()); @@ -1366,7 +1729,9 @@ bool TableSchema::IsInitialized() const { void TableSchema::InternalSwap(TableSchema* other) { using std::swap; _internal_metadata_.Swap(&other->_internal_metadata_); - swap(table_name_, other->table_name_); + table_name_.Swap(&other->table_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); + swap(status_, other->status_); swap(dimension_, other->dimension_); swap(index_file_size_, other->index_file_size_); swap(metric_type_, other->metric_type_); @@ -5928,25 +6293,31 @@ void Index::InternalSwap(Index* other) { // =================================================================== void IndexParam::InitAsDefaultInstance() { - ::milvus::grpc::_IndexParam_default_instance_._instance.get_mutable()->table_name_ = const_cast< ::milvus::grpc::TableName*>( - ::milvus::grpc::TableName::internal_default_instance()); + ::milvus::grpc::_IndexParam_default_instance_._instance.get_mutable()->status_ = const_cast< ::milvus::grpc::Status*>( + ::milvus::grpc::Status::internal_default_instance()); ::milvus::grpc::_IndexParam_default_instance_._instance.get_mutable()->index_ = const_cast< ::milvus::grpc::Index*>( ::milvus::grpc::Index::internal_default_instance()); } class IndexParam::_Internal { public: - static const ::milvus::grpc::TableName& table_name(const IndexParam* msg); + static const ::milvus::grpc::Status& status(const IndexParam* msg); static const ::milvus::grpc::Index& index(const IndexParam* msg); }; -const ::milvus::grpc::TableName& -IndexParam::_Internal::table_name(const IndexParam* msg) { - return *msg->table_name_; +const ::milvus::grpc::Status& +IndexParam::_Internal::status(const IndexParam* msg) { + return *msg->status_; } const ::milvus::grpc::Index& IndexParam::_Internal::index(const IndexParam* msg) { return *msg->index_; } +void IndexParam::clear_status() { + if (GetArenaNoVirtual() == nullptr && status_ != nullptr) { + delete status_; + } + status_ = nullptr; +} IndexParam::IndexParam() : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) { SharedCtor(); @@ -5956,10 +6327,14 @@ IndexParam::IndexParam(const IndexParam& from) : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) { _internal_metadata_.MergeFrom(from._internal_metadata_); - if (from.has_table_name()) { - table_name_ = new ::milvus::grpc::TableName(*from.table_name_); + table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (!from.table_name().empty()) { + table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); + } + if (from.has_status()) { + status_ = new ::milvus::grpc::Status(*from.status_); } else { - table_name_ = nullptr; + status_ = nullptr; } if (from.has_index()) { index_ = new ::milvus::grpc::Index(*from.index_); @@ -5971,9 +6346,10 @@ IndexParam::IndexParam(const IndexParam& from) void IndexParam::SharedCtor() { ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_IndexParam_milvus_2eproto.base); - ::memset(&table_name_, 0, static_cast( + table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + ::memset(&status_, 0, static_cast( reinterpret_cast(&index_) - - reinterpret_cast(&table_name_)) + sizeof(index_)); + reinterpret_cast(&status_)) + sizeof(index_)); } IndexParam::~IndexParam() { @@ -5982,7 +6358,8 @@ IndexParam::~IndexParam() { } void IndexParam::SharedDtor() { - if (this != internal_default_instance()) delete table_name_; + table_name_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (this != internal_default_instance()) delete status_; if (this != internal_default_instance()) delete index_; } @@ -6001,10 +6378,11 @@ void IndexParam::Clear() { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; - if (GetArenaNoVirtual() == nullptr && table_name_ != nullptr) { - delete table_name_; + table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (GetArenaNoVirtual() == nullptr && status_ != nullptr) { + delete status_; } - table_name_ = nullptr; + status_ = nullptr; if (GetArenaNoVirtual() == nullptr && index_ != nullptr) { delete index_; } @@ -6020,16 +6398,23 @@ const char* IndexParam::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID: ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); CHK_(ptr); switch (tag >> 3) { - // .milvus.grpc.TableName table_name = 1; + // .milvus.grpc.Status status = 1; case 1: if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { - ptr = ctx->ParseMessage(mutable_table_name(), ptr); + ptr = ctx->ParseMessage(mutable_status(), ptr); CHK_(ptr); } else goto handle_unusual; continue; - // .milvus.grpc.Index index = 2; + // string table_name = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_table_name(), ptr, ctx, "milvus.grpc.IndexParam.table_name"); + CHK_(ptr); + } else goto handle_unusual; + continue; + // .milvus.grpc.Index index = 3; + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 26)) { ptr = ctx->ParseMessage(mutable_index(), ptr); CHK_(ptr); } else goto handle_unusual; @@ -6064,20 +6449,35 @@ bool IndexParam::MergePartialFromCodedStream( tag = p.first; if (!p.second) goto handle_unusual; switch (::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // .milvus.grpc.TableName table_name = 1; + // .milvus.grpc.Status status = 1; case 1: { if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (10 & 0xFF)) { DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( - input, mutable_table_name())); + input, mutable_status())); } else { goto handle_unusual; } break; } - // .milvus.grpc.Index index = 2; + // string table_name = 2; case 2: { if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( + input, this->mutable_table_name())); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->table_name().data(), static_cast(this->table_name().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.grpc.IndexParam.table_name")); + } else { + goto handle_unusual; + } + break; + } + + // .milvus.grpc.Index index = 3; + case 3: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (26 & 0xFF)) { DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( input, mutable_index())); } else { @@ -6113,16 +6513,26 @@ void IndexParam::SerializeWithCachedSizes( ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; - // .milvus.grpc.TableName table_name = 1; - if (this->has_table_name()) { + // .milvus.grpc.Status status = 1; + if (this->has_status()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( - 1, _Internal::table_name(this), output); + 1, _Internal::status(this), output); } - // .milvus.grpc.Index index = 2; + // string table_name = 2; + if (this->table_name().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->table_name().data(), static_cast(this->table_name().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.IndexParam.table_name"); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased( + 2, this->table_name(), output); + } + + // .milvus.grpc.Index index = 3; if (this->has_index()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( - 2, _Internal::index(this), output); + 3, _Internal::index(this), output); } if (_internal_metadata_.have_unknown_fields()) { @@ -6138,18 +6548,29 @@ void IndexParam::SerializeWithCachedSizes( ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; - // .milvus.grpc.TableName table_name = 1; - if (this->has_table_name()) { + // .milvus.grpc.Status status = 1; + if (this->has_status()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessageToArray( - 1, _Internal::table_name(this), target); + 1, _Internal::status(this), target); } - // .milvus.grpc.Index index = 2; + // string table_name = 2; + if (this->table_name().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->table_name().data(), static_cast(this->table_name().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.IndexParam.table_name"); + target = + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray( + 2, this->table_name(), target); + } + + // .milvus.grpc.Index index = 3; if (this->has_index()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: InternalWriteMessageToArray( - 2, _Internal::index(this), target); + 3, _Internal::index(this), target); } if (_internal_metadata_.have_unknown_fields()) { @@ -6173,14 +6594,21 @@ size_t IndexParam::ByteSizeLong() const { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; - // .milvus.grpc.TableName table_name = 1; - if (this->has_table_name()) { + // string table_name = 2; + if (this->table_name().size() > 0) { total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( - *table_name_); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->table_name()); } - // .milvus.grpc.Index index = 2; + // .milvus.grpc.Status status = 1; + if (this->has_status()) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( + *status_); + } + + // .milvus.grpc.Index index = 3; if (this->has_index()) { total_size += 1 + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( @@ -6214,8 +6642,12 @@ void IndexParam::MergeFrom(const IndexParam& from) { ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; - if (from.has_table_name()) { - mutable_table_name()->::milvus::grpc::TableName::MergeFrom(from.table_name()); + if (from.table_name().size() > 0) { + + table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); + } + if (from.has_status()) { + mutable_status()->::milvus::grpc::Status::MergeFrom(from.status()); } if (from.has_index()) { mutable_index()->::milvus::grpc::Index::MergeFrom(from.index()); @@ -6243,7 +6675,9 @@ bool IndexParam::IsInitialized() const { void IndexParam::InternalSwap(IndexParam* other) { using std::swap; _internal_metadata_.Swap(&other->_internal_metadata_); - swap(table_name_, other->table_name_); + table_name_.Swap(&other->table_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); + swap(status_, other->status_); swap(index_, other->index_); } @@ -6588,6 +7022,9 @@ PROTOBUF_NAMESPACE_OPEN template<> PROTOBUF_NOINLINE ::milvus::grpc::TableName* Arena::CreateMaybeMessage< ::milvus::grpc::TableName >(Arena* arena) { return Arena::CreateInternal< ::milvus::grpc::TableName >(arena); } +template<> PROTOBUF_NOINLINE ::milvus::grpc::TableNameList* Arena::CreateMaybeMessage< ::milvus::grpc::TableNameList >(Arena* arena) { + return Arena::CreateInternal< ::milvus::grpc::TableNameList >(arena); +} template<> PROTOBUF_NOINLINE ::milvus::grpc::TableSchema* Arena::CreateMaybeMessage< ::milvus::grpc::TableSchema >(Arena* arena) { return Arena::CreateInternal< ::milvus::grpc::TableSchema >(arena); } diff --git a/cpp/src/grpc/gen-milvus/milvus.pb.h b/cpp/src/grpc/gen-milvus/milvus.pb.h index 543e5b98c9..5ac3fda023 100644 --- a/cpp/src/grpc/gen-milvus/milvus.pb.h +++ b/cpp/src/grpc/gen-milvus/milvus.pb.h @@ -48,7 +48,7 @@ struct TableStruct_milvus_2eproto { PROTOBUF_SECTION_VARIABLE(protodesc_cold); static const ::PROTOBUF_NAMESPACE_ID::internal::AuxillaryParseTableField aux[] PROTOBUF_SECTION_VARIABLE(protodesc_cold); - static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[18] + static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[19] PROTOBUF_SECTION_VARIABLE(protodesc_cold); static const ::PROTOBUF_NAMESPACE_ID::internal::FieldMetadata field_metadata[]; static const ::PROTOBUF_NAMESPACE_ID::internal::SerializationTable serialization_table[]; @@ -96,6 +96,9 @@ extern StringReplyDefaultTypeInternal _StringReply_default_instance_; class TableName; class TableNameDefaultTypeInternal; extern TableNameDefaultTypeInternal _TableName_default_instance_; +class TableNameList; +class TableNameListDefaultTypeInternal; +extern TableNameListDefaultTypeInternal _TableNameList_default_instance_; class TableRowCount; class TableRowCountDefaultTypeInternal; extern TableRowCountDefaultTypeInternal _TableRowCount_default_instance_; @@ -127,6 +130,7 @@ template<> ::milvus::grpc::SearchInFilesParam* Arena::CreateMaybeMessage<::milvu template<> ::milvus::grpc::SearchParam* Arena::CreateMaybeMessage<::milvus::grpc::SearchParam>(Arena*); template<> ::milvus::grpc::StringReply* Arena::CreateMaybeMessage<::milvus::grpc::StringReply>(Arena*); template<> ::milvus::grpc::TableName* Arena::CreateMaybeMessage<::milvus::grpc::TableName>(Arena*); +template<> ::milvus::grpc::TableNameList* Arena::CreateMaybeMessage<::milvus::grpc::TableNameList>(Arena*); template<> ::milvus::grpc::TableRowCount* Arena::CreateMaybeMessage<::milvus::grpc::TableRowCount>(Arena*); template<> ::milvus::grpc::TableSchema* Arena::CreateMaybeMessage<::milvus::grpc::TableSchema>(Arena*); template<> ::milvus::grpc::TopKQueryResult* Arena::CreateMaybeMessage<::milvus::grpc::TopKQueryResult>(Arena*); @@ -251,10 +255,9 @@ class TableName : // accessors ------------------------------------------------------- enum : int { - kTableNameFieldNumber = 2, - kStatusFieldNumber = 1, + kTableNameFieldNumber = 1, }; - // string table_name = 2; + // string table_name = 1; void clear_table_name(); const std::string& table_name() const; void set_table_name(const std::string& value); @@ -265,6 +268,150 @@ class TableName : std::string* release_table_name(); void set_allocated_table_name(std::string* table_name); + // @@protoc_insertion_point(class_scope:milvus.grpc.TableName) + private: + class _Internal; + + ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr table_name_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + friend struct ::TableStruct_milvus_2eproto; +}; +// ------------------------------------------------------------------- + +class TableNameList : + public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.grpc.TableNameList) */ { + public: + TableNameList(); + virtual ~TableNameList(); + + TableNameList(const TableNameList& from); + TableNameList(TableNameList&& from) noexcept + : TableNameList() { + *this = ::std::move(from); + } + + inline TableNameList& operator=(const TableNameList& from) { + CopyFrom(from); + return *this; + } + inline TableNameList& operator=(TableNameList&& from) noexcept { + if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { + if (this != &from) InternalSwap(&from); + } else { + CopyFrom(from); + } + return *this; + } + + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { + return GetDescriptor(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { + return GetMetadataStatic().descriptor; + } + static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { + return GetMetadataStatic().reflection; + } + static const TableNameList& default_instance(); + + static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY + static inline const TableNameList* internal_default_instance() { + return reinterpret_cast( + &_TableNameList_default_instance_); + } + static constexpr int kIndexInFileMessages = + 1; + + friend void swap(TableNameList& a, TableNameList& b) { + a.Swap(&b); + } + inline void Swap(TableNameList* other) { + if (other == this) return; + InternalSwap(other); + } + + // implements Message ---------------------------------------------- + + inline TableNameList* New() const final { + return CreateMaybeMessage(nullptr); + } + + TableNameList* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { + return CreateMaybeMessage(arena); + } + void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void CopyFrom(const TableNameList& from); + void MergeFrom(const TableNameList& from); + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; + bool IsInitialized() const final; + + size_t ByteSizeLong() const final; + #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + #else + bool MergePartialFromCodedStream( + ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final; + #endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + void SerializeWithCachedSizes( + ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final; + ::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray( + ::PROTOBUF_NAMESPACE_ID::uint8* target) const final; + int GetCachedSize() const final { return _cached_size_.Get(); } + + private: + inline void SharedCtor(); + inline void SharedDtor(); + void SetCachedSize(int size) const final; + void InternalSwap(TableNameList* other); + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "milvus.grpc.TableNameList"; + } + private: + inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const { + return nullptr; + } + inline void* MaybeArenaPtr() const { + return nullptr; + } + public: + + ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; + private: + static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() { + ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_milvus_2eproto); + return ::descriptor_table_milvus_2eproto.file_level_metadata[kIndexInFileMessages]; + } + + public: + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + enum : int { + kTableNamesFieldNumber = 2, + kStatusFieldNumber = 1, + }; + // repeated string table_names = 2; + int table_names_size() const; + void clear_table_names(); + const std::string& table_names(int index) const; + std::string* mutable_table_names(int index); + void set_table_names(int index, const std::string& value); + void set_table_names(int index, std::string&& value); + void set_table_names(int index, const char* value); + void set_table_names(int index, const char* value, size_t size); + std::string* add_table_names(); + void add_table_names(const std::string& value); + void add_table_names(std::string&& value); + void add_table_names(const char* value); + void add_table_names(const char* value, size_t size); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& table_names() const; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* mutable_table_names(); + // .milvus.grpc.Status status = 1; bool has_status() const; void clear_status(); @@ -273,12 +420,12 @@ class TableName : ::milvus::grpc::Status* mutable_status(); void set_allocated_status(::milvus::grpc::Status* status); - // @@protoc_insertion_point(class_scope:milvus.grpc.TableName) + // @@protoc_insertion_point(class_scope:milvus.grpc.TableNameList) private: class _Internal; ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; - ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr table_name_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField table_names_; ::milvus::grpc::Status* status_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_milvus_2eproto; @@ -327,7 +474,7 @@ class TableSchema : &_TableSchema_default_instance_); } static constexpr int kIndexInFileMessages = - 1; + 2; friend void swap(TableSchema& a, TableSchema& b) { a.Swap(&b); @@ -398,30 +545,42 @@ class TableSchema : // accessors ------------------------------------------------------- enum : int { - kTableNameFieldNumber = 1, - kDimensionFieldNumber = 2, - kIndexFileSizeFieldNumber = 3, - kMetricTypeFieldNumber = 4, + kTableNameFieldNumber = 2, + kStatusFieldNumber = 1, + kDimensionFieldNumber = 3, + kIndexFileSizeFieldNumber = 4, + kMetricTypeFieldNumber = 5, }; - // .milvus.grpc.TableName table_name = 1; - bool has_table_name() const; + // string table_name = 2; void clear_table_name(); - const ::milvus::grpc::TableName& table_name() const; - ::milvus::grpc::TableName* release_table_name(); - ::milvus::grpc::TableName* mutable_table_name(); - void set_allocated_table_name(::milvus::grpc::TableName* table_name); + const std::string& table_name() const; + void set_table_name(const std::string& value); + void set_table_name(std::string&& value); + void set_table_name(const char* value); + void set_table_name(const char* value, size_t size); + std::string* mutable_table_name(); + std::string* release_table_name(); + void set_allocated_table_name(std::string* table_name); - // int64 dimension = 2; + // .milvus.grpc.Status status = 1; + bool has_status() const; + void clear_status(); + const ::milvus::grpc::Status& status() const; + ::milvus::grpc::Status* release_status(); + ::milvus::grpc::Status* mutable_status(); + void set_allocated_status(::milvus::grpc::Status* status); + + // int64 dimension = 3; void clear_dimension(); ::PROTOBUF_NAMESPACE_ID::int64 dimension() const; void set_dimension(::PROTOBUF_NAMESPACE_ID::int64 value); - // int64 index_file_size = 3; + // int64 index_file_size = 4; void clear_index_file_size(); ::PROTOBUF_NAMESPACE_ID::int64 index_file_size() const; void set_index_file_size(::PROTOBUF_NAMESPACE_ID::int64 value); - // int32 metric_type = 4; + // int32 metric_type = 5; void clear_metric_type(); ::PROTOBUF_NAMESPACE_ID::int32 metric_type() const; void set_metric_type(::PROTOBUF_NAMESPACE_ID::int32 value); @@ -431,7 +590,8 @@ class TableSchema : class _Internal; ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; - ::milvus::grpc::TableName* table_name_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr table_name_; + ::milvus::grpc::Status* status_; ::PROTOBUF_NAMESPACE_ID::int64 dimension_; ::PROTOBUF_NAMESPACE_ID::int64 index_file_size_; ::PROTOBUF_NAMESPACE_ID::int32 metric_type_; @@ -482,7 +642,7 @@ class Range : &_Range_default_instance_); } static constexpr int kIndexInFileMessages = - 2; + 3; friend void swap(Range& a, Range& b) { a.Swap(&b); @@ -632,7 +792,7 @@ class RowRecord : &_RowRecord_default_instance_); } static constexpr int kIndexInFileMessages = - 3; + 4; friend void swap(RowRecord& a, RowRecord& b) { a.Swap(&b); @@ -770,7 +930,7 @@ class InsertParam : &_InsertParam_default_instance_); } static constexpr int kIndexInFileMessages = - 4; + 5; friend void swap(InsertParam& a, InsertParam& b) { a.Swap(&b); @@ -934,7 +1094,7 @@ class VectorIds : &_VectorIds_default_instance_); } static constexpr int kIndexInFileMessages = - 5; + 6; friend void swap(VectorIds& a, VectorIds& b) { a.Swap(&b); @@ -1082,7 +1242,7 @@ class SearchParam : &_SearchParam_default_instance_); } static constexpr int kIndexInFileMessages = - 6; + 7; friend void swap(SearchParam& a, SearchParam& b) { a.Swap(&b); @@ -1259,7 +1419,7 @@ class SearchInFilesParam : &_SearchInFilesParam_default_instance_); } static constexpr int kIndexInFileMessages = - 7; + 8; friend void swap(SearchInFilesParam& a, SearchInFilesParam& b) { a.Swap(&b); @@ -1412,7 +1572,7 @@ class QueryResult : &_QueryResult_default_instance_); } static constexpr int kIndexInFileMessages = - 8; + 9; friend void swap(QueryResult& a, QueryResult& b) { a.Swap(&b); @@ -1550,7 +1710,7 @@ class TopKQueryResult : &_TopKQueryResult_default_instance_); } static constexpr int kIndexInFileMessages = - 9; + 10; friend void swap(TopKQueryResult& a, TopKQueryResult& b) { a.Swap(&b); @@ -1687,7 +1847,7 @@ class TopKQueryResultList : &_TopKQueryResultList_default_instance_); } static constexpr int kIndexInFileMessages = - 10; + 11; friend void swap(TopKQueryResultList& a, TopKQueryResultList& b) { a.Swap(&b); @@ -1834,7 +1994,7 @@ class StringReply : &_StringReply_default_instance_); } static constexpr int kIndexInFileMessages = - 11; + 12; friend void swap(StringReply& a, StringReply& b) { a.Swap(&b); @@ -1981,7 +2141,7 @@ class BoolReply : &_BoolReply_default_instance_); } static constexpr int kIndexInFileMessages = - 12; + 13; friend void swap(BoolReply& a, BoolReply& b) { a.Swap(&b); @@ -2122,7 +2282,7 @@ class TableRowCount : &_TableRowCount_default_instance_); } static constexpr int kIndexInFileMessages = - 13; + 14; friend void swap(TableRowCount& a, TableRowCount& b) { a.Swap(&b); @@ -2263,7 +2423,7 @@ class Command : &_Command_default_instance_); } static constexpr int kIndexInFileMessages = - 14; + 15; friend void swap(Command& a, Command& b) { a.Swap(&b); @@ -2400,7 +2560,7 @@ class Index : &_Index_default_instance_); } static constexpr int kIndexInFileMessages = - 15; + 16; friend void swap(Index& a, Index& b) { a.Swap(&b); @@ -2538,7 +2698,7 @@ class IndexParam : &_IndexParam_default_instance_); } static constexpr int kIndexInFileMessages = - 16; + 17; friend void swap(IndexParam& a, IndexParam& b) { a.Swap(&b); @@ -2609,18 +2769,30 @@ class IndexParam : // accessors ------------------------------------------------------- enum : int { - kTableNameFieldNumber = 1, - kIndexFieldNumber = 2, + kTableNameFieldNumber = 2, + kStatusFieldNumber = 1, + kIndexFieldNumber = 3, }; - // .milvus.grpc.TableName table_name = 1; - bool has_table_name() const; + // string table_name = 2; void clear_table_name(); - const ::milvus::grpc::TableName& table_name() const; - ::milvus::grpc::TableName* release_table_name(); - ::milvus::grpc::TableName* mutable_table_name(); - void set_allocated_table_name(::milvus::grpc::TableName* table_name); + const std::string& table_name() const; + void set_table_name(const std::string& value); + void set_table_name(std::string&& value); + void set_table_name(const char* value); + void set_table_name(const char* value, size_t size); + std::string* mutable_table_name(); + std::string* release_table_name(); + void set_allocated_table_name(std::string* table_name); - // .milvus.grpc.Index index = 2; + // .milvus.grpc.Status status = 1; + bool has_status() const; + void clear_status(); + const ::milvus::grpc::Status& status() const; + ::milvus::grpc::Status* release_status(); + ::milvus::grpc::Status* mutable_status(); + void set_allocated_status(::milvus::grpc::Status* status); + + // .milvus.grpc.Index index = 3; bool has_index() const; void clear_index(); const ::milvus::grpc::Index& index() const; @@ -2633,7 +2805,8 @@ class IndexParam : class _Internal; ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; - ::milvus::grpc::TableName* table_name_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr table_name_; + ::milvus::grpc::Status* status_; ::milvus::grpc::Index* index_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_milvus_2eproto; @@ -2682,7 +2855,7 @@ class DeleteByRangeParam : &_DeleteByRangeParam_default_instance_); } static constexpr int kIndexInFileMessages = - 17; + 18; friend void swap(DeleteByRangeParam& a, DeleteByRangeParam& b) { a.Swap(&b); @@ -2796,52 +2969,7 @@ class DeleteByRangeParam : #endif // __GNUC__ // TableName -// .milvus.grpc.Status status = 1; -inline bool TableName::has_status() const { - return this != internal_default_instance() && status_ != nullptr; -} -inline const ::milvus::grpc::Status& TableName::status() const { - const ::milvus::grpc::Status* p = status_; - // @@protoc_insertion_point(field_get:milvus.grpc.TableName.status) - return p != nullptr ? *p : *reinterpret_cast( - &::milvus::grpc::_Status_default_instance_); -} -inline ::milvus::grpc::Status* TableName::release_status() { - // @@protoc_insertion_point(field_release:milvus.grpc.TableName.status) - - ::milvus::grpc::Status* temp = status_; - status_ = nullptr; - return temp; -} -inline ::milvus::grpc::Status* TableName::mutable_status() { - - if (status_ == nullptr) { - auto* p = CreateMaybeMessage<::milvus::grpc::Status>(GetArenaNoVirtual()); - status_ = p; - } - // @@protoc_insertion_point(field_mutable:milvus.grpc.TableName.status) - return status_; -} -inline void TableName::set_allocated_status(::milvus::grpc::Status* status) { - ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaNoVirtual(); - if (message_arena == nullptr) { - delete reinterpret_cast< ::PROTOBUF_NAMESPACE_ID::MessageLite*>(status_); - } - if (status) { - ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = nullptr; - if (message_arena != submessage_arena) { - status = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( - message_arena, status, submessage_arena); - } - - } else { - - } - status_ = status; - // @@protoc_insertion_point(field_set_allocated:milvus.grpc.TableName.status) -} - -// string table_name = 2; +// string table_name = 1; inline void TableName::clear_table_name() { table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); } @@ -2894,60 +3022,219 @@ inline void TableName::set_allocated_table_name(std::string* table_name) { // ------------------------------------------------------------------- -// TableSchema +// TableNameList -// .milvus.grpc.TableName table_name = 1; -inline bool TableSchema::has_table_name() const { - return this != internal_default_instance() && table_name_ != nullptr; +// .milvus.grpc.Status status = 1; +inline bool TableNameList::has_status() const { + return this != internal_default_instance() && status_ != nullptr; } -inline void TableSchema::clear_table_name() { - if (GetArenaNoVirtual() == nullptr && table_name_ != nullptr) { - delete table_name_; - } - table_name_ = nullptr; +inline const ::milvus::grpc::Status& TableNameList::status() const { + const ::milvus::grpc::Status* p = status_; + // @@protoc_insertion_point(field_get:milvus.grpc.TableNameList.status) + return p != nullptr ? *p : *reinterpret_cast( + &::milvus::grpc::_Status_default_instance_); } -inline const ::milvus::grpc::TableName& TableSchema::table_name() const { - const ::milvus::grpc::TableName* p = table_name_; - // @@protoc_insertion_point(field_get:milvus.grpc.TableSchema.table_name) - return p != nullptr ? *p : *reinterpret_cast( - &::milvus::grpc::_TableName_default_instance_); -} -inline ::milvus::grpc::TableName* TableSchema::release_table_name() { - // @@protoc_insertion_point(field_release:milvus.grpc.TableSchema.table_name) +inline ::milvus::grpc::Status* TableNameList::release_status() { + // @@protoc_insertion_point(field_release:milvus.grpc.TableNameList.status) - ::milvus::grpc::TableName* temp = table_name_; - table_name_ = nullptr; + ::milvus::grpc::Status* temp = status_; + status_ = nullptr; return temp; } -inline ::milvus::grpc::TableName* TableSchema::mutable_table_name() { +inline ::milvus::grpc::Status* TableNameList::mutable_status() { - if (table_name_ == nullptr) { - auto* p = CreateMaybeMessage<::milvus::grpc::TableName>(GetArenaNoVirtual()); - table_name_ = p; + if (status_ == nullptr) { + auto* p = CreateMaybeMessage<::milvus::grpc::Status>(GetArenaNoVirtual()); + status_ = p; } - // @@protoc_insertion_point(field_mutable:milvus.grpc.TableSchema.table_name) - return table_name_; + // @@protoc_insertion_point(field_mutable:milvus.grpc.TableNameList.status) + return status_; } -inline void TableSchema::set_allocated_table_name(::milvus::grpc::TableName* table_name) { +inline void TableNameList::set_allocated_status(::milvus::grpc::Status* status) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaNoVirtual(); if (message_arena == nullptr) { - delete table_name_; + delete reinterpret_cast< ::PROTOBUF_NAMESPACE_ID::MessageLite*>(status_); } - if (table_name) { + if (status) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = nullptr; if (message_arena != submessage_arena) { - table_name = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( - message_arena, table_name, submessage_arena); + status = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( + message_arena, status, submessage_arena); } } else { } - table_name_ = table_name; + status_ = status; + // @@protoc_insertion_point(field_set_allocated:milvus.grpc.TableNameList.status) +} + +// repeated string table_names = 2; +inline int TableNameList::table_names_size() const { + return table_names_.size(); +} +inline void TableNameList::clear_table_names() { + table_names_.Clear(); +} +inline const std::string& TableNameList::table_names(int index) const { + // @@protoc_insertion_point(field_get:milvus.grpc.TableNameList.table_names) + return table_names_.Get(index); +} +inline std::string* TableNameList::mutable_table_names(int index) { + // @@protoc_insertion_point(field_mutable:milvus.grpc.TableNameList.table_names) + return table_names_.Mutable(index); +} +inline void TableNameList::set_table_names(int index, const std::string& value) { + // @@protoc_insertion_point(field_set:milvus.grpc.TableNameList.table_names) + table_names_.Mutable(index)->assign(value); +} +inline void TableNameList::set_table_names(int index, std::string&& value) { + // @@protoc_insertion_point(field_set:milvus.grpc.TableNameList.table_names) + table_names_.Mutable(index)->assign(std::move(value)); +} +inline void TableNameList::set_table_names(int index, const char* value) { + GOOGLE_DCHECK(value != nullptr); + table_names_.Mutable(index)->assign(value); + // @@protoc_insertion_point(field_set_char:milvus.grpc.TableNameList.table_names) +} +inline void TableNameList::set_table_names(int index, const char* value, size_t size) { + table_names_.Mutable(index)->assign( + reinterpret_cast(value), size); + // @@protoc_insertion_point(field_set_pointer:milvus.grpc.TableNameList.table_names) +} +inline std::string* TableNameList::add_table_names() { + // @@protoc_insertion_point(field_add_mutable:milvus.grpc.TableNameList.table_names) + return table_names_.Add(); +} +inline void TableNameList::add_table_names(const std::string& value) { + table_names_.Add()->assign(value); + // @@protoc_insertion_point(field_add:milvus.grpc.TableNameList.table_names) +} +inline void TableNameList::add_table_names(std::string&& value) { + table_names_.Add(std::move(value)); + // @@protoc_insertion_point(field_add:milvus.grpc.TableNameList.table_names) +} +inline void TableNameList::add_table_names(const char* value) { + GOOGLE_DCHECK(value != nullptr); + table_names_.Add()->assign(value); + // @@protoc_insertion_point(field_add_char:milvus.grpc.TableNameList.table_names) +} +inline void TableNameList::add_table_names(const char* value, size_t size) { + table_names_.Add()->assign(reinterpret_cast(value), size); + // @@protoc_insertion_point(field_add_pointer:milvus.grpc.TableNameList.table_names) +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& +TableNameList::table_names() const { + // @@protoc_insertion_point(field_list:milvus.grpc.TableNameList.table_names) + return table_names_; +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* +TableNameList::mutable_table_names() { + // @@protoc_insertion_point(field_mutable_list:milvus.grpc.TableNameList.table_names) + return &table_names_; +} + +// ------------------------------------------------------------------- + +// TableSchema + +// .milvus.grpc.Status status = 1; +inline bool TableSchema::has_status() const { + return this != internal_default_instance() && status_ != nullptr; +} +inline const ::milvus::grpc::Status& TableSchema::status() const { + const ::milvus::grpc::Status* p = status_; + // @@protoc_insertion_point(field_get:milvus.grpc.TableSchema.status) + return p != nullptr ? *p : *reinterpret_cast( + &::milvus::grpc::_Status_default_instance_); +} +inline ::milvus::grpc::Status* TableSchema::release_status() { + // @@protoc_insertion_point(field_release:milvus.grpc.TableSchema.status) + + ::milvus::grpc::Status* temp = status_; + status_ = nullptr; + return temp; +} +inline ::milvus::grpc::Status* TableSchema::mutable_status() { + + if (status_ == nullptr) { + auto* p = CreateMaybeMessage<::milvus::grpc::Status>(GetArenaNoVirtual()); + status_ = p; + } + // @@protoc_insertion_point(field_mutable:milvus.grpc.TableSchema.status) + return status_; +} +inline void TableSchema::set_allocated_status(::milvus::grpc::Status* status) { + ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaNoVirtual(); + if (message_arena == nullptr) { + delete reinterpret_cast< ::PROTOBUF_NAMESPACE_ID::MessageLite*>(status_); + } + if (status) { + ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = nullptr; + if (message_arena != submessage_arena) { + status = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( + message_arena, status, submessage_arena); + } + + } else { + + } + status_ = status; + // @@protoc_insertion_point(field_set_allocated:milvus.grpc.TableSchema.status) +} + +// string table_name = 2; +inline void TableSchema::clear_table_name() { + table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline const std::string& TableSchema::table_name() const { + // @@protoc_insertion_point(field_get:milvus.grpc.TableSchema.table_name) + return table_name_.GetNoArena(); +} +inline void TableSchema::set_table_name(const std::string& value) { + + table_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.table_name) +} +inline void TableSchema::set_table_name(std::string&& value) { + + table_name_.SetNoArena( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:milvus.grpc.TableSchema.table_name) +} +inline void TableSchema::set_table_name(const char* value) { + GOOGLE_DCHECK(value != nullptr); + + table_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:milvus.grpc.TableSchema.table_name) +} +inline void TableSchema::set_table_name(const char* value, size_t size) { + + table_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:milvus.grpc.TableSchema.table_name) +} +inline std::string* TableSchema::mutable_table_name() { + + // @@protoc_insertion_point(field_mutable:milvus.grpc.TableSchema.table_name) + return table_name_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline std::string* TableSchema::release_table_name() { + // @@protoc_insertion_point(field_release:milvus.grpc.TableSchema.table_name) + + return table_name_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline void TableSchema::set_allocated_table_name(std::string* table_name) { + if (table_name != nullptr) { + + } else { + + } + table_name_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), table_name); // @@protoc_insertion_point(field_set_allocated:milvus.grpc.TableSchema.table_name) } -// int64 dimension = 2; +// int64 dimension = 3; inline void TableSchema::clear_dimension() { dimension_ = PROTOBUF_LONGLONG(0); } @@ -2961,7 +3248,7 @@ inline void TableSchema::set_dimension(::PROTOBUF_NAMESPACE_ID::int64 value) { // @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.dimension) } -// int64 index_file_size = 3; +// int64 index_file_size = 4; inline void TableSchema::clear_index_file_size() { index_file_size_ = PROTOBUF_LONGLONG(0); } @@ -2975,7 +3262,7 @@ inline void TableSchema::set_index_file_size(::PROTOBUF_NAMESPACE_ID::int64 valu // @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.index_file_size) } -// int32 metric_type = 4; +// int32 metric_type = 5; inline void TableSchema::clear_metric_type() { metric_type_ = 0; } @@ -4048,58 +4335,103 @@ inline void Index::set_nlist(::PROTOBUF_NAMESPACE_ID::int32 value) { // IndexParam -// .milvus.grpc.TableName table_name = 1; -inline bool IndexParam::has_table_name() const { - return this != internal_default_instance() && table_name_ != nullptr; +// .milvus.grpc.Status status = 1; +inline bool IndexParam::has_status() const { + return this != internal_default_instance() && status_ != nullptr; } -inline void IndexParam::clear_table_name() { - if (GetArenaNoVirtual() == nullptr && table_name_ != nullptr) { - delete table_name_; - } - table_name_ = nullptr; +inline const ::milvus::grpc::Status& IndexParam::status() const { + const ::milvus::grpc::Status* p = status_; + // @@protoc_insertion_point(field_get:milvus.grpc.IndexParam.status) + return p != nullptr ? *p : *reinterpret_cast( + &::milvus::grpc::_Status_default_instance_); } -inline const ::milvus::grpc::TableName& IndexParam::table_name() const { - const ::milvus::grpc::TableName* p = table_name_; - // @@protoc_insertion_point(field_get:milvus.grpc.IndexParam.table_name) - return p != nullptr ? *p : *reinterpret_cast( - &::milvus::grpc::_TableName_default_instance_); -} -inline ::milvus::grpc::TableName* IndexParam::release_table_name() { - // @@protoc_insertion_point(field_release:milvus.grpc.IndexParam.table_name) +inline ::milvus::grpc::Status* IndexParam::release_status() { + // @@protoc_insertion_point(field_release:milvus.grpc.IndexParam.status) - ::milvus::grpc::TableName* temp = table_name_; - table_name_ = nullptr; + ::milvus::grpc::Status* temp = status_; + status_ = nullptr; return temp; } -inline ::milvus::grpc::TableName* IndexParam::mutable_table_name() { +inline ::milvus::grpc::Status* IndexParam::mutable_status() { - if (table_name_ == nullptr) { - auto* p = CreateMaybeMessage<::milvus::grpc::TableName>(GetArenaNoVirtual()); - table_name_ = p; + if (status_ == nullptr) { + auto* p = CreateMaybeMessage<::milvus::grpc::Status>(GetArenaNoVirtual()); + status_ = p; } - // @@protoc_insertion_point(field_mutable:milvus.grpc.IndexParam.table_name) - return table_name_; + // @@protoc_insertion_point(field_mutable:milvus.grpc.IndexParam.status) + return status_; } -inline void IndexParam::set_allocated_table_name(::milvus::grpc::TableName* table_name) { +inline void IndexParam::set_allocated_status(::milvus::grpc::Status* status) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaNoVirtual(); if (message_arena == nullptr) { - delete table_name_; + delete reinterpret_cast< ::PROTOBUF_NAMESPACE_ID::MessageLite*>(status_); } - if (table_name) { + if (status) { ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = nullptr; if (message_arena != submessage_arena) { - table_name = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( - message_arena, table_name, submessage_arena); + status = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( + message_arena, status, submessage_arena); } } else { } - table_name_ = table_name; + status_ = status; + // @@protoc_insertion_point(field_set_allocated:milvus.grpc.IndexParam.status) +} + +// string table_name = 2; +inline void IndexParam::clear_table_name() { + table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline const std::string& IndexParam::table_name() const { + // @@protoc_insertion_point(field_get:milvus.grpc.IndexParam.table_name) + return table_name_.GetNoArena(); +} +inline void IndexParam::set_table_name(const std::string& value) { + + table_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:milvus.grpc.IndexParam.table_name) +} +inline void IndexParam::set_table_name(std::string&& value) { + + table_name_.SetNoArena( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:milvus.grpc.IndexParam.table_name) +} +inline void IndexParam::set_table_name(const char* value) { + GOOGLE_DCHECK(value != nullptr); + + table_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:milvus.grpc.IndexParam.table_name) +} +inline void IndexParam::set_table_name(const char* value, size_t size) { + + table_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:milvus.grpc.IndexParam.table_name) +} +inline std::string* IndexParam::mutable_table_name() { + + // @@protoc_insertion_point(field_mutable:milvus.grpc.IndexParam.table_name) + return table_name_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline std::string* IndexParam::release_table_name() { + // @@protoc_insertion_point(field_release:milvus.grpc.IndexParam.table_name) + + return table_name_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline void IndexParam::set_allocated_table_name(std::string* table_name) { + if (table_name != nullptr) { + + } else { + + } + table_name_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), table_name); // @@protoc_insertion_point(field_set_allocated:milvus.grpc.IndexParam.table_name) } -// .milvus.grpc.Index index = 2; +// .milvus.grpc.Index index = 3; inline bool IndexParam::has_index() const { return this != internal_default_instance() && index_ != nullptr; } @@ -4293,6 +4625,8 @@ inline void DeleteByRangeParam::set_allocated_table_name(std::string* table_name // ------------------------------------------------------------------- +// ------------------------------------------------------------------- + // @@protoc_insertion_point(namespace_scope) diff --git a/cpp/src/grpc/milvus.proto b/cpp/src/grpc/milvus.proto index fa3709e4e5..2856dfc6d9 100644 --- a/cpp/src/grpc/milvus.proto +++ b/cpp/src/grpc/milvus.proto @@ -8,18 +8,26 @@ package milvus.grpc; * @brief Table Name */ message TableName { + string table_name = 1; +} + +/** + * @brief Table Name List + */ +message TableNameList { Status status = 1; - string table_name = 2; + repeated string table_names = 2; } /** * @brief Table Schema */ message TableSchema { - TableName table_name = 1; - int64 dimension = 2; - int64 index_file_size = 3; - int32 metric_type = 4; + Status status = 1; + string table_name = 2; + int64 dimension = 3; + int64 index_file_size = 4; + int32 metric_type = 5; } /** @@ -141,8 +149,9 @@ message Index { * @brief Index params */ message IndexParam { - TableName table_name = 1; - Index index = 2; + Status status = 1; + string table_name = 2; + Index index = 3; } /** @@ -264,7 +273,7 @@ service MilvusService { * * @return table names. */ - rpc ShowTables(Command) returns (stream TableName) {} + rpc ShowTables(Command) returns (TableNameList) {} /** * @brief Give the server status diff --git a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp index 6b11946150..16fc2ee168 100644 --- a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp @@ -227,9 +227,9 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::cout << "All tables: " << std::endl; for(auto& table : tables) { int64_t row_count = 0; - conn->DropTable(table); -// stat = conn->CountTable(table, row_count); -// std::cout << "\t" << table << "(" << row_count << " rows)" << std::endl; +// conn->DropTable(table); + stat = conn->CountTable(table, row_count); + std::cout << "\t" << table << "(" << row_count << " rows)" << std::endl; } } @@ -343,8 +343,8 @@ ClientTest::Test(const std::string& address, const std::string& port) { } {//delete table - Status stat = conn->DropTable(TABLE_NAME); - std::cout << "DeleteTable function call status: " << stat.message() << std::endl; +// Status stat = conn->DropTable(TABLE_NAME); +// std::cout << "DeleteTable function call status: " << stat.message() << std::endl; } {//server status diff --git a/cpp/src/sdk/grpc/ClientProxy.cpp b/cpp/src/sdk/grpc/ClientProxy.cpp index 2a87f7f929..3bcfd699fe 100644 --- a/cpp/src/sdk/grpc/ClientProxy.cpp +++ b/cpp/src/sdk/grpc/ClientProxy.cpp @@ -93,7 +93,7 @@ Status ClientProxy::CreateTable(const TableSchema ¶m) { try { ::milvus::grpc::TableSchema schema; - schema.mutable_table_name()->set_table_name(param.table_name); + schema.set_table_name(param.table_name); schema.set_dimension(param.dimension); schema.set_index_file_size(param.index_file_size); schema.set_metric_type((int32_t)param.metric_type); @@ -129,7 +129,7 @@ ClientProxy::CreateIndex(const IndexParam &index_param) { try { //TODO:add index params ::milvus::grpc::IndexParam grpc_index_param; - grpc_index_param.mutable_table_name()->set_table_name(index_param.table_name); + grpc_index_param.set_table_name(index_param.table_name); grpc_index_param.mutable_index()->set_index_type((int32_t)index_param.index_type); grpc_index_param.mutable_index()->set_nlist(index_param.nlist); return client_ptr_->CreateIndex(grpc_index_param); @@ -281,7 +281,7 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch Status status = client_ptr_->DescribeTable(grpc_schema, table_name); - table_schema.table_name = grpc_schema.table_name().table_name(); + table_schema.table_name = grpc_schema.table_name(); table_schema.dimension = grpc_schema.dimension(); table_schema.index_file_size = grpc_schema.index_file_size(); table_schema.metric_type = (MetricType)grpc_schema.metric_type(); @@ -307,7 +307,15 @@ ClientProxy::CountTable(const std::string &table_name, int64_t &row_count) { Status ClientProxy::ShowTables(std::vector &table_array) { try { - return client_ptr_->ShowTables(table_array); + Status status; + milvus::grpc::TableNameList table_name_list; + status = client_ptr_->ShowTables(table_name_list); + + table_array.resize(table_name_list.table_names_size()); + for (uint64_t i = 0; i < table_name_list.table_names_size(); ++i) { + table_array[i] = table_name_list.table_names(i); + } + return status; } catch (std::exception &ex) { return Status(StatusCode::UnknownError, "fail to show tables: " + std::string(ex.what())); diff --git a/cpp/src/sdk/grpc/GrpcClient.cpp b/cpp/src/sdk/grpc/GrpcClient.cpp index 146a96ec29..67bb85da8e 100644 --- a/cpp/src/sdk/grpc/GrpcClient.cpp +++ b/cpp/src/sdk/grpc/GrpcClient.cpp @@ -167,10 +167,10 @@ GrpcClient::DescribeTable(::milvus::grpc::TableSchema& grpc_schema, return Status(StatusCode::RPCFailed, grpc_status.error_message()); } - if (grpc_schema.table_name().status().error_code() != grpc::SUCCESS) { - std::cerr << grpc_schema.table_name().status().reason() << std::endl; + if (grpc_schema.status().error_code() != grpc::SUCCESS) { + std::cerr << grpc_schema.status().reason() << std::endl; return Status(StatusCode::ServerFailed, - grpc_schema.table_name().status().reason()); + grpc_schema.status().reason()); } return Status::OK(); @@ -201,17 +201,10 @@ GrpcClient::CountTable(const std::string& table_name, Status& status) { } Status -GrpcClient::ShowTables(std::vector &table_array) { +GrpcClient::ShowTables(milvus::grpc::TableNameList &table_name_list) { ClientContext context; ::milvus::grpc::Command command; - std::unique_ptr > reader( - stub_->ShowTables(&context, command)); - - ::milvus::grpc::TableName table_name; - while (reader->Read(&table_name)) { - table_array.emplace_back(table_name.table_name()); - } - ::grpc::Status grpc_status = reader->Finish(); + ::grpc::Status grpc_status = stub_->ShowTables(&context, command, &table_name_list); if (!grpc_status.ok()) { std::cerr << "ShowTables gRPC failed!" << std::endl; @@ -219,10 +212,10 @@ GrpcClient::ShowTables(std::vector &table_array) { return Status(StatusCode::RPCFailed, grpc_status.error_message()); } - if (table_name.status().error_code() != grpc::SUCCESS) { - std::cerr << table_name.status().reason() << std::endl; + if (table_name_list.status().error_code() != grpc::SUCCESS) { + std::cerr << table_name_list.status().reason() << std::endl; return Status(StatusCode::ServerFailed, - table_name.status().reason()); + table_name_list.status().reason()); } return Status::OK(); @@ -302,9 +295,9 @@ GrpcClient::DescribeIndex(grpc::TableName &table_name, grpc::IndexParam &index_p std::cerr << "DescribeIndex rpc failed!" << std::endl; return Status(StatusCode::RPCFailed, grpc_status.error_message()); } - if (index_param.mutable_table_name()->status().error_code() != grpc::SUCCESS) { - std::cerr << index_param.mutable_table_name()->status().reason() << std::endl; - return Status(StatusCode::ServerFailed, index_param.mutable_table_name()->status().reason()); + if (index_param.status().error_code() != grpc::SUCCESS) { + std::cerr << index_param.status().reason() << std::endl; + return Status(StatusCode::ServerFailed, index_param.status().reason()); } return Status::OK(); diff --git a/cpp/src/sdk/grpc/GrpcClient.h b/cpp/src/sdk/grpc/GrpcClient.h index ba9ba05f19..08714d10e0 100644 --- a/cpp/src/sdk/grpc/GrpcClient.h +++ b/cpp/src/sdk/grpc/GrpcClient.h @@ -73,7 +73,7 @@ public: CountTable(const std::string& table_name, Status& status); Status - ShowTables(std::vector &table_array); + ShowTables(milvus::grpc::TableNameList &table_name_list); Status Cmd(std::string &result, const std::string& cmd); diff --git a/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp b/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp index 4fe86c3d1d..c85d1b2fcd 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp @@ -122,8 +122,8 @@ GrpcRequestHandler::DescribeTable(::grpc::ServerContext *context, BaseTaskPtr task_ptr = DescribeTableTask::Create(request->table_name(), response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); - response->mutable_table_name()->mutable_status()->set_error_code(grpc_status.error_code()); - response->mutable_table_name()->mutable_status()->set_reason(grpc_status.reason()); + response->mutable_status()->set_error_code(grpc_status.error_code()); + response->mutable_status()->set_reason(grpc_status.reason()); return ::grpc::Status::OK; } @@ -145,17 +145,14 @@ GrpcRequestHandler::CountTable(::grpc::ServerContext *context, ::grpc::Status GrpcRequestHandler::ShowTables(::grpc::ServerContext *context, const ::milvus::grpc::Command *request, - ::grpc::ServerWriter<::milvus::grpc::TableName> *writer) { + ::milvus::grpc::TableNameList *response) { - BaseTaskPtr task_ptr = ShowTablesTask::Create(writer); + BaseTaskPtr task_ptr = ShowTablesTask::Create(response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); - if (grpc_status.error_code() != SERVER_SUCCESS) { - ::grpc::Status status(::grpc::UNKNOWN, grpc_status.reason()); - return status; - } else { - return ::grpc::Status::OK; - } + response->mutable_status()->set_error_code(grpc_status.error_code()); + response->mutable_status()->set_reason(grpc_status.reason()); + return ::grpc::Status::OK; } ::grpc::Status @@ -204,8 +201,8 @@ GrpcRequestHandler::DescribeIndex(::grpc::ServerContext *context, BaseTaskPtr task_ptr = DescribeIndexTask::Create(request->table_name(), response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); - response->mutable_table_name()->mutable_status()->set_reason(grpc_status.reason()); - response->mutable_table_name()->mutable_status()->set_error_code(grpc_status.error_code()); + response->mutable_status()->set_reason(grpc_status.reason()); + response->mutable_status()->set_error_code(grpc_status.error_code()); return ::grpc::Status::OK; } diff --git a/cpp/src/server/grpc_impl/GrpcRequestHandler.h b/cpp/src/server/grpc_impl/GrpcRequestHandler.h index b855cdebcb..a8a70eb88d 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestHandler.h +++ b/cpp/src/server/grpc_impl/GrpcRequestHandler.h @@ -234,7 +234,7 @@ public: ::grpc::Status ShowTables(::grpc::ServerContext *context, const ::milvus::grpc::Command *request, - ::grpc::ServerWriter<::milvus::grpc::TableName> *writer) override; + ::milvus::grpc::TableNameList *table_name_list) override; /** * @brief Give the server status diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index 111104c423..ca6b77e960 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -137,7 +137,7 @@ CreateTableTask::OnExecute() { try { //step 1: check arguments - auto status = ValidationUtil::ValidateTableName(schema_->table_name().table_name()); + auto status = ValidationUtil::ValidateTableName(schema_->table_name()); if (!status.ok()) { return status; } @@ -159,7 +159,7 @@ CreateTableTask::OnExecute() { //step 2: construct table schema engine::meta::TableSchema table_info; - table_info.table_id_ = schema_->table_name().table_name(); + table_info.table_id_ = schema_->table_name(); table_info.dimension_ = (uint16_t) schema_->dimension(); table_info.index_file_size_ = schema_->index_file_size(); table_info.metric_type_ = schema_->metric_type(); @@ -214,7 +214,7 @@ DescribeTableTask::OnExecute() { return status; } - schema_->mutable_table_name()->set_table_name(table_info.table_id_); + schema_->set_table_name(table_info.table_id_); schema_->set_dimension(table_info.dimension_); schema_->set_index_file_size(table_info.index_file_size_); schema_->set_metric_type(table_info.metric_type_); @@ -249,7 +249,7 @@ CreateIndexTask::OnExecute() { TimeRecorder rc("CreateIndexTask"); //step 1: check arguments - std::string table_name_ = index_param_->table_name().table_name(); + std::string table_name_ = index_param_->table_name(); auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { return status; @@ -384,15 +384,15 @@ DropTableTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -ShowTablesTask::ShowTablesTask(::grpc::ServerWriter<::milvus::grpc::TableName> *writer) +ShowTablesTask::ShowTablesTask(::milvus::grpc::TableNameList *table_name_list) : GrpcBaseTask(DDL_DML_TASK_GROUP), - writer_(writer) { + table_name_list_(table_name_list) { } BaseTaskPtr -ShowTablesTask::Create(::grpc::ServerWriter<::milvus::grpc::TableName> *writer) { - return std::shared_ptr(new ShowTablesTask(writer)); +ShowTablesTask::Create(::milvus::grpc::TableNameList *table_name_list) { + return std::shared_ptr(new ShowTablesTask(table_name_list)); } Status @@ -404,11 +404,7 @@ ShowTablesTask::OnExecute() { } for (auto &schema : schema_array) { - ::milvus::grpc::TableName tableName; - tableName.set_table_name(schema.table_id_); - if (!writer_->Write(tableName)) { - return Status(SERVER_WRITE_ERROR, "Write table name failed!"); - } + table_name_list_->add_table_names(schema.table_id_); } return Status::OK(); } @@ -920,7 +916,7 @@ DescribeIndexTask::OnExecute() { return status; } - index_param_->mutable_table_name()->set_table_name(table_name_); + index_param_->set_table_name(table_name_); index_param_->mutable_index()->set_index_type(index.engine_type_); index_param_->mutable_index()->set_nlist(index.nlist_); diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.h b/cpp/src/server/grpc_impl/GrpcRequestTask.h index b7aa4af1d8..a0c4540cb3 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.h +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.h @@ -124,17 +124,17 @@ private: class ShowTablesTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(::grpc::ServerWriter<::milvus::grpc::TableName> *writer); + Create(::milvus::grpc::TableNameList *table_name_list); protected: explicit - ShowTablesTask(::grpc::ServerWriter<::milvus::grpc::TableName> *writer); + ShowTablesTask(::milvus::grpc::TableNameList *table_name_list); Status OnExecute() override; private: - ::grpc::ServerWriter<::milvus::grpc::TableName> *writer_; + ::milvus::grpc::TableNameList *table_name_list_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index c20f58fd3b..66ec94236a 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -72,7 +72,7 @@ set(unittest_files ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp) set(helper_files - ${MILVUS_ENGINE_SRC}/server/ServerConfig.cpp +# ${MILVUS_ENGINE_SRC}/server/ServerConfig.cpp ${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp ${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp ${MILVUS_ENGINE_SRC}/utils/Status.cpp From 48023fbde037c573d887ca34df45efcccd881c7a Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Thu, 26 Sep 2019 11:54:11 +0800 Subject: [PATCH 111/217] MS-579 Server log format: need add month/day in void of the same log name Former-commit-id: 601dc7c3f8d87bae181a3c30eda0bb3b1853e0a3 --- cpp/conf/log_config.template | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/conf/log_config.template b/cpp/conf/log_config.template index cc33899589..277ef910ef 100644 --- a/cpp/conf/log_config.template +++ b/cpp/conf/log_config.template @@ -1,6 +1,6 @@ * GLOBAL: FORMAT = "%datetime | %level | %logger | %msg" - FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-global.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%Y/%M/%d-%H:%m}-global.log" ENABLED = true TO_FILE = true TO_STANDARD_OUTPUT = false @@ -8,12 +8,12 @@ PERFORMANCE_TRACKING = false MAX_LOG_FILE_SIZE = 209715200 ## Throw log files away after 200MB * DEBUG: - FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-debug.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%Y/%M/%d-%H:%m}-debug.log" ENABLED = true * WARNING: - FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-warning.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%Y/%M/%d-%H:%m}-warning.log" * TRACE: - FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-trace.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%Y/%M/%d-%H:%m}-trace.log" * VERBOSE: FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" TO_FILE = false @@ -21,7 +21,7 @@ ## Error logs * ERROR: ENABLED = true - FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-error.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%Y/%M/%d-%H:%m}-error.log" * FATAL: ENABLED = true - FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-fatal.log" \ No newline at end of file + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%Y/%M/%d-%H:%m}-fatal.log" \ No newline at end of file From 99f0602009c51a1248b46a23a41d65a1263f292f Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Thu, 26 Sep 2019 12:04:27 +0800 Subject: [PATCH 112/217] MS-579 change Server log format Former-commit-id: 176378dd8e45401e014e7594e2c4dd23c9268cdc --- cpp/conf/log_config.template | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/conf/log_config.template b/cpp/conf/log_config.template index 277ef910ef..a3fb15af6d 100644 --- a/cpp/conf/log_config.template +++ b/cpp/conf/log_config.template @@ -1,6 +1,6 @@ * GLOBAL: FORMAT = "%datetime | %level | %logger | %msg" - FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%Y/%M/%d-%H:%m}-global.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-global.log" ENABLED = true TO_FILE = true TO_STANDARD_OUTPUT = false @@ -8,12 +8,12 @@ PERFORMANCE_TRACKING = false MAX_LOG_FILE_SIZE = 209715200 ## Throw log files away after 200MB * DEBUG: - FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%Y/%M/%d-%H:%m}-debug.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-debug.log" ENABLED = true * WARNING: - FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%Y/%M/%d-%H:%m}-warning.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-warning.log" * TRACE: - FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%Y/%M/%d-%H:%m}-trace.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-trace.log" * VERBOSE: FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" TO_FILE = false @@ -21,7 +21,7 @@ ## Error logs * ERROR: ENABLED = true - FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%Y/%M/%d-%H:%m}-error.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-error.log" * FATAL: ENABLED = true - FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%Y/%M/%d-%H:%m}-fatal.log" \ No newline at end of file + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log" \ No newline at end of file From 9eb1ebc6776be8bb0084ddc85b6faa6d80c9788a Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 26 Sep 2019 12:23:47 +0800 Subject: [PATCH 113/217] fix unittest build error Former-commit-id: 2bacb09b54ef6e84a9594de5064dc87461d215fc --- .clang-format | 2 +- cpp/unittest/CMakeLists.txt | 2 +- cpp/unittest/db/search_test.cpp | 9 --- cpp/unittest/metrics/metrics_test.cpp | 6 +- cpp/unittest/metrics/prometheus_test.cpp | 6 +- cpp/unittest/scheduler/schedinst_test.cpp | 3 +- cpp/unittest/server/cache_test.cpp | 1 - cpp/unittest/server/config_test.cpp | 94 +++++++++++------------ cpp/unittest/server/rpc_test.cpp | 23 +++--- cpp/unittest/server/util_test.cpp | 5 +- 10 files changed, 68 insertions(+), 83 deletions(-) diff --git a/.clang-format b/.clang-format index 06453dfbb2..819c2d361e 100644 --- a/.clang-format +++ b/.clang-format @@ -17,4 +17,4 @@ --- BasedOnStyle: Google DerivePointerAlignment: false -ColumnLimit: 90 +ColumnLimit: 120 diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index c20f58fd3b..0df8bfd791 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -72,7 +72,7 @@ set(unittest_files ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp) set(helper_files - ${MILVUS_ENGINE_SRC}/server/ServerConfig.cpp + ${MILVUS_ENGINE_SRC}/server/Config.cpp ${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp ${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp ${MILVUS_ENGINE_SRC}/utils/Status.cpp diff --git a/cpp/unittest/db/search_test.cpp b/cpp/unittest/db/search_test.cpp index 0e8f19949f..33917ea1b7 100644 --- a/cpp/unittest/db/search_test.cpp +++ b/cpp/unittest/db/search_test.cpp @@ -22,7 +22,6 @@ #include "scheduler/task/SearchTask.h" #include "utils/TimeRecorder.h" - using namespace zilliz::milvus; namespace { @@ -226,10 +225,6 @@ TEST(DBSearchTest, MERGE_TEST) { } TEST(DBSearchTest, PARALLEL_CLUSTER_TEST) { - server::ServerConfig &config = server::ServerConfig::GetInstance(); - server::ConfigNode& db_config = config.GetConfig(server::CONFIG_DB); - db_config.SetValue(server::CONFIG_DB_PARALLEL_REDUCE, "false");//lvoc cannot work for std::function, set to false - bool ascending = true; std::vector target_ids; std::vector target_distence; @@ -262,10 +257,6 @@ TEST(DBSearchTest, PARALLEL_CLUSTER_TEST) { } TEST(DBSearchTest, PARALLEL_TOPK_TEST) { - server::ServerConfig &config = server::ServerConfig::GetInstance(); - server::ConfigNode& db_config = config.GetConfig(server::CONFIG_DB); - db_config.SetValue(server::CONFIG_DB_PARALLEL_REDUCE, "false");//lvoc cannot work for std::function, set to false - std::vector target_ids; std::vector target_distence; scheduler::ResultSet src_result; diff --git a/cpp/unittest/metrics/metrics_test.cpp b/cpp/unittest/metrics/metrics_test.cpp index 636e2ffbb1..14c1ead3f3 100644 --- a/cpp/unittest/metrics/metrics_test.cpp +++ b/cpp/unittest/metrics/metrics_test.cpp @@ -25,6 +25,7 @@ //#include "prometheus/registry.h" //#include "prometheus/exposer.h" #include +#include #include "metrics/Metrics.h" #include "utils.h" @@ -36,10 +37,9 @@ using namespace zilliz::milvus; TEST_F(MetricTest, METRIC_TEST) { - server::ConfigNode &configNode = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_METRIC); - configNode.SetValue(server::CONFIG_METRIC_COLLECTOR, "zabbix"); + server::Config::GetInstance().SetMetricConfigCollector("zabbix"); server::Metrics::GetInstance(); - configNode.SetValue(server::CONFIG_METRIC_COLLECTOR, "prometheus"); + server::Config::GetInstance().SetMetricConfigCollector("prometheus"); server::Metrics::GetInstance(); server::SystemInfo::GetInstance().Init(); diff --git a/cpp/unittest/metrics/prometheus_test.cpp b/cpp/unittest/metrics/prometheus_test.cpp index b4fa84401a..6dbc05e948 100644 --- a/cpp/unittest/metrics/prometheus_test.cpp +++ b/cpp/unittest/metrics/prometheus_test.cpp @@ -17,6 +17,7 @@ #include "metrics/PrometheusMetrics.h" +#include "server/Config.h" #include #include @@ -24,8 +25,7 @@ using namespace zilliz::milvus; TEST(PrometheusTest, PROMETHEUS_TEST){ - server::ConfigNode &configNode = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_METRIC); - configNode.SetValue(server::CONFIG_METRIC_IS_STARTUP, "on"); + server::Config::GetInstance().SetMetricConfigAutoBootup("on"); server::PrometheusMetrics instance = server::PrometheusMetrics::GetInstance(); instance.Init(); @@ -76,7 +76,7 @@ TEST(PrometheusTest, PROMETHEUS_TEST){ instance.GPUTemperature(); instance.CPUTemperature(); - configNode.SetValue(server::CONFIG_METRIC_IS_STARTUP, "off"); + server::Config::GetInstance().SetMetricConfigAutoBootup("off"); instance.Init(); instance.CPUCoreUsagePercentSet(); instance.GPUTemperature(); diff --git a/cpp/unittest/scheduler/schedinst_test.cpp b/cpp/unittest/scheduler/schedinst_test.cpp index a02e0f1d41..cf1003f5ce 100644 --- a/cpp/unittest/scheduler/schedinst_test.cpp +++ b/cpp/unittest/scheduler/schedinst_test.cpp @@ -18,6 +18,7 @@ #include #include +#include "server/Config.h" #include "scheduler/SchedInst.h" namespace zilliz { @@ -65,7 +66,7 @@ protected: fstream << ss.str(); fstream.close(); - server::ServerConfig::GetInstance().LoadConfigFile(CONFIG_FILE); + server::Config::GetInstance().LoadConfigFile(CONFIG_FILE); } void diff --git a/cpp/unittest/server/cache_test.cpp b/cpp/unittest/server/cache_test.cpp index 6c2160c9ab..290caf4a97 100644 --- a/cpp/unittest/server/cache_test.cpp +++ b/cpp/unittest/server/cache_test.cpp @@ -18,7 +18,6 @@ #include #include "cache/CpuCacheMgr.h" #include "cache/GpuCacheMgr.h" -#include "server/ServerConfig.h" #include "utils/Error.h" #include "wrapper/VecIndex.h" diff --git a/cpp/unittest/server/config_test.cpp b/cpp/unittest/server/config_test.cpp index 65ce562722..f886e4f05a 100644 --- a/cpp/unittest/server/config_test.cpp +++ b/cpp/unittest/server/config_test.cpp @@ -21,6 +21,7 @@ #include "config/ConfigMgr.h" #include "utils/CommonUtil.h" #include "utils/ValidationUtil.h" +#include "server/Config.h" using namespace zilliz::milvus; @@ -105,54 +106,53 @@ TEST(ConfigTest, CONFIG_TEST) { } TEST(ConfigTest, SERVER_CONFIG_TEST) { - server::ServerConfig& config = server::ServerConfig::GetInstance(); - auto status = config.LoadConfigFile(CONFIG_FILE_PATH); + auto status = server::Config::GetInstance().LoadConfigFile(CONFIG_FILE_PATH); ASSERT_TRUE(status.ok()); - status = server::ServerConfig::GetInstance().ValidateConfig(); - ASSERT_TRUE(status.ok()); +// status = server::Config::GetInstance().ValidateConfig(); +// ASSERT_TRUE(status.ok()); +// +// const server::ServerConfig& config_const = config; +// server::ConfigNode node1 = config_const.GetConfig("server_config"); +// server::ConfigNode& node2 = config.GetConfig("cache_config"); +// node1.Combine(node2); +// +// int32_t cap = node1.GetInt32Value("cpu_cache_capacity"); +// ASSERT_GT(cap, 0); +// +// node1.SetValue("bool", "true"); +// bool bt = node1.GetBoolValue("bool"); +// ASSERT_TRUE(bt); - const server::ServerConfig& config_const = config; - server::ConfigNode node1 = config_const.GetConfig("server_config"); - server::ConfigNode& node2 = config.GetConfig("cache_config"); - node1.Combine(node2); - - int32_t cap = node1.GetInt32Value("cpu_cache_capacity"); - ASSERT_GT(cap, 0); - - node1.SetValue("bool", "true"); - bool bt = node1.GetBoolValue("bool"); - ASSERT_TRUE(bt); - - config.PrintAll(); - - unsigned long total_mem = 0, free_mem = 0; - server::CommonUtil::GetSystemMemInfo(total_mem, free_mem); - - size_t gpu_mem = 0; - server::ValidationUtil::GetGpuMemory(0, gpu_mem); - - server::ConfigNode& server_config = config.GetConfig("server_config"); - server::ConfigNode& db_config = config.GetConfig("db_config"); - server::ConfigNode& cache_config = config.GetConfig(server::CONFIG_CACHE); - cache_config.SetValue(server::CACHE_FREE_PERCENT, "2.0"); - status = config.ValidateConfig(); - ASSERT_FALSE(status.ok()); - - size_t cache_cap = 16; - size_t insert_buffer_size = (total_mem - cache_cap*GB + 1*GB)/GB; - db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size)); - cache_config.SetValue(server::CONFIG_CPU_CACHE_CAPACITY, std::to_string(cache_cap)); - status = config.ValidateConfig(); - ASSERT_FALSE(status.ok()); - - cache_cap = total_mem/GB + 2; - cache_config.SetValue(server::CONFIG_CPU_CACHE_CAPACITY, std::to_string(cache_cap)); - status = config.ValidateConfig(); - ASSERT_FALSE(status.ok()); - - insert_buffer_size = total_mem/GB + 2; - db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size)); - status = config.ValidateConfig(); - ASSERT_FALSE(status.ok()); +// server::Config::GetInstance().PrintAll(); +// +// unsigned long total_mem = 0, free_mem = 0; +// server::CommonUtil::GetSystemMemInfo(total_mem, free_mem); +// +// size_t gpu_mem = 0; +// server::ValidationUtil::GetGpuMemory(0, gpu_mem); +// +// server::ConfigNode& server_config = config.GetConfig("server_config"); +// server::ConfigNode& db_config = config.GetConfig("db_config"); +// server::ConfigNode& cache_config = config.GetConfig(server::CONFIG_CACHE); +// cache_config.SetValue(server::CACHE_FREE_PERCENT, "2.0"); +// status = config.ValidateConfig(); +// ASSERT_FALSE(status.ok()); +// +// size_t cache_cap = 16; +// size_t insert_buffer_size = (total_mem - cache_cap*GB + 1*GB)/GB; +// db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size)); +// cache_config.SetValue(server::CONFIG_CPU_CACHE_CAPACITY, std::to_string(cache_cap)); +// status = config.ValidateConfig(); +// ASSERT_FALSE(status.ok()); +// +// cache_cap = total_mem/GB + 2; +// cache_config.SetValue(server::CONFIG_CPU_CACHE_CAPACITY, std::to_string(cache_cap)); +// status = config.ValidateConfig(); +// ASSERT_FALSE(status.ok()); +// +// insert_buffer_size = total_mem/GB + 2; +// db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size)); +// status = config.ValidateConfig(); +// ASSERT_FALSE(status.ok()); } \ No newline at end of file diff --git a/cpp/unittest/server/rpc_test.cpp b/cpp/unittest/server/rpc_test.cpp index fccebefb82..efb71292a4 100644 --- a/cpp/unittest/server/rpc_test.cpp +++ b/cpp/unittest/server/rpc_test.cpp @@ -29,6 +29,7 @@ #include "grpc/gen-status/status.pb.h" #include "server/DBWrapper.h" +#include "server/Config.h" #include "scheduler/SchedInst.h" #include "scheduler/ResourceFactory.h" #include "utils/CommonUtil.h" @@ -65,20 +66,14 @@ class RpcHandlerTest : public testing::Test { engine::DBOptions opt; - server::ConfigNode &db_config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_DB); - db_config.SetValue(server::CONFIG_DB_BACKEND_URL, "sqlite://:@:/"); - db_config.SetValue(server::CONFIG_DB_PATH, "/tmp/milvus_test"); - db_config.SetValue(server::CONFIG_DB_SLAVE_PATH, ""); - db_config.SetValue(server::CONFIG_DB_ARCHIVE_DISK, ""); - db_config.SetValue(server::CONFIG_DB_ARCHIVE_DAYS, ""); + server::Config::GetInstance().SetDBConfigBackendUrl("sqlite://:@:/"); + server::Config::GetInstance().SetDBConfigPath("/tmp/milvus_test"); + server::Config::GetInstance().SetDBConfigSlavePath(""); + server::Config::GetInstance().SetDBConfigArchiveDiskThreshold(""); + server::Config::GetInstance().SetDBConfigArchiveDaysThreshold(""); + server::Config::GetInstance().SetCacheConfigCacheInsertData(""); + server::Config::GetInstance().SetEngineConfigOmpThreadNum(""); - server::ConfigNode &cache_config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); - cache_config.SetValue(server::CONFIG_INSERT_CACHE_IMMEDIATELY, ""); - - server::ConfigNode &engine_config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_ENGINE); - engine_config.SetValue(server::CONFIG_ENGINE_OMP_THREAD_NUM, ""); - - server::ConfigNode &serverConfig = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_SERVER); // serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "cluster"); // DBWrapper::GetInstance().GetInstance().StartService(); // DBWrapper::GetInstance().GetInstance().StopService(); @@ -87,7 +82,7 @@ class RpcHandlerTest : public testing::Test { // DBWrapper::GetInstance().GetInstance().StartService(); // DBWrapper::GetInstance().GetInstance().StopService(); - serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "single"); + server::Config::GetInstance().SetResourceConfigMode("single"); server::DBWrapper::GetInstance().StartService(); //initialize handler, create table diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index 169b2e23eb..1757a7ddfe 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -331,9 +331,8 @@ TEST(ValidationUtilTest, VALIDATE_BOOL_TEST) { } TEST(ValidationUtilTest, VALIDATE_DOUBLE_TEST) { - double ret = 0.0; - ASSERT_EQ(server::ValidationUtil::ValidateStringIsDouble("2.5", ret).code(), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateStringIsDouble("not double", ret).code(), SERVER_SUCCESS); + ASSERT_EQ(server::ValidationUtil::ValidateStringIsFloat("2.5").code(), SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateStringIsFloat("not double").code(), SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_DBURI_TEST) { From 3ed012660ae15a362d624dd4fc5d71230bfe3c74 Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Thu, 26 Sep 2019 15:11:48 +0800 Subject: [PATCH 114/217] fix conflicts Former-commit-id: b76eb04b7a5490d6ac28024f0367580b4fbb7e8a --- cpp/unittest/server/rpc_test.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/unittest/server/rpc_test.cpp b/cpp/unittest/server/rpc_test.cpp index efb71292a4..7895d081a1 100644 --- a/cpp/unittest/server/rpc_test.cpp +++ b/cpp/unittest/server/rpc_test.cpp @@ -90,7 +90,7 @@ class RpcHandlerTest : public testing::Test { ::grpc::ServerContext context; ::milvus::grpc::TableSchema request; ::milvus::grpc::Status status; - request.mutable_table_name()->set_table_name(TABLE_NAME); + request.set_table_name(TABLE_NAME); request.set_dimension(TABLE_DIM); request.set_index_file_size(INDEX_FILE_SIZE); request.set_metric_type(1); @@ -159,10 +159,10 @@ TEST_F(RpcHandlerTest, INDEX_TEST) { ::milvus::grpc::IndexParam request; ::milvus::grpc::Status response; ::grpc::Status grpc_status = handler->CreateIndex(&context, &request, &response); - request.mutable_table_name()->set_table_name("test1"); + request.set_table_name("test1"); handler->CreateIndex(&context, &request, &response); - request.mutable_table_name()->set_table_name(TABLE_NAME); + request.set_table_name(TABLE_NAME); handler->CreateIndex(&context, &request, &response); request.mutable_index()->set_index_type(1); @@ -289,7 +289,7 @@ TEST_F(RpcHandlerTest, TABLES_TEST) { //test invalid table name handler->CreateTable(&context, &tableschema, &response); //test invalid table dimension - tableschema.mutable_table_name()->set_table_name(tablename); + tableschema.set_table_name(tablename); handler->CreateTable(&context, &tableschema, &response); //test invalid index file size tableschema.set_dimension(TABLE_DIM); From 9e4e60f570bca50acd1ba8e80304215ed93ab85b Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 26 Sep 2019 15:12:23 +0800 Subject: [PATCH 115/217] fix ci issue Former-commit-id: 02a03f10641926de5f466449d738700fb22652ed --- ci/jenkinsfile/dev_test.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy index cf8d9b66b0..6fb9388545 100644 --- a/ci/jenkinsfile/dev_test.groovy +++ b/ci/jenkinsfile/dev_test.groovy @@ -3,7 +3,7 @@ timeout(time: 30, unit: 'MINUTES') { dir ("${PROJECT_NAME}_test") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local" } // mysql database backend test From d78d032b829ff91d8cb6980ed850019036730833 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Thu, 26 Sep 2019 17:03:46 +0800 Subject: [PATCH 116/217] MS-565 remove jsoncons v1 Former-commit-id: cedf253075a00eb4a7d999ffcceb90f8fa795f2e --- cpp/src/core/CMakeLists.txt | 2 +- cpp/src/core/knowhere/CMakeLists.txt | 2 +- .../knowhere/adapter/SptagAdapter.cpp | 3 +- .../core/knowhere/knowhere/common/Config.h | 33 +- .../index/vector_index/IndexGPUIVF.cpp | 18 +- .../index/vector_index/IndexGPUIVFPQ.cpp | 23 +- .../index/vector_index/IndexGPUIVFSQ.cpp | 14 +- .../index/vector_index/IndexIDMAP.cpp | 14 +- .../knowhere/index/vector_index/IndexIVF.cpp | 36 +- .../index/vector_index/IndexIVFPQ.cpp | 23 +- .../index/vector_index/IndexIVFSQ.cpp | 13 +- .../knowhere/index/vector_index/IndexKDT.cpp | 4 +- .../knowhere/index/vector_index/IndexNSG.cpp | 54 +- .../knowhere/index/vector_index/VectorIndex.h | 1 + .../vector_index/helpers/IndexParameter.cpp | 35 + .../vector_index/helpers/IndexParameter.h | 142 + .../vector_index/helpers/KDTParameterMgr.cpp | 2 +- cpp/src/core/test/CMakeLists.txt | 64 +- cpp/src/core/test/test_idmap.cpp | 61 +- cpp/src/core/test/test_ivf.cpp | 295 +- cpp/src/core/test/test_kdt.cpp | 49 +- .../thirdparty/jsoncons-0.126.0/.doozer.json | 43 - .../thirdparty/jsoncons-0.126.0/.gitignore | 30 - .../thirdparty/jsoncons-0.126.0/.travis.yml | 240 - .../jsoncons-0.126.0/CMakeLists.txt | 73 - .../thirdparty/jsoncons-0.126.0/ChangeLog.md | 2129 ------- .../core/thirdparty/jsoncons-0.126.0/LICENSE | 28 - .../thirdparty/jsoncons-0.126.0/README.md | 951 ---- .../thirdparty/jsoncons-0.126.0/Roadmap.md | 15 - .../jsoncons-0.126.0/acknowledgements.md | 76 - .../thirdparty/jsoncons-0.126.0/appveyor.yml | 86 - .../core/thirdparty/jsoncons-0.126.0/build.sh | 5 - .../build_files/cmake/Darwin.cmake | 7 - .../build_files/cmake/Linux.cmake | 16 - .../build_files/cmake/Windows.cmake | 6 - .../build_files/cmake/config.cmake.in | 13 - .../jsoncons-0.126.0/css/jsoncons.css | 1 - .../jsoncons-0.126.0/doc/Examples.md | 1554 ----- .../thirdparty/jsoncons-0.126.0/doc/Home.md | 64 - .../jsoncons-0.126.0/doc/Pages/index.md | 812 --- .../jsoncons-0.126.0/doc/Tutorials/Basics.md | 448 -- .../doc/Tutorials/Custom Allocators.md | 156 - .../doc/Tutorials/Unicode support.md | 182 - .../thirdparty/jsoncons-0.126.0/doc/build.md | 6 - .../doc/ref/big_integer_chars_format.md | 101 - .../jsoncons-0.126.0/doc/ref/bignum.md | 279 - .../jsoncons-0.126.0/doc/ref/bson/bson.md | 29 - .../doc/ref/bson/bson_encoder.md | 147 - .../doc/ref/bson/decode_bson.md | 30 - .../doc/ref/bson/diagrams/bson_encoder.png | Bin 5216 -> 0 bytes .../doc/ref/bson/diagrams/bson_encoder.xml | 1 - .../doc/ref/bson/encode_bson.md | 25 - .../jsoncons-0.126.0/doc/ref/byte_string.md | 130 - .../doc/ref/byte_string_chars_format.md | 58 - .../jsoncons-0.126.0/doc/ref/cbor/cbor.md | 175 - .../doc/ref/cbor/cbor_decode_options.md | 19 - .../doc/ref/cbor/cbor_encode_options.md | 20 - .../doc/ref/cbor/cbor_encoder.md | 197 - .../doc/ref/cbor/cbor_options.md | 43 - .../doc/ref/cbor/decode_cbor.md | 240 - .../doc/ref/cbor/diagrams/cbor_encoder.xml | 1 - .../doc/ref/cbor/diagrams/cbor_options.png | Bin 5744 -> 0 bytes .../doc/ref/cbor/diagrams/cbor_options.xml | 1 - .../doc/ref/cbor/encode_cbor.md | 273 - .../jsoncons-0.126.0/doc/ref/chars_format.md | 17 - .../jsoncons-0.126.0/doc/ref/csv/csv.md | 175 - .../doc/ref/csv/csv_decode_options.md | 67 - .../doc/ref/csv/csv_encode_options.md | 33 - .../doc/ref/csv/csv_encoder.md | 260 - .../doc/ref/csv/csv_options.md | 111 - .../doc/ref/csv/csv_reader.md | 432 -- .../doc/ref/csv/decode_csv.md | 338 -- .../doc/ref/csv/diagrams/csv_encoder.png | Bin 5268 -> 0 bytes .../doc/ref/csv/diagrams/csv_encoder.xml | 1 - .../doc/ref/csv/diagrams/csv_options.png | Bin 6333 -> 0 bytes .../doc/ref/csv/diagrams/csv_options.xml | 1 - .../doc/ref/csv/encode_csv.md | 71 - .../jsoncons-0.126.0/doc/ref/data-model.md | 209 - .../jsoncons-0.126.0/doc/ref/decode_json.md | 78 - .../doc/ref/default_parse_error_handler.md | 21 - .../jsoncons-0.126.0/doc/ref/deprecated.md | 110 - .../doc/ref/diagrams/json_decoder.png | Bin 4080 -> 0 bytes .../doc/ref/diagrams/json_decoder.xml | 1 - .../doc/ref/diagrams/json_encoder.png | Bin 5225 -> 0 bytes .../doc/ref/diagrams/json_encoder.xml | 1 - .../doc/ref/diagrams/json_filter.png | Bin 4163 -> 0 bytes .../doc/ref/diagrams/json_filter.xml | 1 - .../doc/ref/diagrams/json_options.png | Bin 6279 -> 0 bytes .../doc/ref/diagrams/json_options.xml | 1 - .../diagrams/rename_object_member_filter.png | Bin 5760 -> 0 bytes .../diagrams/rename_object_member_filter.xml | 1 - .../doc/ref/diagrams/ser_error.png | Bin 9729 -> 0 bytes .../doc/ref/diagrams/ser_error.xml | 1 - .../jsoncons-0.126.0/doc/ref/encode_json.md | 200 - .../jsoncons-0.126.0/doc/ref/indenting.md | 13 - .../jsoncons-0.126.0/doc/ref/json.md | 397 -- .../doc/ref/json/array_range.md | 45 - .../jsoncons-0.126.0/doc/ref/json/as.md | 120 - .../doc/ref/json/constructor.md | 150 - .../doc/ref/json/destructor.md | 9 - .../jsoncons-0.126.0/doc/ref/json/dump.md | 177 - .../jsoncons-0.126.0/doc/ref/json/emplace.md | 44 - .../doc/ref/json/emplace_back.md | 35 - .../jsoncons-0.126.0/doc/ref/json/erase.md | 29 - .../jsoncons-0.126.0/doc/ref/json/insert.md | 125 - .../doc/ref/json/insert_or_assign.md | 36 - .../jsoncons-0.126.0/doc/ref/json/is.md | 107 - .../doc/ref/json/key_value.md | 53 - .../doc/ref/json/make_array.md | 94 - .../jsoncons-0.126.0/doc/ref/json/merge.md | 60 - .../doc/ref/json/merge_or_update.md | 59 - .../doc/ref/json/object_range.md | 63 - .../doc/ref/json/operator=.md | 16 - .../jsoncons-0.126.0/doc/ref/json/parse.md | 113 - .../doc/ref/json/push_back.md | 50 - .../doc/ref/json/try_emplace.md | 57 - .../doc/ref/json_content_handler.md | 245 - .../jsoncons-0.126.0/doc/ref/json_cursor.md | 358 -- .../doc/ref/json_decode_options.md | 47 - .../jsoncons-0.126.0/doc/ref/json_decoder.md | 39 - .../doc/ref/json_encode_options.md | 105 - .../jsoncons-0.126.0/doc/ref/json_encoder.md | 160 - .../jsoncons-0.126.0/doc/ref/json_error.md | 38 - .../jsoncons-0.126.0/doc/ref/json_filter.md | 210 - .../jsoncons-0.126.0/doc/ref/json_options.md | 390 -- .../jsoncons-0.126.0/doc/ref/json_parser.md | 272 - .../jsoncons-0.126.0/doc/ref/json_reader.md | 239 - .../doc/ref/json_type_traits.md | 826 --- .../doc/ref/jsonpatch/apply_patch.md | 100 - .../jsonpatch/diagrams/jsonpatch_error.png | Bin 9022 -> 0 bytes .../jsonpatch/diagrams/jsonpatch_error.xml | 1 - .../doc/ref/jsonpatch/from_diff.md | 72 - .../doc/ref/jsonpatch/jsonpatch.md | 87 - .../doc/ref/jsonpatch/jsonpatch_errc.md | 25 - .../doc/ref/jsonpatch/jsonpatch_error.md | 65 - .../ref/jsonpath/diagrams/jsonpath_error.png | Bin 8983 -> 0 bytes .../ref/jsonpath/diagrams/jsonpath_error.xml | 1 - .../doc/ref/jsonpath/json_query.md | 356 -- .../doc/ref/jsonpath/json_replace.md | 112 - .../ref/jsonpath/jsoncons-jsonpath-abnf.md | 48 - .../doc/ref/jsonpath/jsonpath.md | 225 - .../doc/ref/jsonpath/jsonpath_error.md | 39 - .../doc/ref/jsonpointer/address.md | 206 - .../doc/ref/jsonpointer/address.md.orig | 275 - .../doc/ref/jsonpointer/contains.md | 55 - .../diagrams/jsonpointer_error.html | 12 - .../diagrams/jsonpointer_error.png | Bin 9017 -> 0 bytes .../diagrams/jsonpointer_error.xml | 1 - .../doc/ref/jsonpointer/get.md | 284 - .../doc/ref/jsonpointer/get.md.orig | 288 - .../doc/ref/jsonpointer/insert.md | 188 - .../doc/ref/jsonpointer/insert_or_assign.md | 67 - .../doc/ref/jsonpointer/jsonpointer.md | 153 - .../doc/ref/jsonpointer/jsonpointer_errc.md | 26 - .../doc/ref/jsonpointer/jsonpointer_error.md | 55 - .../doc/ref/jsonpointer/pointer.md | 200 - .../doc/ref/jsonpointer/remove.md | 92 - .../doc/ref/jsonpointer/replace.md | 100 - .../doc/ref/line_split_kind.md | 6 - .../doc/ref/msgpack/decode_msgpack.md | 30 - .../ref/msgpack/diagrams/msgpack_encoder.png | Bin 5466 -> 0 bytes .../ref/msgpack/diagrams/msgpack_encoder.xml | 1 - .../doc/ref/msgpack/encode_msgpack.md | 103 - .../doc/ref/msgpack/msgpack.md | 94 - .../doc/ref/msgpack/msgpack_encoder.md | 100 - .../jsoncons-0.126.0/doc/ref/ojson.md | 87 - .../doc/ref/parse_error_handler.md | 42 - .../doc/ref/rename_object_member_filter.md | 55 - .../jsoncons-0.126.0/doc/ref/semantic_tag.md | 19 - .../jsoncons-0.126.0/doc/ref/ser_context.md | 22 - .../jsoncons-0.126.0/doc/ref/ser_error.md | 68 - .../jsoncons-0.126.0/doc/ref/spaces_option.md | 12 - .../doc/ref/staj_array_iterator.md | 192 - .../jsoncons-0.126.0/doc/ref/staj_event.md | 40 - .../doc/ref/staj_event_type.md | 27 - .../jsoncons-0.126.0/doc/ref/staj_filter.md | 20 - .../doc/ref/staj_object_iterator.md | 131 - .../jsoncons-0.126.0/doc/ref/staj_reader.md | 59 - .../doc/ref/ubjson/decode_ubjson.md | 30 - .../ref/ubjson/diagrams/ubjson_encoder.png | Bin 5346 -> 0 bytes .../ref/ubjson/diagrams/ubjson_encoder.xml | 1 - .../doc/ref/ubjson/encode_ubjson.md | 25 - .../jsoncons-0.126.0/doc/ref/ubjson/ubjson.md | 179 - .../doc/ref/ubjson/ubjson_encoder.md | 100 - .../jsoncons-0.126.0/doc/ref/wjson.md | 27 - .../jsoncons-0.126.0/doc/ref/wjson_encoder.md | 15 - .../jsoncons-0.126.0/doc/ref/wjson_options.md | 16 - .../jsoncons-0.126.0/doc/ref/wjson_reader.md | 16 - .../jsoncons-0.126.0/doc/ref/wojson.md | 31 - .../examples/input/address-book.json | 13 - .../examples/input/booklist.json | 28 - .../examples/input/books.json | 30 - .../examples/input/countries.json | 7 - .../examples/input/employees.json | 30 - .../examples/input/multiple-json-objects.json | 3 - .../jsoncons-0.126.0/examples/input/sales.csv | 6 - .../jsoncons-0.126.0/examples/input/tasks.csv | 6 - .../examples/output/booklist.json | 37 - .../examples/output/booklist2.json | 37 - .../examples/output/export_settings.json | 1 - .../examples/output/new-address-book1.json | 13 - .../examples/output/new-address-book2.json | 13 - .../examples/output/tasks.csv | 6 - .../examples/output/tasks.json | 33 - .../jsoncons-0.126.0/examples/output/xxx.txt | 3 - .../examples/src/array_examples.cpp | 227 - .../examples/src/basics_examples.cpp | 171 - .../examples/src/basics_wexamples.cpp | 170 - .../examples/src/bson_examples.cpp | 54 - .../examples/src/byte_string_examples.cpp | 93 - .../examples/src/cbor_examples.cpp | 400 -- .../examples/src/container_examples.cpp | 88 - .../examples/src/csv_examples.cpp | 409 -- .../examples/src/data_model_examples.cpp | 126 - .../examples/src/example_types.hpp | 105 - .../examples/src/examples.cpp | 588 -- .../examples/src/json_accessor_examples.cpp | 122 - .../examples/src/json_cursor_examples.cpp | 143 - .../examples/src/json_filter_examples.cpp | 128 - .../examples/src/json_parser_examples.cpp | 152 - .../examples/src/jsonpatch_examples.cpp | 116 - .../examples/src/jsonpath_examples.cpp | 248 - .../examples/src/jsonpointer_examples.cpp | 474 -- .../src/jsonpointer_examples.cpp.orig | 494 -- .../examples/src/msgpack_examples.cpp | 92 - .../examples/src/ojson_examples.cpp | 45 - .../examples/src/polymorphic_examples.cpp | 181 - .../examples/src/readme_examples.cpp | 154 - .../examples/src/serialization_examples.cpp | 461 -- .../examples/src/staj_iterator_examples.cpp | 135 - .../src/type_extensibility_examples.cpp | 162 - .../src/type_extensibility_examples2.cpp | 134 - .../examples/src/ubjson_examples.cpp | 84 - .../examples/src/unicode_examples.cpp | 32 - .../examples/src/wjson_examples.cpp | 85 - .../interprocess_allocator/shared_memory.cpp | 107 - .../more_examples/src/extensibility.cpp | 240 - .../more_examples/src/more_examples.cpp | 22 - .../more_examples/src/pool_allocator.cpp | 33 - .../include/jsoncons/basic_json.hpp | 4984 ----------------- .../include/jsoncons/bignum.hpp | 1604 ------ .../include/jsoncons/byte_string.hpp | 710 --- .../include/jsoncons/config/binary_detail.hpp | 547 -- .../jsoncons/config/jsoncons_config.hpp | 114 - .../include/jsoncons/config/version.hpp | 40 - .../include/jsoncons/detail/grisu3.hpp | 405 -- .../jsoncons/detail/heap_only_string.hpp | 156 - .../include/jsoncons/detail/parse_number.hpp | 526 -- .../include/jsoncons/detail/print_number.hpp | 344 -- .../include/jsoncons/detail/string_view.hpp | 525 -- .../include/jsoncons/detail/type_traits.hpp | 228 - .../include/jsoncons/encode_decode_json.hpp | 280 - .../include/jsoncons/json.hpp | 14 - .../include/jsoncons/json_container_types.hpp | 1962 ------- .../include/jsoncons/json_content_handler.hpp | 431 -- .../jsoncons/json_conversion_traits.hpp | 196 - .../include/jsoncons/json_cursor.hpp | 680 --- .../include/jsoncons/json_decoder.hpp | 357 -- .../include/jsoncons/json_encoder.hpp | 1470 ----- .../include/jsoncons/json_error.hpp | 154 - .../include/jsoncons/json_exception.hpp | 183 - .../include/jsoncons/json_filter.hpp | 382 -- .../include/jsoncons/json_fwd.hpp | 23 - .../include/jsoncons/json_options.hpp | 837 --- .../include/jsoncons/json_parser.hpp | 2841 ---------- .../include/jsoncons/json_pull_reader.hpp | 13 - .../include/jsoncons/json_reader.hpp | 514 -- .../include/jsoncons/json_serializer.hpp | 12 - .../include/jsoncons/json_type_traits.hpp | 1047 ---- .../jsoncons/json_type_traits_macros.hpp | 181 - .../include/jsoncons/parse_error_handler.hpp | 74 - .../include/jsoncons/pretty_print.hpp | 89 - .../include/jsoncons/result.hpp | 284 - .../include/jsoncons/ser_context.hpp | 50 - .../include/jsoncons/source.hpp | 746 --- .../include/jsoncons/staj_iterator.hpp | 354 -- .../include/jsoncons/staj_reader.hpp | 434 -- .../include/jsoncons/unicode_traits.hpp | 1492 ----- .../include/jsoncons_ext/bson/bson.hpp | 106 - .../include/jsoncons_ext/bson/bson_detail.hpp | 42 - .../jsoncons_ext/bson/bson_encoder.hpp | 342 -- .../include/jsoncons_ext/bson/bson_error.hpp | 70 - .../include/jsoncons_ext/bson/bson_reader.hpp | 297 - .../include/jsoncons_ext/cbor/cbor.hpp | 130 - .../include/jsoncons_ext/cbor/cbor_detail.hpp | 72 - .../jsoncons_ext/cbor/cbor_encoder.hpp | 1062 ---- .../include/jsoncons_ext/cbor/cbor_error.hpp | 96 - .../jsoncons_ext/cbor/cbor_options.hpp | 66 - .../include/jsoncons_ext/cbor/cbor_reader.hpp | 1491 ----- .../include/jsoncons_ext/csv/csv.hpp | 14 - .../include/jsoncons_ext/csv/csv_encoder.hpp | 619 -- .../include/jsoncons_ext/csv/csv_error.hpp | 77 - .../include/jsoncons_ext/csv/csv_options.hpp | 771 --- .../include/jsoncons_ext/csv/csv_parser.hpp | 1357 ----- .../include/jsoncons_ext/csv/csv_reader.hpp | 328 -- .../jsoncons_ext/csv/csv_serializer.hpp | 12 - .../jsoncons_ext/jsonpatch/jsonpatch.hpp | 492 -- .../jsonpatch/jsonpatch_error.hpp | 114 - .../jsoncons_ext/jsonpath/json_query.hpp | 1808 ------ .../jsoncons_ext/jsonpath/jsonpath_error.hpp | 206 - .../jsoncons_ext/jsonpath/jsonpath_filter.hpp | 1953 ------- .../jsonpath/jsonpath_function.hpp | 225 - .../jsoncons_ext/jsonpointer/jsonpointer.hpp | 959 ---- .../jsonpointer/jsonpointer.hpp.orig | 1060 ---- .../jsonpointer/jsonpointer_error.hpp | 109 - .../include/jsoncons_ext/msgpack/msgpack.hpp | 126 - .../jsoncons_ext/msgpack/msgpack_detail.hpp | 50 - .../jsoncons_ext/msgpack/msgpack_encoder.hpp | 459 -- .../jsoncons_ext/msgpack/msgpack_error.hpp | 82 - .../jsoncons_ext/msgpack/msgpack_reader.hpp | 757 --- .../include/jsoncons_ext/ubjson/ubjson.hpp | 106 - .../jsoncons_ext/ubjson/ubjson_detail.hpp | 42 - .../jsoncons_ext/ubjson/ubjson_encoder.hpp | 433 -- .../jsoncons_ext/ubjson/ubjson_error.hpp | 89 - .../jsoncons_ext/ubjson/ubjson_reader.hpp | 589 -- .../jsoncons-0.126.0/tests/CMakeLists.txt | 99 - .../jsoncons-0.126.0/tests/Readme.txt | 37 - .../input/JSONPathTestSuite/document.json | 65 - .../tests/input/JSONPathTestSuite/query1.json | 30 - .../input/JSONPathTestSuite/query1.jsonpath | 1 - .../tests/input/JSONPathTestSuite/query2.json | 4 - .../input/JSONPathTestSuite/query2.jsonpath | 1 - .../tests/input/JSONPathTestSuite/query3.json | 5 - .../input/JSONPathTestSuite/query3.jsonpath | 1 - .../tests/input/JSONPathTestSuite/query4.json | 16 - .../input/JSONPathTestSuite/query4.jsonpath | 1 - .../tests/input/JSONPathTestSuite/query5.json | 6 - .../input/JSONPathTestSuite/query5.jsonpath | 1 - .../tests/input/JSONPathTestSuite/query6.json | 8 - .../input/JSONPathTestSuite/query6.jsonpath | 1 - .../tests/input/JSONPathTestSuite/query7.json | 15 - .../input/JSONPathTestSuite/query7.jsonpath | 1 - .../tests/input/JSONPathTestSuite/query8.json | 16 - .../input/JSONPathTestSuite/query8.jsonpath | 1 - .../tests/input/JSONPathTestSuite/query9.json | 6 - .../input/JSONPathTestSuite/query9.jsonpath | 1 - .../i_number_double_huge_neg_exp.json | 1 - .../JSONTestSuite/i_number_huge_exp.json | 1 - .../i_number_neg_int_huge_exp.json | 1 - .../i_number_pos_double_huge_exp.json | 1 - .../i_number_real_neg_overflow.json | 1 - .../i_number_real_pos_overflow.json | 1 - .../i_number_real_underflow.json | 1 - .../i_number_too_big_neg_int.json | 1 - .../i_number_too_big_pos_int.json | 1 - .../i_number_very_big_negative_int.json | 1 - .../i_object_key_lone_2nd_surrogate.json | 1 - ..._string_1st_surrogate_but_2nd_missing.json | 1 - ...tring_1st_valid_surrogate_2nd_invalid.json | 1 - .../i_string_UTF-16LE_with_BOM.json | Bin 12 -> 0 bytes .../i_string_UTF-8_invalid_sequence.json | 1 - .../i_string_UTF8_surrogate_U+D800.json | 1 - ...incomplete_surrogate_and_escape_valid.json | 1 - .../i_string_incomplete_surrogate_pair.json | 1 - ...ng_incomplete_surrogates_escape_valid.json | 1 - .../i_string_invalid_lonely_surrogate.json | 1 - .../i_string_invalid_surrogate.json | 1 - .../JSONTestSuite/i_string_invalid_utf-8.json | 1 - .../i_string_inverted_surrogates_U+1D11E.json | 1 - .../JSONTestSuite/i_string_iso_latin_1.json | 1 - .../i_string_lone_second_surrogate.json | 1 - .../i_string_lone_utf8_continuation_byte.json | 1 - .../i_string_not_in_unicode_range.json | 1 - .../i_string_overlong_sequence_2_bytes.json | 1 - .../i_string_overlong_sequence_6_bytes.json | 1 - ...string_overlong_sequence_6_bytes_null.json | 1 - .../i_string_truncated-utf-8.json | 1 - .../i_string_utf16BE_no_BOM.json | Bin 10 -> 0 bytes .../i_string_utf16LE_no_BOM.json | Bin 10 -> 0 bytes .../i_structure_500_nested_arrays.json | 1 - .../i_structure_UTF-8_BOM_empty_object.json | 1 - .../n_array_1_true_without_comma.json | 1 - .../JSONTestSuite/n_array_a_invalid_utf8.json | 1 - .../n_array_colon_instead_of_comma.json | 1 - .../n_array_comma_after_close.json | 1 - .../n_array_comma_and_number.json | 1 - .../JSONTestSuite/n_array_double_comma.json | 1 - .../n_array_double_extra_comma.json | 1 - .../JSONTestSuite/n_array_extra_close.json | 1 - .../JSONTestSuite/n_array_extra_comma.json | 1 - .../JSONTestSuite/n_array_incomplete.json | 1 - .../n_array_incomplete_invalid_value.json | 1 - .../n_array_inner_array_no_comma.json | 1 - .../JSONTestSuite/n_array_invalid_utf8.json | 1 - .../n_array_items_separated_by_semicolon.json | 1 - .../JSONTestSuite/n_array_just_comma.json | 1 - .../JSONTestSuite/n_array_just_minus.json | 1 - .../JSONTestSuite/n_array_missing_value.json | 1 - .../n_array_newlines_unclosed.json | 3 - .../n_array_number_and_comma.json | 1 - .../n_array_number_and_several_commas.json | 1 - .../n_array_spaces_vertical_tab_formfeed.json | 1 - .../JSONTestSuite/n_array_star_inside.json | 1 - .../input/JSONTestSuite/n_array_unclosed.json | 1 - .../n_array_unclosed_trailing_comma.json | 1 - .../n_array_unclosed_with_new_lines.json | 3 - .../n_array_unclosed_with_object_inside.json | 1 - .../JSONTestSuite/n_incomplete_false.json | 1 - .../JSONTestSuite/n_incomplete_null.json | 1 - .../JSONTestSuite/n_incomplete_true.json | 1 - .../n_multidigit_number_then_00.json | Bin 4 -> 0 bytes .../input/JSONTestSuite/n_number_++.json | 1 - .../input/JSONTestSuite/n_number_+1.json | 1 - .../input/JSONTestSuite/n_number_+Inf.json | 1 - .../input/JSONTestSuite/n_number_-01.json | 1 - .../input/JSONTestSuite/n_number_-1.0..json | 1 - .../input/JSONTestSuite/n_number_-2..json | 1 - .../input/JSONTestSuite/n_number_-NaN.json | 1 - .../input/JSONTestSuite/n_number_.-1.json | 1 - .../input/JSONTestSuite/n_number_.2e-3.json | 1 - .../input/JSONTestSuite/n_number_0.1.2.json | 1 - .../input/JSONTestSuite/n_number_0.3e+.json | 1 - .../input/JSONTestSuite/n_number_0.3e.json | 1 - .../input/JSONTestSuite/n_number_0.e1.json | 1 - .../JSONTestSuite/n_number_0_capital_E+.json | 1 - .../JSONTestSuite/n_number_0_capital_E.json | 1 - .../input/JSONTestSuite/n_number_0e+.json | 1 - .../input/JSONTestSuite/n_number_0e.json | 1 - .../input/JSONTestSuite/n_number_1.0e+.json | 1 - .../input/JSONTestSuite/n_number_1.0e-.json | 1 - .../input/JSONTestSuite/n_number_1.0e.json | 1 - .../input/JSONTestSuite/n_number_1_000.json | 1 - .../input/JSONTestSuite/n_number_1eE2.json | 1 - .../input/JSONTestSuite/n_number_2.e+3.json | 1 - .../input/JSONTestSuite/n_number_2.e-3.json | 1 - .../input/JSONTestSuite/n_number_2.e3.json | 1 - .../input/JSONTestSuite/n_number_9.e+.json | 1 - .../input/JSONTestSuite/n_number_Inf.json | 1 - .../input/JSONTestSuite/n_number_NaN.json | 1 - .../n_number_U+FF11_fullwidth_digit_one.json | 1 - .../JSONTestSuite/n_number_expression.json | 1 - .../JSONTestSuite/n_number_hex_1_digit.json | 1 - .../JSONTestSuite/n_number_hex_2_digits.json | 1 - .../JSONTestSuite/n_number_infinity.json | 1 - .../JSONTestSuite/n_number_invalid+-.json | 1 - .../n_number_invalid-negative-real.json | 1 - .../n_number_invalid-utf-8-in-bigger-int.json | 1 - .../n_number_invalid-utf-8-in-exponent.json | 1 - .../n_number_invalid-utf-8-in-int.json | 1 - .../n_number_minus_infinity.json | 1 - ...mber_minus_sign_with_trailing_garbage.json | 1 - .../JSONTestSuite/n_number_minus_space_1.json | 1 - .../n_number_neg_int_starting_with_zero.json | 1 - .../n_number_neg_real_without_int_part.json | 1 - .../n_number_neg_with_garbage_at_end.json | 1 - .../n_number_real_garbage_after_e.json | 1 - ...number_real_with_invalid_utf8_after_e.json | 1 - ...n_number_real_without_fractional_part.json | 1 - .../n_number_starting_with_dot.json | 1 - .../JSONTestSuite/n_number_with_alpha.json | 1 - .../n_number_with_alpha_char.json | 1 - .../n_number_with_leading_zero.json | 1 - .../JSONTestSuite/n_object_bad_value.json | 1 - .../JSONTestSuite/n_object_bracket_key.json | 1 - .../n_object_comma_instead_of_colon.json | 1 - .../JSONTestSuite/n_object_double_colon.json | 1 - .../input/JSONTestSuite/n_object_emoji.json | 1 - .../n_object_garbage_at_end.json | 1 - .../n_object_key_with_single_quotes.json | 1 - ...uation_byte_in_key_and_trailing_comma.json | 1 - .../JSONTestSuite/n_object_missing_colon.json | 1 - .../JSONTestSuite/n_object_missing_key.json | 1 - .../n_object_missing_semicolon.json | 1 - .../JSONTestSuite/n_object_missing_value.json | 1 - .../JSONTestSuite/n_object_no-colon.json | 1 - .../n_object_non_string_key.json | 1 - ...on_string_key_but_huge_number_instead.json | 1 - .../n_object_repeated_null_null.json | 1 - .../n_object_several_trailing_commas.json | 1 - .../JSONTestSuite/n_object_single_quote.json | 1 - .../n_object_trailing_comma.json | 1 - .../n_object_trailing_comment.json | 1 - .../n_object_trailing_comment_open.json | 1 - .../n_object_trailing_comment_slash_open.json | 1 - ...railing_comment_slash_open_incomplete.json | 1 - .../n_object_two_commas_in_a_row.json | 1 - .../JSONTestSuite/n_object_unquoted_key.json | 1 - .../n_object_unterminated-value.json | 1 - .../n_object_with_single_string.json | 1 - .../n_object_with_trailing_garbage.json | 1 - .../input/JSONTestSuite/n_single_space.json | 1 - .../n_string_1_surrogate_then_escape.json | 1 - .../n_string_1_surrogate_then_escape_u.json | 1 - .../n_string_1_surrogate_then_escape_u1.json | 1 - .../n_string_1_surrogate_then_escape_u1x.json | 1 - .../n_string_accentuated_char_no_quotes.json | 1 - .../JSONTestSuite/n_string_backslash_00.json | Bin 6 -> 0 bytes .../JSONTestSuite/n_string_escape_x.json | 1 - .../n_string_escaped_backslash_bad.json | 1 - .../n_string_escaped_ctrl_char_tab.json | 1 - .../JSONTestSuite/n_string_escaped_emoji.json | 1 - .../n_string_incomplete_escape.json | 1 - ...n_string_incomplete_escaped_character.json | 1 - .../n_string_incomplete_surrogate.json | 1 - ...g_incomplete_surrogate_escape_invalid.json | 1 - .../n_string_invalid-utf-8-in-escape.json | 1 - .../n_string_invalid_backslash_esc.json | 1 - .../n_string_invalid_unicode_escape.json | 1 - .../n_string_invalid_utf8_after_escape.json | 1 - .../n_string_leading_uescaped_thinspace.json | 1 - .../n_string_no_quotes_with_bad_escape.json | 1 - .../n_string_single_doublequote.json | 1 - .../JSONTestSuite/n_string_single_quote.json | 1 - ...string_single_string_no_double_quotes.json | 1 - .../n_string_start_escape_unclosed.json | 1 - .../n_string_unescaped_crtl_char.json | Bin 7 -> 0 bytes .../n_string_unescaped_newline.json | 2 - .../JSONTestSuite/n_string_unescaped_tab.json | 1 - .../n_string_unicode_CapitalU.json | 1 - .../n_string_with_trailing_garbage.json | 1 - .../n_structure_100000_opening_arrays.json | 1 - .../n_structure_U+2060_word_joined.json | 1 - .../n_structure_UTF8_BOM_no_data.json | 1 - .../n_structure_angle_bracket_..json | 1 - .../n_structure_angle_bracket_null.json | 1 - .../n_structure_array_trailing_garbage.json | 1 - ...tructure_array_with_extra_array_close.json | 1 - ..._structure_array_with_unclosed_string.json | 1 - .../n_structure_ascii-unicode-identifier.json | 1 - .../n_structure_capitalized_True.json | 1 - .../n_structure_close_unopened_array.json | 1 - ...ucture_comma_instead_of_closing_brace.json | 1 - .../n_structure_double_array.json | 1 - .../JSONTestSuite/n_structure_end_array.json | 1 - .../n_structure_incomplete_UTF8_BOM.json | 1 - .../n_structure_lone-invalid-utf-8.json | 1 - .../n_structure_lone-open-bracket.json | 1 - .../JSONTestSuite/n_structure_no_data.json | 0 .../n_structure_null-byte-outside-string.json | Bin 3 -> 0 bytes ...tructure_number_with_trailing_garbage.json | 1 - ...ure_object_followed_by_closing_object.json | 1 - .../n_structure_object_unclosed_no_value.json | 1 - .../n_structure_object_with_comment.json | 1 - ...tructure_object_with_trailing_garbage.json | 1 - .../n_structure_open_array_apostrophe.json | 1 - .../n_structure_open_array_comma.json | 1 - .../n_structure_open_array_object.json | 1 - .../n_structure_open_array_open_object.json | 1 - .../n_structure_open_array_open_string.json | 1 - .../n_structure_open_array_string.json | 1 - .../n_structure_open_object.json | 1 - .../n_structure_open_object_close_array.json | 1 - .../n_structure_open_object_comma.json | 1 - .../n_structure_open_object_open_array.json | 1 - .../n_structure_open_object_open_string.json | 1 - ...e_open_object_string_with_apostrophes.json | 1 - .../JSONTestSuite/n_structure_open_open.json | 1 - .../n_structure_single_eacute.json | 1 - .../n_structure_single_star.json | 1 - .../JSONTestSuite/n_structure_trailing_#.json | 1 - ...n_structure_uescaped_LF_before_string.json | 1 - .../n_structure_unclosed_array.json | 1 - ...structure_unclosed_array_partial_null.json | 1 - ...cture_unclosed_array_unfinished_false.json | 1 - ...ucture_unclosed_array_unfinished_true.json | 1 - .../n_structure_unclosed_object.json | 1 - .../n_structure_unicode-identifier.json | 1 - ...ructure_whitespace_U+2060_word_joiner.json | 1 - .../n_structure_whitespace_formfeed.json | 1 - .../y_array_arraysWithSpaces.json | 1 - .../JSONTestSuite/y_array_empty-string.json | 1 - .../input/JSONTestSuite/y_array_empty.json | 1 - .../y_array_ending_with_newline.json | 1 - .../input/JSONTestSuite/y_array_false.json | 1 - .../JSONTestSuite/y_array_heterogeneous.json | 1 - .../input/JSONTestSuite/y_array_null.json | 1 - .../y_array_with_1_and_newline.json | 2 - .../y_array_with_leading_space.json | 1 - .../y_array_with_several_null.json | 1 - .../y_array_with_trailing_space.json | 1 - .../tests/input/JSONTestSuite/y_number.json | 1 - .../input/JSONTestSuite/y_number_0e+1.json | 1 - .../input/JSONTestSuite/y_number_0e1.json | 1 - .../JSONTestSuite/y_number_after_space.json | 1 - .../y_number_double_close_to_zero.json | 1 - .../JSONTestSuite/y_number_int_with_exp.json | 1 - .../JSONTestSuite/y_number_minus_zero.json | 1 - .../JSONTestSuite/y_number_negative_int.json | 1 - .../JSONTestSuite/y_number_negative_one.json | 1 - .../JSONTestSuite/y_number_negative_zero.json | 1 - .../y_number_real_capital_e.json | 1 - .../y_number_real_capital_e_neg_exp.json | 1 - .../y_number_real_capital_e_pos_exp.json | 1 - .../JSONTestSuite/y_number_real_exponent.json | 1 - .../y_number_real_fraction_exponent.json | 1 - .../JSONTestSuite/y_number_real_neg_exp.json | 1 - .../y_number_real_pos_exponent.json | 1 - .../JSONTestSuite/y_number_simple_int.json | 1 - .../JSONTestSuite/y_number_simple_real.json | 1 - .../tests/input/JSONTestSuite/y_object.json | 1 - .../input/JSONTestSuite/y_object_basic.json | 1 - .../y_object_duplicated_key.json | 1 - .../y_object_duplicated_key_and_value.json | 1 - .../input/JSONTestSuite/y_object_empty.json | 1 - .../JSONTestSuite/y_object_empty_key.json | 1 - .../y_object_escaped_null_in_key.json | 1 - .../y_object_extreme_numbers.json | 1 - .../JSONTestSuite/y_object_long_strings.json | 1 - .../input/JSONTestSuite/y_object_simple.json | 1 - .../y_object_string_unicode.json | 1 - .../JSONTestSuite/y_object_with_newlines.json | 3 - .../y_string_1_2_3_bytes_UTF-8_sequences.json | 1 - .../y_string_accepted_surrogate_pair.json | 1 - .../y_string_accepted_surrogate_pairs.json | 1 - .../y_string_allowed_escapes.json | 1 - ...y_string_backslash_and_u_escaped_zero.json | 1 - .../y_string_backslash_doublequotes.json | 1 - .../JSONTestSuite/y_string_comments.json | 1 - .../y_string_double_escape_a.json | 1 - .../y_string_double_escape_n.json | 1 - .../y_string_escaped_control_character.json | 1 - .../y_string_escaped_noncharacter.json | 1 - .../JSONTestSuite/y_string_in_array.json | 1 - .../y_string_in_array_with_leading_space.json | 1 - .../y_string_last_surrogates_1_and_2.json | 1 - .../JSONTestSuite/y_string_nbsp_uescaped.json | 1 - ...y_string_nonCharacterInUTF-8_U+10FFFF.json | 1 - .../y_string_nonCharacterInUTF-8_U+FFFF.json | 1 - .../JSONTestSuite/y_string_null_escape.json | 1 - .../y_string_one-byte-utf-8.json | 1 - .../input/JSONTestSuite/y_string_pi.json | 1 - ...ring_reservedCharacterInUTF-8_U+1BFFF.json | 1 - .../JSONTestSuite/y_string_simple_ascii.json | 1 - .../input/JSONTestSuite/y_string_space.json | 1 - ...rogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF.json | 1 - .../y_string_three-byte-utf-8.json | 1 - .../y_string_two-byte-utf-8.json | 1 - .../y_string_u+2028_line_sep.json | 1 - .../y_string_u+2029_par_sep.json | 1 - .../input/JSONTestSuite/y_string_uEscape.json | 1 - .../y_string_uescaped_newline.json | 1 - .../y_string_unescaped_char_delete.json | 1 - .../input/JSONTestSuite/y_string_unicode.json | 1 - .../y_string_unicodeEscapedBackslash.json | 1 - .../JSONTestSuite/y_string_unicode_2.json | 1 - .../y_string_unicode_U+10FFFE_nonchar.json | 1 - .../y_string_unicode_U+1FFFE_nonchar.json | 1 - ...tring_unicode_U+200B_ZERO_WIDTH_SPACE.json | 1 - ..._string_unicode_U+2064_invisible_plus.json | 1 - .../y_string_unicode_U+FDD0_nonchar.json | 1 - .../y_string_unicode_U+FFFE_nonchar.json | 1 - ...y_string_unicode_escaped_double_quote.json | 1 - .../input/JSONTestSuite/y_string_utf8.json | 1 - .../y_string_with_del_character.json | 1 - .../y_structure_lonely_false.json | 1 - .../JSONTestSuite/y_structure_lonely_int.json | 1 - .../y_structure_lonely_negative_real.json | 1 - .../y_structure_lonely_null.json | 1 - .../y_structure_lonely_string.json | 1 - .../y_structure_lonely_true.json | 1 - .../y_structure_string_empty.json | 1 - .../y_structure_trailing_newline.json | 1 - .../y_structure_true_in_array.json | 1 - .../y_structure_whitespace_array.json | 1 - .../input/JSON_checker/Acknowledgement.txt | 3 - .../tests/input/JSON_checker/fail1.json | 1 - .../tests/input/JSON_checker/fail10.json | 1 - .../tests/input/JSON_checker/fail11.json | 1 - .../tests/input/JSON_checker/fail12.json | 1 - .../tests/input/JSON_checker/fail13.json | 1 - .../tests/input/JSON_checker/fail14.json | 1 - .../tests/input/JSON_checker/fail15.json | 1 - .../tests/input/JSON_checker/fail16.json | 1 - .../tests/input/JSON_checker/fail17.json | 1 - .../tests/input/JSON_checker/fail18.json | 1 - .../tests/input/JSON_checker/fail19.json | 1 - .../tests/input/JSON_checker/fail2.json | 1 - .../tests/input/JSON_checker/fail20.json | 1 - .../tests/input/JSON_checker/fail21.json | 1 - .../tests/input/JSON_checker/fail22.json | 1 - .../tests/input/JSON_checker/fail23.json | 1 - .../tests/input/JSON_checker/fail24.json | 1 - .../tests/input/JSON_checker/fail25.json | 1 - .../tests/input/JSON_checker/fail26.json | 1 - .../tests/input/JSON_checker/fail27.json | 2 - .../tests/input/JSON_checker/fail28.json | 2 - .../tests/input/JSON_checker/fail29.json | 1 - .../tests/input/JSON_checker/fail3.json | 1 - .../tests/input/JSON_checker/fail30.json | 1 - .../tests/input/JSON_checker/fail31.json | 1 - .../tests/input/JSON_checker/fail32.json | 1 - .../tests/input/JSON_checker/fail33.json | 1 - .../tests/input/JSON_checker/fail4.json | 1 - .../tests/input/JSON_checker/fail5.json | 1 - .../tests/input/JSON_checker/fail6.json | 1 - .../tests/input/JSON_checker/fail7.json | 1 - .../tests/input/JSON_checker/fail8.json | 1 - .../tests/input/JSON_checker/fail9.json | 1 - .../tests/input/JSON_checker/pass1.json | 58 - .../tests/input/JSON_checker/pass2.json | 1 - .../tests/input/JSON_checker/pass3.json | 6 - .../tests/input/address-book.json | 13 - .../tests/input/countries.csv | 5 - .../tests/input/countries.json | 7 - .../tests/input/cyrillic.json | 731 --- .../tests/input/employees.json | 30 - .../tests/input/employees.txt | 6 - .../tests/input/json-exception-1.json | 20 - .../tests/input/json-exception-2.json | 18 - .../tests/input/json-multiline-comment.json | 2524 --------- .../tests/input/locations.json | 17 - .../jsoncons-0.126.0/tests/input/members.json | 17 - .../jsoncons-0.126.0/tests/input/persons.json | 3 - .../tests/output/address-book-new.json | 13 - .../jsoncons-0.126.0/tests/output/store.cbor | 1 - .../jsoncons-0.126.0/tests/output/test.json | 1502 ----- .../tests/src/JSONTestSuite_tests.cpp | 76 - .../tests/src/bignum_tests.cpp | 329 -- .../tests/src/bson/bson_encoder_tests.cpp | 247 - .../tests/src/bson/bson_reader_tests.cpp | 40 - .../tests/src/byte_string_tests.cpp | 106 - .../tests/src/cbor/cbor_encoder_tests.cpp | 393 -- .../tests/src/cbor/cbor_reader_tests.cpp | 733 --- .../tests/src/cbor/cbor_tests.cpp | 185 - .../tests/src/cbor/decode_cbor_tests.cpp | 663 --- .../tests/src/cbor/encode_cbor_tests.cpp | 143 - .../tests/src/csv/csv_subfield_tests.cpp | 150 - .../tests/src/csv/csv_tests.cpp | 1227 ---- .../tests/src/double_round_trip_tests.cpp | 45 - .../tests/src/double_to_string_tests.cpp | 170 - .../jsoncons-0.126.0/tests/src/dtoa_tests.cpp | 100 - .../tests/src/error_recovery_tests.cpp | 79 - .../tests/src/heap_only_string_tests.cpp | 36 - .../tests/src/json_array_tests.cpp | 462 -- .../tests/src/json_as_tests.cpp | 75 - .../tests/src/json_checker_tests.cpp | 716 --- .../tests/src/json_cursor_tests.cpp | 336 -- .../src/json_encode_and_decode_tests.cpp | 327 -- .../tests/src/json_encoder_tests.cpp | 31 - .../tests/src/json_equal_to_tests.cpp | 159 - .../tests/src/json_exception_tests.cpp | 74 - .../tests/src/json_filter_tests.cpp | 170 - .../tests/src/json_integer_tests.cpp | 57 - .../tests/src/json_less_tests.cpp | 157 - .../tests/src/json_line_split_tests.cpp | 241 - .../tests/src/json_literal_operator_tests.cpp | 91 - .../tests/src/json_member_tests.cpp | 102 - .../tests/src/json_object_tests.cpp | 1219 ---- .../tests/src/json_options_tests.cpp | 438 -- .../tests/src/json_parse_error_tests.cpp | 239 - .../tests/src/json_parser_tests.cpp | 279 - .../tests/src/json_reader_exception_tests.cpp | 252 - .../tests/src/json_reader_tests.cpp | 226 - .../tests/src/json_swap_tests.cpp | 149 - .../src/json_type_traits_container_tests.cpp | 373 -- .../src/json_type_traits_macros_tests.cpp | 153 - .../tests/src/json_type_traits_tests.cpp | 141 - .../tests/src/json_variant_tests.cpp | 209 - .../tests/src/jsoncons_tests.cpp | 230 - .../tests/src/jsonpatch/jsonpatch_tests.cpp | 496 -- .../src/jsonpath/JSONPathTestSuite_tests.cpp | 116 - .../src/jsonpath/jsonpath_error_tests.cpp | 156 - .../src/jsonpath/jsonpath_filter_tests.cpp | 488 -- .../src/jsonpath/jsonpath_function_tests.cpp | 182 - .../jsonpath_normalized_path_tests.cpp | 251 - .../tests/src/jsonpath/jsonpath_tests.cpp | 1845 ------ .../src/jsonpointer/jsonpointer_tests.cpp | 316 -- .../src/msgpack/decode_msgpack_tests.cpp | 129 - .../src/msgpack/encode_msgpack_tests.cpp | 127 - .../src/msgpack/msgpack_encoder_tests.cpp | 108 - .../tests/src/msgpack/msgpack_tests.cpp | 147 - .../tests/src/ojson_tests.cpp | 102 - .../order_preserving_json_object_tests.cpp | 290 - .../tests/src/parse_string_tests.cpp | 133 - .../tests/src/short_string_tests.cpp | 32 - .../tests/src/staj_iterator_tests.cpp | 82 - .../tests/src/stateful_allocator_tests.cpp | 189 - .../tests/src/string_to_double_tests.cpp | 55 - .../tests/src/string_view_tests.cpp | 28 - .../jsoncons-0.126.0/tests/src/tests_main.cpp | 3 - .../tests/src/ubjson/decode_ubjson_tests.cpp | 181 - .../tests/src/ubjson/encode_ubjson_tests.cpp | 177 - .../tests/src/ubjson/ubjson_encoder_tests.cpp | 107 - .../tests/src/unicode_tests.cpp | 59 - .../tests/src/wjson_tests.cpp | 53 - 775 files changed, 575 insertions(+), 92174 deletions(-) create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp create mode 100644 cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/.doozer.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/.gitignore delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/.travis.yml delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/CMakeLists.txt delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/ChangeLog.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/LICENSE delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/README.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/Roadmap.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/acknowledgements.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/appveyor.yml delete mode 100755 cpp/src/core/thirdparty/jsoncons-0.126.0/build.sh delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/build_files/cmake/Darwin.cmake delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/build_files/cmake/Linux.cmake delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/build_files/cmake/Windows.cmake delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/build_files/cmake/config.cmake.in delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/css/jsoncons.css delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Examples.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Home.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Pages/index.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Tutorials/Basics.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Tutorials/Custom Allocators.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Tutorials/Unicode support.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/build.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/big_integer_chars_format.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bignum.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/bson.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/bson_encoder.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/decode_bson.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/diagrams/bson_encoder.png delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/diagrams/bson_encoder.xml delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/encode_bson.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/byte_string.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/byte_string_chars_format.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor_decode_options.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor_encode_options.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor_encoder.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor_options.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/decode_cbor.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/diagrams/cbor_encoder.xml delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/diagrams/cbor_options.png delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/diagrams/cbor_options.xml delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/encode_cbor.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/chars_format.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_decode_options.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_encode_options.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_encoder.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_options.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_reader.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/decode_csv.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/diagrams/csv_encoder.png delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/diagrams/csv_encoder.xml delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/diagrams/csv_options.png delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/diagrams/csv_options.xml delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/encode_csv.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/data-model.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/decode_json.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/default_parse_error_handler.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/deprecated.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_decoder.png delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_decoder.xml delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_encoder.png delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_encoder.xml delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_filter.png delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_filter.xml delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_options.png delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_options.xml delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/rename_object_member_filter.png delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/rename_object_member_filter.xml delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/ser_error.png delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/ser_error.xml delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/encode_json.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/indenting.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/array_range.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/as.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/constructor.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/destructor.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/dump.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/emplace.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/emplace_back.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/erase.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/insert.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/insert_or_assign.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/is.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/key_value.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/make_array.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/merge.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/merge_or_update.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/object_range.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/operator=.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/parse.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/push_back.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/try_emplace.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_content_handler.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_cursor.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_decode_options.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_decoder.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_encode_options.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_encoder.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_error.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_filter.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_options.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_parser.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_reader.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_type_traits.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/apply_patch.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/diagrams/jsonpatch_error.png delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/diagrams/jsonpatch_error.xml delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/from_diff.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/jsonpatch.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/jsonpatch_errc.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/jsonpatch_error.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/diagrams/jsonpath_error.png delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/diagrams/jsonpath_error.xml delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/json_query.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/json_replace.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/jsoncons-jsonpath-abnf.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/jsonpath.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/jsonpath_error.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/address.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/address.md.orig delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/contains.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/diagrams/jsonpointer_error.html delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/diagrams/jsonpointer_error.png delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/diagrams/jsonpointer_error.xml delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/get.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/get.md.orig delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/insert.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/insert_or_assign.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/jsonpointer.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/jsonpointer_errc.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/jsonpointer_error.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/pointer.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/remove.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/replace.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/line_split_kind.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/decode_msgpack.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/diagrams/msgpack_encoder.png delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/diagrams/msgpack_encoder.xml delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/encode_msgpack.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/msgpack.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/msgpack_encoder.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ojson.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/parse_error_handler.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/rename_object_member_filter.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/semantic_tag.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ser_context.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ser_error.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/spaces_option.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_array_iterator.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_event.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_event_type.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_filter.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_object_iterator.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_reader.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/decode_ubjson.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/diagrams/ubjson_encoder.png delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/diagrams/ubjson_encoder.xml delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/encode_ubjson.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/ubjson.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/ubjson_encoder.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wjson.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wjson_encoder.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wjson_options.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wjson_reader.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wojson.md delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/address-book.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/booklist.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/books.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/countries.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/employees.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/multiple-json-objects.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/sales.csv delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/tasks.csv delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/booklist.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/booklist2.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/export_settings.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/new-address-book1.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/new-address-book2.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/tasks.csv delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/tasks.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/xxx.txt delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/array_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/basics_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/basics_wexamples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/bson_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/byte_string_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/cbor_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/container_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/csv_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/data_model_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/example_types.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/json_accessor_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/json_cursor_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/json_filter_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/json_parser_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/jsonpatch_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/jsonpath_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/jsonpointer_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/jsonpointer_examples.cpp.orig delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/msgpack_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/ojson_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/polymorphic_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/readme_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/serialization_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/staj_iterator_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/type_extensibility_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/type_extensibility_examples2.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/ubjson_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/unicode_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/wjson_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples_boost/interprocess_allocator/shared_memory.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples_boost/more_examples/src/extensibility.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples_boost/more_examples/src/more_examples.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/examples_boost/more_examples/src/pool_allocator.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/basic_json.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/bignum.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/byte_string.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/config/binary_detail.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/config/jsoncons_config.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/config/version.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/grisu3.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/heap_only_string.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/parse_number.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/print_number.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/string_view.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/type_traits.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/encode_decode_json.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_container_types.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_content_handler.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_conversion_traits.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_cursor.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_decoder.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_encoder.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_error.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_exception.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_filter.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_fwd.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_options.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_parser.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_pull_reader.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_reader.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_serializer.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_type_traits.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_type_traits_macros.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/parse_error_handler.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/pretty_print.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/result.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/ser_context.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/source.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/staj_iterator.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/staj_reader.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/unicode_traits.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson_detail.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson_encoder.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson_error.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson_reader.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_detail.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_encoder.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_error.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_options.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_reader.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_encoder.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_error.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_options.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_parser.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_reader.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_serializer.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpatch/jsonpatch.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpatch/jsonpatch_error.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpath/json_query.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpath/jsonpath_error.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpath/jsonpath_filter.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpath/jsonpath_function.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpointer/jsonpointer.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpointer/jsonpointer.hpp.orig delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpointer/jsonpointer_error.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack_detail.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack_encoder.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack_error.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack_reader.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson_detail.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson_encoder.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson_error.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson_reader.hpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/CMakeLists.txt delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/Readme.txt delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/document.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query1.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query1.jsonpath delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query2.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query2.jsonpath delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query3.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query3.jsonpath delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query4.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query4.jsonpath delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query5.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query5.jsonpath delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query6.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query6.jsonpath delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query7.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query7.jsonpath delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query8.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query8.jsonpath delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query9.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query9.jsonpath delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_double_huge_neg_exp.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_huge_exp.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_neg_int_huge_exp.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_pos_double_huge_exp.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_real_neg_overflow.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_real_pos_overflow.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_real_underflow.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_too_big_neg_int.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_too_big_pos_int.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_very_big_negative_int.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_object_key_lone_2nd_surrogate.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_1st_surrogate_but_2nd_missing.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_1st_valid_surrogate_2nd_invalid.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_UTF-16LE_with_BOM.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_UTF-8_invalid_sequence.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_UTF8_surrogate_U+D800.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_incomplete_surrogate_and_escape_valid.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_incomplete_surrogate_pair.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_incomplete_surrogates_escape_valid.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_invalid_lonely_surrogate.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_invalid_surrogate.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_invalid_utf-8.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_inverted_surrogates_U+1D11E.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_iso_latin_1.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_lone_second_surrogate.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_lone_utf8_continuation_byte.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_not_in_unicode_range.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_overlong_sequence_2_bytes.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_overlong_sequence_6_bytes.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_overlong_sequence_6_bytes_null.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_truncated-utf-8.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_utf16BE_no_BOM.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_utf16LE_no_BOM.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_structure_500_nested_arrays.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_structure_UTF-8_BOM_empty_object.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_1_true_without_comma.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_a_invalid_utf8.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_colon_instead_of_comma.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_comma_after_close.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_comma_and_number.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_double_comma.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_double_extra_comma.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_extra_close.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_extra_comma.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_incomplete.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_incomplete_invalid_value.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_inner_array_no_comma.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_invalid_utf8.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_items_separated_by_semicolon.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_just_comma.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_just_minus.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_missing_value.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_newlines_unclosed.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_number_and_comma.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_number_and_several_commas.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_spaces_vertical_tab_formfeed.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_star_inside.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_unclosed.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_unclosed_trailing_comma.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_unclosed_with_new_lines.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_unclosed_with_object_inside.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_incomplete_false.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_incomplete_null.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_incomplete_true.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_multidigit_number_then_00.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_++.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_+1.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_+Inf.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_-01.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_-1.0..json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_-2..json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_-NaN.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_.-1.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_.2e-3.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0.1.2.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0.3e+.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0.3e.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0.e1.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0_capital_E+.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0_capital_E.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0e+.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0e.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1.0e+.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1.0e-.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1.0e.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1_000.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1eE2.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_2.e+3.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_2.e-3.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_2.e3.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_9.e+.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_Inf.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_NaN.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_U+FF11_fullwidth_digit_one.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_expression.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_hex_1_digit.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_hex_2_digits.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_infinity.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid+-.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid-negative-real.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid-utf-8-in-bigger-int.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid-utf-8-in-exponent.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid-utf-8-in-int.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_minus_infinity.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_minus_sign_with_trailing_garbage.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_minus_space_1.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_neg_int_starting_with_zero.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_neg_real_without_int_part.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_neg_with_garbage_at_end.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_real_garbage_after_e.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_real_with_invalid_utf8_after_e.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_real_without_fractional_part.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_starting_with_dot.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_with_alpha.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_with_alpha_char.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_with_leading_zero.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_bad_value.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_bracket_key.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_comma_instead_of_colon.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_double_colon.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_emoji.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_garbage_at_end.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_key_with_single_quotes.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_lone_continuation_byte_in_key_and_trailing_comma.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_missing_colon.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_missing_key.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_missing_semicolon.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_missing_value.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_no-colon.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_non_string_key.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_non_string_key_but_huge_number_instead.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_repeated_null_null.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_several_trailing_commas.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_single_quote.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comma.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comment.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comment_open.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comment_slash_open.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comment_slash_open_incomplete.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_two_commas_in_a_row.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_unquoted_key.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_unterminated-value.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_with_single_string.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_with_trailing_garbage.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_single_space.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_1_surrogate_then_escape.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_1_surrogate_then_escape_u.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_1_surrogate_then_escape_u1.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_1_surrogate_then_escape_u1x.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_accentuated_char_no_quotes.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_backslash_00.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_escape_x.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_escaped_backslash_bad.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_escaped_ctrl_char_tab.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_escaped_emoji.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_incomplete_escape.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_incomplete_escaped_character.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_incomplete_surrogate.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_incomplete_surrogate_escape_invalid.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_invalid-utf-8-in-escape.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_invalid_backslash_esc.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_invalid_unicode_escape.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_invalid_utf8_after_escape.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_leading_uescaped_thinspace.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_no_quotes_with_bad_escape.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_single_doublequote.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_single_quote.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_single_string_no_double_quotes.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_start_escape_unclosed.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_unescaped_crtl_char.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_unescaped_newline.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_unescaped_tab.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_unicode_CapitalU.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_with_trailing_garbage.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_100000_opening_arrays.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_U+2060_word_joined.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_UTF8_BOM_no_data.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_angle_bracket_..json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_angle_bracket_null.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_array_trailing_garbage.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_array_with_extra_array_close.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_array_with_unclosed_string.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_ascii-unicode-identifier.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_capitalized_True.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_close_unopened_array.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_comma_instead_of_closing_brace.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_double_array.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_end_array.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_incomplete_UTF8_BOM.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_lone-invalid-utf-8.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_lone-open-bracket.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_no_data.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_null-byte-outside-string.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_number_with_trailing_garbage.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_object_followed_by_closing_object.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_object_unclosed_no_value.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_object_with_comment.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_object_with_trailing_garbage.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_apostrophe.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_comma.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_object.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_open_object.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_open_string.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_string.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_close_array.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_comma.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_open_array.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_open_string.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_string_with_apostrophes.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_open.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_single_eacute.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_single_star.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_trailing_#.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_uescaped_LF_before_string.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_array.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_array_partial_null.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_array_unfinished_false.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_array_unfinished_true.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_object.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unicode-identifier.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_whitespace_U+2060_word_joiner.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_whitespace_formfeed.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_arraysWithSpaces.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_empty-string.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_empty.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_ending_with_newline.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_false.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_heterogeneous.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_null.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_with_1_and_newline.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_with_leading_space.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_with_several_null.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_with_trailing_space.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_0e+1.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_0e1.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_after_space.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_double_close_to_zero.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_int_with_exp.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_minus_zero.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_negative_int.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_negative_one.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_negative_zero.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_capital_e.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_capital_e_neg_exp.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_capital_e_pos_exp.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_exponent.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_fraction_exponent.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_neg_exp.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_pos_exponent.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_simple_int.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_simple_real.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_basic.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_duplicated_key.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_duplicated_key_and_value.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_empty.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_empty_key.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_escaped_null_in_key.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_extreme_numbers.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_long_strings.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_simple.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_string_unicode.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_with_newlines.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_1_2_3_bytes_UTF-8_sequences.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_accepted_surrogate_pair.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_accepted_surrogate_pairs.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_allowed_escapes.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_backslash_and_u_escaped_zero.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_backslash_doublequotes.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_comments.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_double_escape_a.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_double_escape_n.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_escaped_control_character.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_escaped_noncharacter.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_in_array.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_in_array_with_leading_space.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_last_surrogates_1_and_2.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_nbsp_uescaped.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_nonCharacterInUTF-8_U+10FFFF.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_nonCharacterInUTF-8_U+FFFF.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_null_escape.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_one-byte-utf-8.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_pi.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_reservedCharacterInUTF-8_U+1BFFF.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_simple_ascii.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_space.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_three-byte-utf-8.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_two-byte-utf-8.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_u+2028_line_sep.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_u+2029_par_sep.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_uEscape.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_uescaped_newline.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unescaped_char_delete.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicodeEscapedBackslash.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_2.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+10FFFE_nonchar.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+1FFFE_nonchar.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+200B_ZERO_WIDTH_SPACE.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+2064_invisible_plus.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+FDD0_nonchar.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+FFFE_nonchar.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_escaped_double_quote.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_utf8.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_with_del_character.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_false.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_int.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_negative_real.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_null.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_string.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_true.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_string_empty.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_trailing_newline.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_true_in_array.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_whitespace_array.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/Acknowledgement.txt delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail1.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail10.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail11.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail12.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail13.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail14.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail15.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail16.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail17.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail18.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail19.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail2.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail20.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail21.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail22.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail23.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail24.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail25.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail26.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail27.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail28.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail29.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail3.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail30.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail31.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail32.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail33.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail4.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail5.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail6.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail7.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail8.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail9.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/pass1.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/pass2.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/pass3.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/address-book.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/countries.csv delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/countries.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/cyrillic.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/employees.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/employees.txt delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/json-exception-1.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/json-exception-2.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/json-multiline-comment.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/locations.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/members.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/persons.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/output/address-book-new.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/output/store.cbor delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/output/test.json delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/JSONTestSuite_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/bignum_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/bson/bson_encoder_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/bson/bson_reader_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/byte_string_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/cbor_encoder_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/cbor_reader_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/cbor_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/decode_cbor_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/encode_cbor_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/csv/csv_subfield_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/csv/csv_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/double_round_trip_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/double_to_string_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/dtoa_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/error_recovery_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/heap_only_string_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_array_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_as_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_checker_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_cursor_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_encode_and_decode_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_encoder_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_equal_to_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_exception_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_filter_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_integer_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_less_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_line_split_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_literal_operator_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_member_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_object_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_options_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_parse_error_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_parser_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_reader_exception_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_reader_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_swap_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_type_traits_container_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_type_traits_macros_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_type_traits_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_variant_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsoncons_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpatch/jsonpatch_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/JSONPathTestSuite_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_error_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_filter_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_function_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_normalized_path_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpointer/jsonpointer_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/msgpack/decode_msgpack_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/msgpack/encode_msgpack_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/msgpack/msgpack_encoder_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/msgpack/msgpack_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/ojson_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/order_preserving_json_object_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/parse_string_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/short_string_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/staj_iterator_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/stateful_allocator_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/string_to_double_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/string_view_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/tests_main.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/ubjson/decode_ubjson_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/ubjson/encode_ubjson_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/ubjson/ubjson_encoder_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/unicode_tests.cpp delete mode 100644 cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/wjson_tests.cpp diff --git a/cpp/src/core/CMakeLists.txt b/cpp/src/core/CMakeLists.txt index b126cce031..bef3f5e2f6 100644 --- a/cpp/src/core/CMakeLists.txt +++ b/cpp/src/core/CMakeLists.txt @@ -93,7 +93,7 @@ endif() set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) if(BUILD_UNIT_TEST STREQUAL "ON") -# add_subdirectory(test) + add_subdirectory(test) endif() config_summary() diff --git a/cpp/src/core/knowhere/CMakeLists.txt b/cpp/src/core/knowhere/CMakeLists.txt index 76750f4f07..864d03583a 100644 --- a/cpp/src/core/knowhere/CMakeLists.txt +++ b/cpp/src/core/knowhere/CMakeLists.txt @@ -8,7 +8,6 @@ link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib64) include_directories(${CORE_SOURCE_DIR}/knowhere) include_directories(${CORE_SOURCE_DIR}/thirdparty) include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) -include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) set(SPTAG_SOURCE_DIR ${CORE_SOURCE_DIR}/thirdparty/SPTAG) file(GLOB HDR_FILES @@ -55,6 +54,7 @@ set(index_srcs knowhere/index/vector_index/IndexGPUIVFPQ.cpp knowhere/index/vector_index/FaissBaseIndex.cpp knowhere/index/vector_index/helpers/FaissIO.cpp + knowhere/index/vector_index/helpers/IndexParameter.cpp ) set(depend_libs diff --git a/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp index ee5a4a01b8..056f4e2cad 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp @@ -69,8 +69,7 @@ ConvertToQueryResult(const DatasetPtr &dataset, const Config &config) { auto dimension = tensor->shape()[1]; auto rows = tensor->shape()[0]; - auto k = config[META_K].as(); - std::vector query_results(rows, SPTAG::QueryResult(nullptr, k, true)); + std::vector query_results(rows, SPTAG::QueryResult(nullptr, config->k, true)); for (auto i = 0; i < rows; ++i) { query_results[i].SetTarget(&p_data[i * dimension]); } diff --git a/cpp/src/core/knowhere/knowhere/common/Config.h b/cpp/src/core/knowhere/knowhere/common/Config.h index 399555c0e9..208641d666 100644 --- a/cpp/src/core/knowhere/knowhere/common/Config.h +++ b/cpp/src/core/knowhere/knowhere/common/Config.h @@ -18,15 +18,42 @@ #pragma once -#include - +#include namespace zilliz { namespace knowhere { +enum class METRICTYPE { + INVALID = 0, + L2 = 1, + IP = 2, +}; -using Config = jsoncons::json; +// General Config +constexpr int64_t INVALID_VALUE = -1; +constexpr int64_t DEFAULT_K = INVALID_VALUE; +constexpr int64_t DEFAULT_DIM = INVALID_VALUE; +constexpr int64_t DEFAULT_GPUID = INVALID_VALUE; +constexpr METRICTYPE DEFAULT_TYPE = METRICTYPE::INVALID; +struct Cfg { + METRICTYPE metric_type = DEFAULT_TYPE; + int64_t k = DEFAULT_K; + int64_t gpu_id = DEFAULT_GPUID; + int64_t d = DEFAULT_DIM; + + Cfg(const int64_t &dim, + const int64_t &k, + const int64_t &gpu_id, + METRICTYPE type) + : d(dim), k(k), gpu_id(gpu_id), metric_type(type) {} + + Cfg() = default; + + virtual bool + CheckValid(){}; +}; +using Config = std::shared_ptr; } // namespace knowhere } // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp index a77702ace3..ba41859135 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp @@ -36,10 +36,11 @@ namespace zilliz { namespace knowhere { IndexModelPtr GPUIVF::Train(const DatasetPtr &dataset, const Config &config) { - auto nlist = config["nlist"].as(); - gpu_id_ = config.get_with_default("gpu_id", gpu_id_); - auto metric_type = config["metric_type"].as_string() == "L2" ? - faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; + auto build_cfg = std::dynamic_pointer_cast(config); + if (build_cfg != nullptr) { + build_cfg->CheckValid(); // throw exception + } + gpu_id_ = build_cfg->gpu_id; GETTENSOR(dataset) @@ -48,7 +49,9 @@ IndexModelPtr GPUIVF::Train(const DatasetPtr &dataset, const Config &config) { ResScope rs(temp_resource, gpu_id_, true); faiss::gpu::GpuIndexIVFFlatConfig idx_config; idx_config.device = gpu_id_; - faiss::gpu::GpuIndexIVFFlat device_index(temp_resource->faiss_res.get(), dim, nlist, metric_type, idx_config); + faiss::gpu::GpuIndexIVFFlat device_index(temp_resource->faiss_res.get(), dim, + build_cfg->nlist, GetMetricType(build_cfg->metric_type), + idx_config); device_index.train(rows, (float *) p_data); std::shared_ptr host_index = nullptr; @@ -143,9 +146,10 @@ void GPUIVF::search_impl(int64_t n, const Config &cfg) { std::lock_guard lk(mutex_); + // TODO(linxj): gpu index support GenParams if (auto device_index = std::static_pointer_cast(index_)) { - auto nprobe = cfg.get_with_default("nprobe", size_t(1)); - device_index->setNumProbes(nprobe); + auto search_cfg = std::dynamic_pointer_cast(cfg); + device_index->setNumProbes(search_cfg->nprobe); { // TODO(linxj): allocate mem diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp index 437d8f8958..03193ea604 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp @@ -29,19 +29,19 @@ namespace zilliz { namespace knowhere { IndexModelPtr GPUIVFPQ::Train(const DatasetPtr &dataset, const Config &config) { - auto nlist = config["nlist"].as(); - auto M = config["M"].as(); // number of subquantizers(subvectors) - auto nbits = config["nbits"].as();// number of bit per subvector index - auto gpu_num = config.get_with_default("gpu_id", gpu_id_); - auto metric_type = config["metric_type"].as_string() == "L2" ? - faiss::METRIC_L2 : faiss::METRIC_L2; // IP not support. + auto build_cfg = std::dynamic_pointer_cast(config); + if (build_cfg != nullptr) { + build_cfg->CheckValid(); // throw exception + } + gpu_id_ = build_cfg->gpu_id; GETTENSOR(dataset) // TODO(linxj): set device here. // TODO(linxj): set gpu resource here. faiss::gpu::StandardGpuResources res; - faiss::gpu::GpuIndexIVFPQ device_index(&res, dim, nlist, M, nbits, metric_type); + faiss::gpu::GpuIndexIVFPQ device_index(&res, dim, build_cfg->nlist, build_cfg->m, + build_cfg->nbits, GetMetricType(build_cfg->metric_type)); // IP not support device_index.train(rows, (float *) p_data); std::shared_ptr host_index = nullptr; @@ -52,10 +52,11 @@ IndexModelPtr GPUIVFPQ::Train(const DatasetPtr &dataset, const Config &config) { std::shared_ptr GPUIVFPQ::GenParams(const Config &config) { auto params = std::make_shared(); - params->nprobe = config.get_with_default("nprobe", size_t(1)); - //params->scan_table_threshold = 0; - //params->polysemous_ht = 0; - //params->max_codes = 0; + auto search_cfg = std::dynamic_pointer_cast(config); + params->nprobe = search_cfg->nprobe; +// params->scan_table_threshold = conf->scan_table_threhold; +// params->polysemous_ht = conf->polysemous_ht; +// params->max_codes = conf->max_codes; return params; } diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp index 2504dd4599..af23267404 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp @@ -28,17 +28,17 @@ namespace zilliz { namespace knowhere { IndexModelPtr GPUIVFSQ::Train(const DatasetPtr &dataset, const Config &config) { - auto nlist = config["nlist"].as(); - auto nbits = config["nbits"].as(); // TODO(linxj): gpu only support SQ4 SQ8 SQ16 - gpu_id_ = config.get_with_default("gpu_id", gpu_id_); - auto metric_type = config["metric_type"].as_string() == "L2" ? - faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; + auto build_cfg = std::dynamic_pointer_cast(config); + if (build_cfg != nullptr) { + build_cfg->CheckValid(); // throw exception + } + gpu_id_ = build_cfg->gpu_id; GETTENSOR(dataset) std::stringstream index_type; - index_type << "IVF" << nlist << "," << "SQ" << nbits; - auto build_index = faiss::index_factory(dim, index_type.str().c_str(), metric_type); + index_type << "IVF" << build_cfg->nlist << "," << "SQ" << build_cfg->nbits; + auto build_index = faiss::index_factory(dim, index_type.str().c_str(), GetMetricType(build_cfg->metric_type)); auto temp_resource = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_); if (temp_resource != nullptr) { diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp index a53a73dad9..f5e3fd8f40 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp @@ -51,24 +51,22 @@ DatasetPtr IDMAP::Search(const DatasetPtr &dataset, const Config &config) { KNOWHERE_THROW_MSG("index not initialize"); } - auto k = config["k"].as(); + config->CheckValid(); //auto metric_type = config["metric_type"].as_string() == "L2" ? // faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; //index_->metric_type = metric_type; GETTENSOR(dataset) - // TODO(linxj): handle malloc exception - auto elems = rows * k; + auto elems = rows * config->k; auto res_ids = (int64_t *) malloc(sizeof(int64_t) * elems); auto res_dis = (float *) malloc(sizeof(float) * elems); - search_impl(rows, (float *) p_data, k, res_dis, res_ids, Config()); + search_impl(rows, (float *) p_data, config->k, res_dis, res_ids, Config()); auto id_buf = MakeMutableBufferSmart((uint8_t *) res_ids, sizeof(int64_t) * elems); auto dist_buf = MakeMutableBufferSmart((uint8_t *) res_dis, sizeof(float) * elems); - // TODO: magic std::vector id_bufs{nullptr, id_buf}; std::vector dist_bufs{nullptr, dist_buf}; @@ -136,11 +134,9 @@ int64_t *IDMAP::GetRawIds() { const char* type = "IDMap,Flat"; void IDMAP::Train(const Config &config) { - auto metric_type = config["metric_type"].as_string() == "L2" ? - faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; - auto dim = config["dim"].as(); + config->CheckValid(); - auto index = faiss::index_factory(dim, type, metric_type); + auto index = faiss::index_factory(config->d, type, GetMetricType(config->metric_type)); index_.reset(index); } diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp index a7adfe921a..2e6d0e9763 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -37,17 +37,20 @@ namespace knowhere { IndexModelPtr IVF::Train(const DatasetPtr &dataset, const Config &config) { - auto nlist = config["nlist"].as(); - auto metric_type = config["metric_type"].as_string() == "L2" ? - faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; + auto build_cfg = std::dynamic_pointer_cast(config); + if (build_cfg != nullptr) { + build_cfg->CheckValid(); // throw exception + } GETTENSOR(dataset) faiss::Index *coarse_quantizer = new faiss::IndexFlatL2(dim); - auto index = std::make_shared(coarse_quantizer, dim, nlist, metric_type); + auto index = std::make_shared(coarse_quantizer, dim, + build_cfg->nlist, + GetMetricType(build_cfg->metric_type)); index->train(rows, (float *) p_data); - // TODO: override here. train return model or not. + // TODO(linxj): override here. train return model or not. return std::make_shared(index); } @@ -60,7 +63,6 @@ void IVF::Add(const DatasetPtr &dataset, const Config &config) { std::lock_guard lk(mutex_); GETTENSOR(dataset) - // TODO: magic here. auto array = dataset->array()[0]; auto p_ids = array->data()->GetValues(1, 0); index_->add_with_ids(rows, (float *) p_data, p_ids); @@ -97,28 +99,22 @@ DatasetPtr IVF::Search(const DatasetPtr &dataset, const Config &config) { KNOWHERE_THROW_MSG("index not initialize or trained"); } - auto k = config["k"].as(); + auto search_cfg = std::dynamic_pointer_cast(config); + if (search_cfg != nullptr) { + search_cfg->CheckValid(); // throw exception + } GETTENSOR(dataset) - // TODO(linxj): handle malloc exception - auto elems = rows * k; + auto elems = rows * search_cfg->k; auto res_ids = (int64_t *) malloc(sizeof(int64_t) * elems); auto res_dis = (float *) malloc(sizeof(float) * elems); - search_impl(rows, (float*) p_data, k, res_dis, res_ids, config); - //faiss::ivflib::search_with_parameters(index_.get(), - // rows, - // (float *) p_data, - // k, - // res_dis, - // res_ids, - // params.get()); + search_impl(rows, (float*) p_data, search_cfg->k, res_dis, res_ids, config); auto id_buf = MakeMutableBufferSmart((uint8_t *) res_ids, sizeof(int64_t) * elems); auto dist_buf = MakeMutableBufferSmart((uint8_t *) res_dis, sizeof(float) * elems); - // TODO: magic std::vector id_bufs{nullptr, id_buf}; std::vector dist_bufs{nullptr, dist_buf}; @@ -146,7 +142,9 @@ void IVF::set_index_model(IndexModelPtr model) { std::shared_ptr IVF::GenParams(const Config &config) { auto params = std::make_shared(); - params->nprobe = config.get_with_default("nprobe", size_t(1)); + + auto search_cfg = std::dynamic_pointer_cast(config); + params->nprobe = search_cfg->nprobe; //params->max_codes = config.get_with_default("max_codes", size_t(0)); return params; diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp index eca63a5f3b..b5418b77cf 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp @@ -27,16 +27,16 @@ namespace zilliz { namespace knowhere { IndexModelPtr IVFPQ::Train(const DatasetPtr &dataset, const Config &config) { - auto nlist = config["nlist"].as(); - auto M = config["M"].as(); // number of subquantizers(subvector) - auto nbits = config["nbits"].as();// number of bit per subvector index - auto metric_type = config["metric_type"].as_string() == "L2" ? - faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; + auto build_cfg = std::dynamic_pointer_cast(config); + if (build_cfg != nullptr) { + build_cfg->CheckValid(); // throw exception + } GETTENSOR(dataset) - faiss::Index *coarse_quantizer = new faiss::IndexFlat(dim, metric_type); - auto index = std::make_shared(coarse_quantizer, dim, nlist, M, nbits); + faiss::Index *coarse_quantizer = new faiss::IndexFlat(dim, GetMetricType(build_cfg->metric_type)); + auto index = std::make_shared(coarse_quantizer, dim, + build_cfg->nlist, build_cfg->m, build_cfg->nbits); index->train(rows, (float *) p_data); return std::make_shared(index); @@ -44,10 +44,11 @@ IndexModelPtr IVFPQ::Train(const DatasetPtr &dataset, const Config &config) { std::shared_ptr IVFPQ::GenParams(const Config &config) { auto params = std::make_shared(); - params->nprobe = config.get_with_default("nprobe", size_t(1)); - //params->scan_table_threshold = 0; - //params->polysemous_ht = 0; - //params->max_codes = 0; + auto search_cfg = std::dynamic_pointer_cast(config); + params->nprobe = search_cfg->nprobe; +// params->scan_table_threshold = conf->scan_table_threhold; +// params->polysemous_ht = conf->polysemous_ht; +// params->max_codes = conf->max_codes; return params; } diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp index 00f191ff02..ec29627ebc 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp @@ -29,16 +29,17 @@ namespace zilliz { namespace knowhere { IndexModelPtr IVFSQ::Train(const DatasetPtr &dataset, const Config &config) { - auto nlist = config["nlist"].as(); - auto nbits = config["nbits"].as(); // TODO(linxj): only support SQ4 SQ6 SQ8 SQ16 - auto metric_type = config["metric_type"].as_string() == "L2" ? - faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; + auto build_cfg = std::dynamic_pointer_cast(config); + if (build_cfg != nullptr) { + build_cfg->CheckValid(); // throw exception + } GETTENSOR(dataset) std::stringstream index_type; - index_type << "IVF" << nlist << "," << "SQ" << nbits; - auto build_index = faiss::index_factory(dim, index_type.str().c_str(), metric_type); + index_type << "IVF" << build_cfg->nlist << "," << "SQ" << build_cfg->nbits; + auto build_index = faiss::index_factory(dim, index_type.str().c_str(), + GetMetricType(build_cfg->metric_type)); build_index->train(rows, (float *) p_data); std::shared_ptr ret_index; diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp index 5e049b72bd..cceb88516a 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp @@ -118,8 +118,8 @@ CPUKDTRNG::Add(const DatasetPtr &origin, const Config &add_config) { void CPUKDTRNG::SetParameters(const Config &config) { for (auto ¶ : KDTParameterMgr::GetInstance().GetKDTParameters()) { - auto value = config.get_with_default(para.first, para.second); - index_ptr_->SetParameter(para.first, value); +// auto value = config.get_with_default(para.first, para.second); + index_ptr_->SetParameter(para.first, para.second); } } diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.cpp index 26dc5e2304..38a7fed23b 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.cpp @@ -67,30 +67,29 @@ void NSG::Load(const BinarySet &index_binary) { } DatasetPtr NSG::Search(const DatasetPtr &dataset, const Config &config) { + auto build_cfg = std::dynamic_pointer_cast(config); + if (build_cfg != nullptr) { + build_cfg->CheckValid(); // throw exception + } + if (!index_ || !index_->is_trained) { KNOWHERE_THROW_MSG("index not initialize or trained"); } - // Required - // if not found throw exception here. - auto k = config["k"].as(); - auto search_length = config.get_with_default("search_length", 30); - GETTENSOR(dataset) - auto elems = rows * k; + auto elems = rows * build_cfg->k; auto res_ids = (int64_t *) malloc(sizeof(int64_t) * elems); auto res_dis = (float *) malloc(sizeof(float) * elems); - // TODO(linxj): get from config algo::SearchParams s_params; - s_params.search_length = search_length; - index_->Search((float *) p_data, rows, dim, k, res_dis, res_ids, s_params); + s_params.search_length = build_cfg->search_length; + index_->Search((float *) p_data, rows, dim, + build_cfg->k, res_dis, res_ids, s_params); auto id_buf = MakeMutableBufferSmart((uint8_t *) res_ids, sizeof(int64_t) * elems); auto dist_buf = MakeMutableBufferSmart((uint8_t *) res_dis, sizeof(float) * elems); - // TODO: magic std::vector id_bufs{nullptr, id_buf}; std::vector dist_bufs{nullptr, dist_buf}; @@ -108,45 +107,41 @@ DatasetPtr NSG::Search(const DatasetPtr &dataset, const Config &config) { } IndexModelPtr NSG::Train(const DatasetPtr &dataset, const Config &config) { - TimeRecorder rc("Interface"); + auto build_cfg = std::dynamic_pointer_cast(config); + if (build_cfg != nullptr) { + build_cfg->CheckValid(); // throw exception + } - auto metric_type = config["metric_type"].as_string(); - if (metric_type != "L2") { KNOWHERE_THROW_MSG("NSG not support this kind of metric type");} + if (build_cfg->metric_type != METRICTYPE::L2) { + KNOWHERE_THROW_MSG("NSG not support this kind of metric type"); + } // TODO(linxj): dev IndexFactory, support more IndexType - auto preprocess_index = std::make_shared(0); - //auto preprocess_index = std::make_shared(); + auto preprocess_index = std::make_shared(build_cfg->gpu_id); auto model = preprocess_index->Train(dataset, config); preprocess_index->set_index_model(model); preprocess_index->AddWithoutIds(dataset, config); - rc.RecordSection("build ivf"); - auto k = config["knng"].as(); Graph knng; - preprocess_index->GenGraph(k, knng, dataset, config); - rc.RecordSection("build knng"); + preprocess_index->GenGraph(build_cfg->knng, knng, dataset, config); + + algo::BuildParams b_params; + b_params.candidate_pool_size = build_cfg->candidate_pool_size; + b_params.out_degree = build_cfg->out_degree; + b_params.search_length = build_cfg->search_length; GETTENSOR(dataset) auto array = dataset->array()[0]; auto p_ids = array->data()->GetValues(1, 0); - algo::BuildParams b_params; - b_params.candidate_pool_size = config["candidate_pool_size"].as(); - b_params.out_degree = config["out_degree"].as(); - b_params.search_length = config["search_length"].as(); - index_ = std::make_shared(dim, rows); index_->SetKnnGraph(knng); index_->Build_with_ids(rows, (float *) p_data, (long *) p_ids, b_params); - rc.RecordSection("build nsg"); - rc.ElapseFromBegin("total cost"); return nullptr; // TODO(linxj): support serialize } void NSG::Add(const DatasetPtr &dataset, const Config &config) { - // TODO(linxj): support incremental index. - - //KNOWHERE_THROW_MSG("Not support yet"); + // do nothing } int64_t NSG::Count() { @@ -156,6 +151,7 @@ int64_t NSG::Count() { int64_t NSG::Dimension() { return index_->dimension; } + VectorIndexPtr NSG::Clone() { KNOWHERE_THROW_MSG("not support"); } diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/VectorIndex.h b/cpp/src/core/knowhere/knowhere/index/vector_index/VectorIndex.h index 76f67920e3..908989d15d 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/VectorIndex.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/VectorIndex.h @@ -22,6 +22,7 @@ #include #include "knowhere/common/Config.h" +#include "knowhere/index/vector_index/helpers/IndexParameter.h" #include "knowhere/common/Dataset.h" #include "knowhere/index/Index.h" #include "knowhere/index/preprocessor/Preprocessor.h" diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp new file mode 100644 index 0000000000..503f0c2f67 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include "IndexParameter.h" +#include "knowhere/common/Exception.h" + +#include + +namespace zilliz { +namespace knowhere { + +faiss::MetricType GetMetricType(METRICTYPE &type) { + if (type == METRICTYPE::L2) return faiss::METRIC_L2; + if (type == METRICTYPE::IP) return faiss::METRIC_INNER_PRODUCT; + if (type == METRICTYPE::INVALID) KNOWHERE_THROW_MSG("Metric type is invalid"); +} + + +} +} diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h new file mode 100644 index 0000000000..521754e2e3 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include "knowhere/common/Config.h" +#include + + +namespace zilliz { +namespace knowhere { + +extern faiss::MetricType GetMetricType(METRICTYPE &type); + +// IVF Config +constexpr int64_t DEFAULT_NLIST = INVALID_VALUE; +constexpr int64_t DEFAULT_NPROBE = INVALID_VALUE; +constexpr int64_t DEFAULT_NSUBVECTORS = INVALID_VALUE; +constexpr int64_t DEFAULT_NBITS = INVALID_VALUE; +constexpr int64_t DEFAULT_SCAN_TABLE_THREHOLD = INVALID_VALUE; +constexpr int64_t DEFAULT_POLYSEMOUS_HT = INVALID_VALUE; +constexpr int64_t DEFAULT_MAX_CODES = INVALID_VALUE; + +// NSG Config +constexpr int64_t DEFAULT_SEARCH_LENGTH = INVALID_VALUE; +constexpr int64_t DEFAULT_OUT_DEGREE = INVALID_VALUE; +constexpr int64_t DEFAULT_CANDIDATE_SISE = INVALID_VALUE; +constexpr int64_t DEFAULT_NNG_K = INVALID_VALUE; + +struct IVFCfg : public Cfg { + int64_t nlist = DEFAULT_NLIST; + int64_t nprobe = DEFAULT_NPROBE; + + IVFCfg(const int64_t &dim, + const int64_t &k, + const int64_t &gpu_id, + const int64_t &nlist, + const int64_t &nprobe, + METRICTYPE type) + : nlist(nlist), nprobe(nprobe), Cfg(dim, k, gpu_id, type) {} + + IVFCfg() = default; + + bool + CheckValid() override {}; +}; +using IVFConfig = std::shared_ptr; + +struct IVFSQCfg : public IVFCfg { + // TODO(linxj): cpu only support SQ4 SQ6 SQ8 SQ16, gpu only support SQ4, SQ8, SQ16 + int64_t nbits = DEFAULT_NBITS; + + IVFSQCfg(const int64_t &dim, + const int64_t &k, + const int64_t &gpu_id, + const int64_t &nlist, + const int64_t &nprobe, + const int64_t &nbits, + METRICTYPE type) + : nbits(nbits), IVFCfg(dim, k, gpu_id, nlist, nprobe, type) {} + + IVFSQCfg() = default; + + bool + CheckValid() override {}; +}; +using IVFSQConfig = std::shared_ptr; + +struct IVFPQCfg : public IVFCfg { + int64_t m = DEFAULT_NSUBVECTORS; // number of subquantizers(subvector) + int64_t nbits = DEFAULT_NBITS; // number of bit per subvector index + + // TODO(linxj): not use yet + int64_t scan_table_threhold = DEFAULT_SCAN_TABLE_THREHOLD; + int64_t polysemous_ht = DEFAULT_POLYSEMOUS_HT; + int64_t max_codes = DEFAULT_MAX_CODES; + + IVFPQCfg(const int64_t &dim, + const int64_t &k, + const int64_t &gpu_id, + const int64_t &nlist, + const int64_t &nprobe, + const int64_t &nbits, + const int64_t &m, + METRICTYPE type) + : nbits(nbits), m(m), IVFCfg(dim, k, gpu_id, nlist, nprobe, type) {} + + IVFPQCfg() = default; + + bool + CheckValid() override {}; +}; +using IVFPQConfig = std::shared_ptr; + +struct NSGCfg : public IVFCfg { + int64_t knng = DEFAULT_NNG_K; + int64_t search_length = DEFAULT_SEARCH_LENGTH; + int64_t out_degree = DEFAULT_OUT_DEGREE; + int64_t candidate_pool_size = DEFAULT_CANDIDATE_SISE; + + NSGCfg(const int64_t &dim, + const int64_t &k, + const int64_t &gpu_id, + const int64_t &nlist, + const int64_t &nprobe, + const int64_t &knng, + const int64_t &search_length, + const int64_t &out_degree, + const int64_t &candidate_size, + METRICTYPE type) + : knng(knng), search_length(search_length), out_degree(out_degree), candidate_pool_size(candidate_size), + IVFCfg(dim, k, gpu_id, nlist, nprobe, type) {} + + NSGCfg() = default; + + bool + CheckValid() override {}; +}; +using NSGConfig = std::shared_ptr; + +struct KDTCfg : public Cfg { + int64_t tptnubmber = -1; +}; + +} // knowhere +} // zilliz + diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp index a085bfcf48..3d9de271b7 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp @@ -35,7 +35,7 @@ KDTParameterMgr::KDTParameterMgr() { {"NumTopDimensionKDTSplit", "5"}, {"NumSamplesKDTSplitConsideration", "100"}, - {"TPTNumber", "32"}, + {"TPTNumber", "1"}, {"TPTLeafSize", "2000"}, {"NumTopDimensionTPTSplit", "5"}, diff --git a/cpp/src/core/test/CMakeLists.txt b/cpp/src/core/test/CMakeLists.txt index cdd500ea72..cc6ea5bdcc 100644 --- a/cpp/src/core/test/CMakeLists.txt +++ b/cpp/src/core/test/CMakeLists.txt @@ -1,17 +1,15 @@ include_directories(${CORE_SOURCE_DIR}/thirdparty) include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) include_directories(${CORE_SOURCE_DIR}/knowhere) -include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include) include_directories(/usr/local/cuda/include) link_directories(/usr/local/cuda/lib64) link_directories(${CORE_SOURCE_DIR}/thirdparty/tbb) -set(unittest_libs - gtest gmock gtest_main gmock_main) - message(STATUS "arrow prefix: ${ARROW_PREFIX}") message(STATUS "libjemalloc_pic path: ${ARROW_PREFIX}/lib/libjemalloc_pic.a") + set(depend_libs + gtest gmock gtest_main gmock_main faiss openblas lapack arrow "${ARROW_PREFIX}/lib/libjemalloc_pic.a" tbb @@ -24,36 +22,45 @@ set(basic_libs set(util_srcs ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/Structure.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/ArrowAdapter.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/common/Exception.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/common/Timer.cpp + utils.cpp ) # set(ivf_srcs - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ivf.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu_ivf.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/cloner.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/idmap.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.cpp - utils.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVF.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp ) if(NOT TARGET test_ivf) add_executable(test_ivf test_ivf.cpp ${ivf_srcs} ${util_srcs}) endif() target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs}) + # set(idmap_srcs - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/idmap.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ivf.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/cloner.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu_ivf.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/FaissGpuResourceMgr.cpp - utils.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVF.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp ) if(NOT TARGET test_idmap) add_executable(test_idmap test_idmap.cpp ${idmap_srcs} ${util_srcs}) @@ -62,15 +69,10 @@ target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs}) # set(kdt_srcs - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/preprocessor/normalize.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/kdt_parameters.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/sptag.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/arrow.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp - utils.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/SptagAdapter.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/preprocessor/Normalize.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexKDT.cpp ) if(NOT TARGET test_kdt) add_executable(test_kdt test_kdt.cpp ${kdt_srcs} ${util_srcs}) diff --git a/cpp/src/core/test/test_idmap.cpp b/cpp/src/core/test/test_idmap.cpp index 82f00455dd..3d71bca931 100644 --- a/cpp/src/core/test/test_idmap.cpp +++ b/cpp/src/core/test/test_idmap.cpp @@ -19,17 +19,17 @@ #include #include -#include #include "knowhere/index/vector_index/IndexIDMAP.h" #include "knowhere/adapter/Structure.h" -#include "knowhere/index/vector_index/utils/Cloner.h" +#include "knowhere/index/vector_index/helpers/Cloner.h" #include "knowhere/common/Exception.h" #include "utils.h" using namespace zilliz::knowhere; +using namespace zilliz::knowhere::cloner; static int device_id = 0; class IDMAPTest : public DataGen, public ::testing::Test { @@ -79,15 +79,19 @@ void PrintResult(const DatasetPtr &result, TEST_F(IDMAPTest, idmap_basic) { ASSERT_TRUE(!xb.empty()); - Config Default_cfg; - index_->Train(Config::object{{"dim", dim}, {"metric_type", "L2"}}); - index_->Add(base_dataset, Default_cfg); + auto conf = std::make_shared(); + conf->d = dim; + conf->k = k; + conf->metric_type = METRICTYPE::L2; + + index_->Train(conf); + index_->Add(base_dataset, conf); EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dimension(), dim); ASSERT_TRUE(index_->GetRawVectors() != nullptr); ASSERT_TRUE(index_->GetRawIds() != nullptr); - auto result = index_->Search(query_dataset, Config::object{{"k", k}}); + auto result = index_->Search(query_dataset, conf); AssertAnns(result, nq, k); PrintResult(result, nq, k); @@ -95,7 +99,7 @@ TEST_F(IDMAPTest, idmap_basic) { auto binaryset = index_->Serialize(); auto new_index = std::make_shared(); new_index->Load(binaryset); - auto re_result = index_->Search(query_dataset, Config::object{{"k", k}}); + auto re_result = index_->Search(query_dataset, conf); AssertAnns(re_result, nq, k); PrintResult(re_result, nq, k); } @@ -109,11 +113,16 @@ TEST_F(IDMAPTest, idmap_serialize) { reader(ret, bin->size); }; + auto conf = std::make_shared(); + conf->d = dim; + conf->k = k; + conf->metric_type = METRICTYPE::L2; + { // serialize index - index_->Train(Config::object{{"dim", dim}, {"metric_type", "L2"}}); + index_->Train(conf); index_->Add(base_dataset, Config()); - auto re_result = index_->Search(query_dataset, Config::object{{"k", k}}); + auto re_result = index_->Search(query_dataset, conf); AssertAnns(re_result, nq, k); PrintResult(re_result, nq, k); EXPECT_EQ(index_->Count(), nb); @@ -133,7 +142,7 @@ TEST_F(IDMAPTest, idmap_serialize) { index_->Load(binaryset); EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dimension(), dim); - auto result = index_->Search(query_dataset, Config::object{{"k", k}}); + auto result = index_->Search(query_dataset, conf); AssertAnns(result, nq, k); PrintResult(result, nq, k); } @@ -141,29 +150,33 @@ TEST_F(IDMAPTest, idmap_serialize) { TEST_F(IDMAPTest, copy_test) { ASSERT_TRUE(!xb.empty()); - Config Default_cfg; - index_->Train(Config::object{{"dim", dim}, {"metric_type", "L2"}}); - index_->Add(base_dataset, Default_cfg); + auto conf = std::make_shared(); + conf->d = dim; + conf->k = k; + conf->metric_type = METRICTYPE::L2; + + index_->Train(conf); + index_->Add(base_dataset, conf); EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dimension(), dim); ASSERT_TRUE(index_->GetRawVectors() != nullptr); ASSERT_TRUE(index_->GetRawIds() != nullptr); - auto result = index_->Search(query_dataset, Config::object{{"k", k}}); + auto result = index_->Search(query_dataset, conf); AssertAnns(result, nq, k); //PrintResult(result, nq, k); { // clone auto clone_index = index_->Clone(); - auto clone_result = clone_index->Search(query_dataset, Config::object{{"k", k}}); + auto clone_result = clone_index->Search(query_dataset, conf); AssertAnns(clone_result, nq, k); } { // cpu to gpu - auto clone_index = CopyCpuToGpu(index_, device_id, Config()); - auto clone_result = clone_index->Search(query_dataset, Config::object{{"k", k}}); + auto clone_index = CopyCpuToGpu(index_, device_id, conf); + auto clone_result = clone_index->Search(query_dataset, conf); AssertAnns(clone_result, nq, k); ASSERT_THROW({ std::static_pointer_cast(clone_index)->GetRawVectors(); }, zilliz::knowhere::KnowhereException); @@ -172,24 +185,24 @@ TEST_F(IDMAPTest, copy_test) { auto binary = clone_index->Serialize(); clone_index->Load(binary); - auto new_result = clone_index->Search(query_dataset, Config::object{{"k", k}}); + auto new_result = clone_index->Search(query_dataset, conf); AssertAnns(new_result, nq, k); auto clone_gpu_idx = clone_index->Clone(); - auto clone_gpu_res = clone_gpu_idx->Search(query_dataset, Config::object{{"k", k}}); + auto clone_gpu_res = clone_gpu_idx->Search(query_dataset, conf); AssertAnns(clone_gpu_res, nq, k); // gpu to cpu - auto host_index = CopyGpuToCpu(clone_index, Config()); - auto host_result = host_index->Search(query_dataset, Config::object{{"k", k}}); + auto host_index = CopyGpuToCpu(clone_index, conf); + auto host_result = host_index->Search(query_dataset, conf); AssertAnns(host_result, nq, k); ASSERT_TRUE(std::static_pointer_cast(host_index)->GetRawVectors() != nullptr); ASSERT_TRUE(std::static_pointer_cast(host_index)->GetRawIds() != nullptr); // gpu to gpu - auto device_index = CopyCpuToGpu(index_, device_id, Config()); - auto new_device_index = std::static_pointer_cast(device_index)->CopyGpuToGpu(device_id, Config()); - auto device_result = new_device_index->Search(query_dataset, Config::object{{"k", k}}); + auto device_index = CopyCpuToGpu(index_, device_id, conf); + auto new_device_index = std::static_pointer_cast(device_index)->CopyGpuToGpu(device_id, conf); + auto device_result = new_device_index->Search(query_dataset, conf); AssertAnns(device_result, nq, k); } } diff --git a/cpp/src/core/test/test_ivf.cpp b/cpp/src/core/test/test_ivf.cpp index 5d29ae29ac..579729f2a1 100644 --- a/cpp/src/core/test/test_ivf.cpp +++ b/cpp/src/core/test/test_ivf.cpp @@ -25,23 +25,32 @@ #include #include -#include "knowhere/index/vector_index/IndexGPUIVF.h" -#include "knowhere/index/vector_index/IndexIVF.h" -#include "knowhere/adapter/Structure.h" -#include "knowhere/index/vector_index/utils/Cloner.h" #include "knowhere/common/Exception.h" #include "knowhere/common/Timer.h" +#include "knowhere/adapter/Structure.h" +#include "knowhere/index/vector_index/helpers/Cloner.h" +#include "knowhere/index/vector_index/IndexIVF.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexIVFPQ.h" +#include "knowhere/index/vector_index/IndexGPUIVFPQ.h" +#include "knowhere/index/vector_index/IndexIVFSQ.h" +#include "knowhere/index/vector_index/IndexGPUIVFSQ.h" #include "utils.h" - using namespace zilliz::knowhere; +using namespace zilliz::knowhere::cloner; using ::testing::TestWithParam; using ::testing::Values; using ::testing::Combine; -static int device_id = 0; +constexpr int device_id = 0; +constexpr int64_t DIM = 128; +constexpr int64_t NB = 1000000/100; +constexpr int64_t NQ = 10; +constexpr int64_t K = 10; + IVFIndexPtr IndexFactory(const std::string &type) { if (type == "IVF") { return std::make_shared(); @@ -58,14 +67,67 @@ IVFIndexPtr IndexFactory(const std::string &type) { } } +enum class ParameterType { + ivf, + ivfpq, + ivfsq, + nsg, +}; + +class ParamGenerator { + public: + static ParamGenerator& GetInstance(){ + static ParamGenerator instance; + return instance; + } + + Config Gen(const ParameterType& type){ + if (type == ParameterType::ivf) { + auto tempconf = std::make_shared(); + tempconf->d = DIM; + tempconf->gpu_id = device_id; + tempconf->nlist = 100; + tempconf->nprobe = 16; + tempconf->k = K; + tempconf->metric_type = METRICTYPE::L2; + return tempconf; + } + else if (type == ParameterType::ivfpq) { + auto tempconf = std::make_shared(); + tempconf->d = DIM; + tempconf->gpu_id = device_id; + tempconf->nlist = 100; + tempconf->nprobe = 16; + tempconf->k = K; + tempconf->m = 8; + tempconf->nbits = 8; + tempconf->metric_type = METRICTYPE::L2; + return tempconf; + } + else if (type == ParameterType::ivfsq) { + auto tempconf = std::make_shared(); + tempconf->d = DIM; + tempconf->gpu_id = device_id; + tempconf->nlist = 100; + tempconf->nprobe = 16; + tempconf->k = K; + tempconf->nbits = 8; + tempconf->metric_type = METRICTYPE::L2; + return tempconf; + } + } +}; + class IVFTest - : public DataGen, public TestWithParam<::std::tuple> { + : public DataGen, public TestWithParam<::std::tuple> { protected: void SetUp() override { - std::tie(index_type, preprocess_cfg, train_cfg, add_cfg, search_cfg) = GetParam(); + ParameterType parameter_type; + std::tie(index_type, parameter_type) = GetParam(); //Init_with_default(); - Generate(128, 1000000/100, 10); + Generate(DIM, NB, NQ); index_ = IndexFactory(index_type); + conf = ParamGenerator::GetInstance().Gen(parameter_type); FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024*1024*200, 1024*1024*600, 2); } void TearDown() override { @@ -74,46 +136,20 @@ class IVFTest protected: std::string index_type; - Config preprocess_cfg; - Config train_cfg; - Config add_cfg; - Config search_cfg; + Config conf; IVFIndexPtr index_ = nullptr; }; + INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest, Values( - std::make_tuple("IVF", - Config(), - Config::object{{"nlist", 100}, {"metric_type", "L2"}}, - Config(), - Config::object{{"k", 10}}), - //std::make_tuple("IVFPQ", - // Config(), - // Config::object{{"nlist", 100}, {"M", 8}, {"nbits", 8}, {"metric_type", "L2"}}, - // Config(), - // Config::object{{"k", 10}}), - std::make_tuple("GPUIVF", - Config(), - Config::object{{"nlist", 100}, {"gpu_id", device_id}, {"metric_type", "L2"}}, - Config(), - Config::object{{"k", 10}}), - //std::make_tuple("GPUIVFPQ", - // Config(), - // Config::object{{"gpu_id", device_id}, {"nlist", 100}, {"M", 8}, {"nbits", 8}, {"metric_type", "L2"}}, - // Config(), - // Config::object{{"k", 10}}), - std::make_tuple("IVFSQ", - Config(), - Config::object{{"nlist", 100}, {"nbits", 8}, {"metric_type", "L2"}}, - Config(), - Config::object{{"k", 10}}), - std::make_tuple("GPUIVFSQ", - Config(), - Config::object{{"gpu_id", device_id}, {"nlist", 100}, {"nbits", 8}, {"metric_type", "L2"}}, - Config(), - Config::object{{"k", 10}}) + std::make_tuple("IVF", ParameterType::ivf), + std::make_tuple("GPUIVF", ParameterType::ivf), + std::make_tuple("IVFPQ", ParameterType::ivfpq), + std::make_tuple("GPUIVFPQ", ParameterType::ivfpq), + std::make_tuple("IVFSQ", ParameterType::ivfsq), + std::make_tuple("GPUIVFSQ", ParameterType::ivfsq) ) ); @@ -149,16 +185,16 @@ void PrintResult(const DatasetPtr &result, TEST_P(IVFTest, ivf_basic) { assert(!xb.empty()); - auto preprocessor = index_->BuildPreprocessor(base_dataset, preprocess_cfg); + auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); index_->set_preprocessor(preprocessor); - auto model = index_->Train(base_dataset, train_cfg); + auto model = index_->Train(base_dataset, conf); index_->set_index_model(model); - index_->Add(base_dataset, add_cfg); + index_->Add(base_dataset, conf); EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dimension(), dim); - auto result = index_->Search(query_dataset, search_cfg); - AssertAnns(result, nq, k); + auto result = index_->Search(query_dataset, conf); + AssertAnns(result, nq, conf->k); //PrintResult(result, nq, k); } @@ -168,20 +204,20 @@ TEST_P(IVFTest, ivf_basic) { // // else // assert(!xb.empty()); // -// auto preprocessor = index_->BuildPreprocessor(base_dataset, preprocess_cfg); +// auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); // index_->set_preprocessor(preprocessor); // -// auto model = index_->Train(base_dataset, train_cfg); +// auto model = index_->Train(base_dataset, conf); // index_->set_index_model(model); -// index_->Add(base_dataset, add_cfg); +// index_->Add(base_dataset, conf); // EXPECT_EQ(index_->Count(), nb); // EXPECT_EQ(index_->Dimension(), dim); -// auto result = index_->Search(query_dataset, search_cfg); +// auto result = index_->Search(query_dataset, conf); // AssertAnns(result, nq, k); // // if (auto device_index = std::dynamic_pointer_cast(index_)) { // auto host_index = device_index->Copy_index_gpu_to_cpu(); -// auto result = host_index->Search(query_dataset, search_cfg); +// auto result = host_index->Search(query_dataset, conf); // AssertAnns(result, nq, k); // } //} @@ -197,7 +233,7 @@ TEST_P(IVFTest, ivf_serialize) { { // serialize index-model - auto model = index_->Train(base_dataset, train_cfg); + auto model = index_->Train(base_dataset, conf); auto binaryset = model->Serialize(); auto bin = binaryset.GetByName("IVF"); @@ -213,16 +249,16 @@ TEST_P(IVFTest, ivf_serialize) { model->Load(binaryset); index_->set_index_model(model); - index_->Add(base_dataset, add_cfg); - auto result = index_->Search(query_dataset, search_cfg); - AssertAnns(result, nq, k); + index_->Add(base_dataset, conf); + auto result = index_->Search(query_dataset, conf); + AssertAnns(result, nq, conf->k); } { // serialize index - auto model = index_->Train(base_dataset, train_cfg); + auto model = index_->Train(base_dataset, conf); index_->set_index_model(model); - index_->Add(base_dataset, add_cfg); + index_->Add(base_dataset, conf); auto binaryset = index_->Serialize(); auto bin = binaryset.GetByName("IVF"); @@ -238,24 +274,24 @@ TEST_P(IVFTest, ivf_serialize) { index_->Load(binaryset); EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dimension(), dim); - auto result = index_->Search(query_dataset, search_cfg); - AssertAnns(result, nq, k); + auto result = index_->Search(query_dataset, conf); + AssertAnns(result, nq, conf->k); } } TEST_P(IVFTest, clone_test) { assert(!xb.empty()); - auto preprocessor = index_->BuildPreprocessor(base_dataset, preprocess_cfg); + auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); index_->set_preprocessor(preprocessor); - auto model = index_->Train(base_dataset, train_cfg); + auto model = index_->Train(base_dataset, conf); index_->set_index_model(model); - index_->Add(base_dataset, add_cfg); + index_->Add(base_dataset, conf); EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dimension(), dim); - auto result = index_->Search(query_dataset, search_cfg); - AssertAnns(result, nq, k); + auto result = index_->Search(query_dataset, conf); + AssertAnns(result, nq, conf->k); //PrintResult(result, nq, k); auto AssertEqual = [&] (DatasetPtr p1, DatasetPtr p2) { @@ -275,8 +311,8 @@ TEST_P(IVFTest, clone_test) { if (finder != support_idx_vec.cend()) { EXPECT_NO_THROW({ auto clone_index = index_->Clone(); - auto clone_result = clone_index->Search(query_dataset, search_cfg); - //AssertAnns(result, nq, k); + auto clone_result = clone_index->Search(query_dataset, conf); + //AssertAnns(result, nq, conf->k); AssertEqual(result, clone_result); std::cout << "inplace clone [" << index_type << "] success" << std::endl; }); @@ -295,7 +331,7 @@ TEST_P(IVFTest, clone_test) { if (finder != support_idx_vec.cend()) { EXPECT_NO_THROW({ auto clone_index = CopyGpuToCpu(index_, Config()); - auto clone_result = clone_index->Search(query_dataset, search_cfg); + auto clone_result = clone_index->Search(query_dataset, conf); AssertEqual(result, clone_result); std::cout << "clone G <=> C [" << index_type << "] success" << std::endl; }); @@ -314,7 +350,7 @@ TEST_P(IVFTest, clone_test) { if (finder != support_idx_vec.cend()) { EXPECT_NO_THROW({ auto clone_index = CopyCpuToGpu(index_, device_id, Config()); - auto clone_result = clone_index->Search(query_dataset, search_cfg); + auto clone_result = clone_index->Search(query_dataset, conf); AssertEqual(result, clone_result); std::cout << "clone C <=> G [" << index_type << "] success" << std::endl; }); @@ -338,17 +374,16 @@ TEST_P(IVFTest, seal_test) { assert(!xb.empty()); - //index_ = std::make_shared(0); - auto preprocessor = index_->BuildPreprocessor(base_dataset, preprocess_cfg); + auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); index_->set_preprocessor(preprocessor); - auto model = index_->Train(base_dataset, train_cfg); + auto model = index_->Train(base_dataset, conf); index_->set_index_model(model); - index_->Add(base_dataset, add_cfg); + index_->Add(base_dataset, conf); EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dimension(), dim); - auto result = index_->Search(query_dataset, search_cfg); - AssertAnns(result, nq, k); + auto result = index_->Search(query_dataset, conf); + AssertAnns(result, nq, conf->k); auto cpu_idx = CopyGpuToCpu(index_, Config()); @@ -367,13 +402,10 @@ class GPURESTEST : public DataGen, public ::testing::Test { protected: void SetUp() override { - //std::tie(index_type, preprocess_cfg, train_cfg, add_cfg, search_cfg) = GetParam(); - //Init_with_default(); Generate(128, 1000000, 1000); - k = 100; - //index_ = IndexFactory(index_type); FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024*1024*200, 1024*1024*300, 2); + k = 100; elems = nq * k; ids = (int64_t *) malloc(sizeof(int64_t) * elems); dis = (float *) malloc(sizeof(float) * elems); @@ -387,10 +419,6 @@ class GPURESTEST protected: std::string index_type; - Config preprocess_cfg; - Config train_cfg; - Config add_cfg; - Config search_cfg; IVFIndexPtr index_ = nullptr; int64_t *ids = nullptr; @@ -404,26 +432,31 @@ const int load_count = 3; TEST_F(GPURESTEST, gpu_ivf_resource_test) { assert(!xb.empty()); - { index_ = std::make_shared(-1); ASSERT_EQ(std::dynamic_pointer_cast(index_)->GetGpuDevice(), -1); std::dynamic_pointer_cast(index_)->SetGpuDevice(device_id); ASSERT_EQ(std::dynamic_pointer_cast(index_)->GetGpuDevice(), device_id); - auto preprocessor = index_->BuildPreprocessor(base_dataset, preprocess_cfg); + auto conf = std::make_shared(); + conf->nlist = 1638; + conf->d = dim; + conf->gpu_id = device_id; + conf->metric_type = METRICTYPE::L2; + conf->k = k; + conf->nprobe = 1; + + auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); index_->set_preprocessor(preprocessor); - train_cfg = Config::object{{"nlist", 1638}, {"gpu_id", device_id}, {"metric_type", "L2"}}; - auto model = index_->Train(base_dataset, train_cfg); + auto model = index_->Train(base_dataset, conf); index_->set_index_model(model); - index_->Add(base_dataset, add_cfg); + index_->Add(base_dataset, conf); EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dimension(), dim); - search_cfg = Config::object{{"k", k}}; TimeRecorder tc("knowere GPUIVF"); for (int i = 0; i < search_count; ++i) { - index_->Search(query_dataset, search_cfg); + index_->Search(query_dataset, conf); if (i > search_count - 6 || i < 5) tc.RecordSection("search once"); } @@ -456,14 +489,22 @@ TEST_F(GPURESTEST, gpuivfsq) { // knowhere gpu ivfsq index_type = "GPUIVFSQ"; index_ = IndexFactory(index_type); - auto preprocessor = index_->BuildPreprocessor(base_dataset, preprocess_cfg); + + auto conf = std::make_shared(); + conf->nlist = 1638; + conf->d = dim; + conf->gpu_id = device_id; + conf->metric_type = METRICTYPE::L2; + conf->k = k; + conf->nbits = 8; + conf->nprobe = 1; + + auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); index_->set_preprocessor(preprocessor); - train_cfg = Config::object{{"gpu_id", device_id}, {"nlist", 1638}, {"nbits", 8}, {"metric_type", "L2"}}; - auto model = index_->Train(base_dataset, train_cfg); + auto model = index_->Train(base_dataset, conf); index_->set_index_model(model); - index_->Add(base_dataset, add_cfg); - search_cfg = Config::object{{"k", k}}; - auto result = index_->Search(query_dataset, search_cfg); + index_->Add(base_dataset, conf); + auto result = index_->Search(query_dataset, conf); AssertAnns(result, nq, k); auto cpu_idx = CopyGpuToCpu(index_, Config()); @@ -473,7 +514,7 @@ TEST_F(GPURESTEST, gpuivfsq) { auto search_idx = CopyCpuToGpu(cpu_idx, device_id, Config()); tc.RecordSection("Copy to gpu"); for (int i = 0; i < search_count; ++i) { - search_idx->Search(query_dataset, search_cfg); + search_idx->Search(query_dataset, conf); if (i > search_count - 6 || i < 5) tc.RecordSection("search once"); } @@ -517,20 +558,27 @@ TEST_F(GPURESTEST, gpuivfsq) { } TEST_F(GPURESTEST, copyandsearch) { + // search and copy at the same time printf("==================\n"); - // search and copy at the same time index_type = "GPUIVFSQ"; - //index_type = "GPUIVF"; index_ = IndexFactory(index_type); - auto preprocessor = index_->BuildPreprocessor(base_dataset, preprocess_cfg); + + auto conf = std::make_shared(); + conf->nlist = 1638; + conf->d = dim; + conf->gpu_id = device_id; + conf->metric_type = METRICTYPE::L2; + conf->k = k; + conf->nbits = 8; + conf->nprobe = 1; + + auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); index_->set_preprocessor(preprocessor); - train_cfg = Config::object{{"gpu_id", device_id}, {"nlist", 1638}, {"nbits", 8}, {"metric_type", "L2"}}; - auto model = index_->Train(base_dataset, train_cfg); + auto model = index_->Train(base_dataset, conf); index_->set_index_model(model); - index_->Add(base_dataset, add_cfg); - search_cfg = Config::object{{"k", k}}; - auto result = index_->Search(query_dataset, search_cfg); + index_->Add(base_dataset, conf); + auto result = index_->Search(query_dataset, conf); AssertAnns(result, nq, k); auto cpu_idx = CopyGpuToCpu(index_, Config()); @@ -541,7 +589,7 @@ TEST_F(GPURESTEST, copyandsearch) { auto search_func = [&] { //TimeRecorder tc("search&load"); for (int i = 0; i < search_count; ++i) { - search_idx->Search(query_dataset, search_cfg); + search_idx->Search(query_dataset, conf); //if (i > search_count - 6 || i == 0) // tc.RecordSection("search once"); } @@ -560,7 +608,7 @@ TEST_F(GPURESTEST, copyandsearch) { TimeRecorder tc("basic"); CopyCpuToGpu(cpu_idx, device_id, Config()); tc.RecordSection("Copy to gpu once"); - search_idx->Search(query_dataset, search_cfg); + search_idx->Search(query_dataset, conf); tc.RecordSection("search once"); search_func(); tc.RecordSection("only search total"); @@ -576,35 +624,40 @@ TEST_F(GPURESTEST, copyandsearch) { TEST_F(GPURESTEST, TrainAndSearch) { index_type = "GPUIVFSQ"; - //index_type = "GPUIVF"; - const int train_count = 1; - const int search_count = 5000; - index_ = IndexFactory(index_type); - auto preprocessor = index_->BuildPreprocessor(base_dataset, preprocess_cfg); + + auto conf = std::make_shared(); + conf->nlist = 1638; + conf->d = dim; + conf->gpu_id = device_id; + conf->metric_type = METRICTYPE::L2; + conf->k = k; + conf->nbits = 8; + conf->nprobe = 1; + + auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); index_->set_preprocessor(preprocessor); - train_cfg = Config::object{{"gpu_id", device_id}, {"nlist", 1638}, {"nbits", 8}, {"metric_type", "L2"}}; - auto model = index_->Train(base_dataset, train_cfg); + auto model = index_->Train(base_dataset, conf); auto new_index = IndexFactory(index_type); new_index->set_index_model(model); - new_index->Add(base_dataset, add_cfg); + new_index->Add(base_dataset, conf); auto cpu_idx = CopyGpuToCpu(new_index, Config()); cpu_idx->Seal(); auto search_idx = CopyCpuToGpu(cpu_idx, device_id, Config()); + constexpr int train_count = 1; + constexpr int search_count = 5000; auto train_stage = [&] { - train_cfg = Config::object{{"gpu_id", device_id}, {"nlist", 1638}, {"nbits", 8}, {"metric_type", "L2"}}; for (int i = 0; i < train_count; ++i) { - auto model = index_->Train(base_dataset, train_cfg); + auto model = index_->Train(base_dataset, conf); auto test_idx = IndexFactory(index_type); test_idx->set_index_model(model); - test_idx->Add(base_dataset, add_cfg); + test_idx->Add(base_dataset, conf); } }; auto search_stage = [&](VectorIndexPtr& search_idx) { - search_cfg = Config::object{{"k", k}}; for (int i = 0; i < search_count; ++i) { - auto result = search_idx->Search(query_dataset, search_cfg); + auto result = search_idx->Search(query_dataset, conf); AssertAnns(result, nq, k); } }; diff --git a/cpp/src/core/test/test_kdt.cpp b/cpp/src/core/test/test_kdt.cpp index 8874b14fc9..fce233aca7 100644 --- a/cpp/src/core/test/test_kdt.cpp +++ b/cpp/src/core/test/test_kdt.cpp @@ -20,10 +20,10 @@ #include #include -#include "knowhere/common/Exception.h" +#include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/IndexKDT.h" -#include "knowhere/index/vector_index/utils/Definitions.h" +#include "knowhere/index/vector_index/helpers/Definitions.h" #include "knowhere/adapter/SptagAdapter.h" #include "knowhere/adapter/Structure.h" @@ -38,31 +38,24 @@ using ::testing::Combine; class KDTTest - : public DataGen, public TestWithParam<::std::tuple> { + : public DataGen, public ::testing::Test { protected: void SetUp() override { - std::tie(preprocess_cfg, train_cfg, add_cfg, search_cfg) = GetParam(); index_ = std::make_shared(); + + auto tempconf = std::make_shared(); + tempconf->tptnubmber = 1; + tempconf->k = 10; + conf = tempconf; + Init_with_default(); } protected: - Config preprocess_cfg; - Config train_cfg; - Config add_cfg; - Config search_cfg; + Config conf; std::shared_ptr index_ = nullptr; }; -INSTANTIATE_TEST_CASE_P(KDTParameters, KDTTest, - Values( - std::make_tuple(Config(), - Config::object{{"TPTNumber", 1}}, - Config(), - Config::object{{"k", 10}}) - ) -); - void AssertAnns(const DatasetPtr &result, const int &nq, const int &k) { @@ -93,16 +86,16 @@ void PrintResult(const DatasetPtr &result, } // TODO(linxj): add test about count() and dimension() -TEST_P(KDTTest, kdt_basic) { +TEST_F(KDTTest, kdt_basic) { assert(!xb.empty()); - auto preprocessor = index_->BuildPreprocessor(base_dataset, preprocess_cfg); + auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); index_->set_preprocessor(preprocessor); - auto model = index_->Train(base_dataset, train_cfg); + auto model = index_->Train(base_dataset, conf); index_->set_index_model(model); - index_->Add(base_dataset, add_cfg); - auto result = index_->Search(query_dataset, search_cfg); + index_->Add(base_dataset, conf); + auto result = index_->Search(query_dataset, conf); AssertAnns(result, nq, k); { @@ -124,18 +117,18 @@ TEST_P(KDTTest, kdt_basic) { } } -TEST_P(KDTTest, kdt_serialize) { +TEST_F(KDTTest, kdt_serialize) { assert(!xb.empty()); - auto preprocessor = index_->BuildPreprocessor(base_dataset, preprocess_cfg); + auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); index_->set_preprocessor(preprocessor); - auto model = index_->Train(base_dataset, train_cfg); - //index_->Add(base_dataset, add_cfg); + auto model = index_->Train(base_dataset, conf); + //index_->Add(base_dataset, conf); auto binaryset = index_->Serialize(); auto new_index = std::make_shared(); new_index->Load(binaryset); - auto result = new_index->Search(query_dataset, search_cfg); + auto result = new_index->Search(query_dataset, conf); AssertAnns(result, nq, k); PrintResult(result, nq, k); ASSERT_EQ(new_index->Count(), nb); @@ -172,7 +165,7 @@ TEST_P(KDTTest, kdt_serialize) { auto new_index = std::make_shared(); new_index->Load(load_data_list); - auto result = new_index->Search(query_dataset, search_cfg); + auto result = new_index->Search(query_dataset, conf); AssertAnns(result, nq, k); PrintResult(result, nq, k); } diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/.doozer.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/.doozer.json deleted file mode 100644 index 13edcea3b0..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/.doozer.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "targets": { - "centos7-x86_64": { - "buildenv": "centos7-x86_64", - "builddeps": ["gcc-c++", "make", "wget"], - "environment": { - "CXXFLAGS": "-std=c++11 -Wall -g3" - }, - "buildcmd": [ - "uname -a", - "rpm -q centos-release", - "g++ --version", - "cd", - "wget https://github.com/Kitware/CMake/releases/download/v3.14.0/cmake-3.14.0.tar.gz", - "tar xfz cmake-3.14.0.tar.gz", - "cd cmake-3.14.0", - "./bootstrap", - "make -j8", - "cd", - "cmake-3.14.0/bin/cmake . -DBUILD_TESTS=ON", - "cmake-3.14.0/bin/cmake --build . --target test_jsoncons", - "cd tests", - "./test_jsoncons;" - ] - }, - "fedora24-x86_64": { - "buildenv": "fedora24-x86_64", - "builddeps": ["cmake", "make", "gcc gcc-c++"], - "environment": { - "CXXFLAGS": "-std=c++11 -Wall -g3" - }, - "buildcmd": [ - "uname -a", - "cat /etc/fedora-release", - "g++ --version", - "cmake . -DBUILD_TESTS=ON", - "cmake --build . --target test_jsoncons", - "cd tests", - "./test_jsoncons;" - ] - } - } -} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/.gitignore b/cpp/src/core/thirdparty/jsoncons-0.126.0/.gitignore deleted file mode 100644 index 97651319ef..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/.gitignore +++ /dev/null @@ -1,30 +0,0 @@ -*.suo -*.sdf -*.VC.opendb -*.VC.db -*.vcxproj.user -*.vcxproj.filters -**/x64/Debug/ -**/x64/Release/ -**/*.vcxproj -**/vs2017 -**/vs2015 -**/vs2013 -**/temp -**/build - -examples/build/cmake/build_llvm.sh -examples/build/cmake/build_llvm - -src/jsoncons/json_structures.hpp.orig - -examples/build/vs2013/ - -src/jsoncons/json_serializer2.hpp - -examples/booklist.json -examples/booklist2.json - -examples/build/vs2017/.vs/vs2017/v15/ipch/AutoPCH/2f913b3e9413499/BASICS_EXAMPLES.ipch -examples/build/vs2017/.vs/vs2017/v15/ipch/AutoPCH/8571a45cc244269f/EXAMPLES.ipch -examples/build/vs2017/vs2017.sln diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/.travis.yml b/cpp/src/core/thirdparty/jsoncons-0.126.0/.travis.yml deleted file mode 100644 index 45773607c2..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/.travis.yml +++ /dev/null @@ -1,240 +0,0 @@ -language: cpp -dist: trusty -sudo: required - -matrix: - include: - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-4.8 - - libgmp-dev - env: COMPILER=gcc VERSION=4.8 CXXFLAGS="-std=c++11" - - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-4.9 - - libgmp-dev - env: COMPILER=gcc VERSION=4.9 JSONCONS_SANITIZE=1 CXXFLAGS="-std=c++11" - - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-5 - env: COMPILER=gcc VERSION=5 CXXFLAGS="-std=gnu++11" - - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-6 - env: COMPILER=gcc VERSION=6 JSONCONS_SANITIZE=1 - - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-7 - env: COMPILER=gcc VERSION=7 JSONCONS_SANITIZE=1 - - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-8 - env: COMPILER=gcc VERSION=8 JSONCONS_SANITIZE=1 CXXFLAGS="-std=c++17 -Werror -Wall -Wextra -Wimplicit-fallthrough -pedantic -Wcast-align -Wcast-qual" - - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-8 - env: COMPILER=gcc VERSION=8 JSONCONS_SANITIZE=1 CXXFLAGS="-std=c++17 -DJSONCONS_HAS_STRING_VIEW" - - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-8 - env: COMPILER=gcc VERSION=8 JSONCONS_SANITIZE=1 CXXFLAGS="-std=gnu++17 -Wall -Wextra -Wimplicit-fallthrough" - - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - clang-3.9 - - g++-4.8-aarch64-linux-gnu - - gcc-4.8-aarch64-linux-gnu - - g++-4.8-multilib - - gcc-4.8-multilib - - qemu - - qemu-system-arm - env: COMPILER=clang VERSION=3.9 CROSS_COMPILE=1 ARM_ARCH_DIR=aarch64-linux-gnu GCC_VER=4.8.4 ARM_SETTINGS="armv8-a -target aarch64-linux-gnueabi" CXXFLAGS="-std=c++11" - - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-trusty-4.0 - packages: - - clang-4.0 - - g++-7 - env: COMPILER=clang VERSION=4.0 CXXFLAGS="-std=c++11" - - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-trusty-5.0 - packages: - - clang-5.0 - - g++-7 - env: COMPILER=clang VERSION=5.0 CXXFLAGS="-std=gnu++11" - - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-trusty-6.0 - packages: - - clang-6.0 - - g++-7 - env: COMPILER=clang VERSION=6.0 JSONCONS_SANITIZE=1 CXXFLAGS="-std=c++11 -Wall -Wextra -Wimplicit-fallthrough" - - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-trusty-6.0 - packages: - - clang-6.0 - - g++-7 - env: COMPILER=clang VERSION=6.0 CXXFLAGS="-DJSONCONS_NO_DEPRECATED" - - - os: osx - osx_image: xcode7.3 - compiler: clang - env: CXXFLAGS="-std=c++11" - - - os: osx - osx_image: xcode8 - compiler: clang - env: CXXFLAGS="-std=c++11" - - - os: osx - osx_image: xcode8.1 - compiler: clang - env: CXXFLAGS="-std=c++11" - - - os: osx - osx_image: xcode8.2 - compiler: clang - env: CXXFLAGS="-std=c++11" - - - os: osx - osx_image: xcode8.3 - compiler: clang - env: CXXFLAGS="-std=c++11" - - - os: osx - osx_image: xcode9 - compiler: clang - env: CXXFLAGS="-std=c++11" - - - os: osx - osx_image: xcode9.1 - compiler: clang - env: CXXFLAGS="-std=c++11" - - - os: osx - osx_image: xcode9.2 - compiler: clang - env: CXXFLAGS="-std=c++11" - - - os: osx - osx_image: xcode9.3 - compiler: clang - env: CXXFLAGS="-std=c++11" - - - os: osx - osx_image: xcode9.4 - compiler: clang - env: CXXFLAGS="-std=c++11" - -before_install: - - | - # Configure build variables - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then - if [[ "$COMPILER" == "gcc" ]]; then - export CXX=g++-$VERSION CC=gcc-$VERSION; - fi - if [[ "$COMPILER" == "clang" ]]; then - export CXX=clang++-$VERSION CC=clang-$VERSION; - fi - elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - export CXX=clang++ CC=clang; - fi -install: - # get CMake (only for systems with brew - macOS) - - | - if [[ (-x $(which brew)) ]]; then - brew update - brew install cmake - brew upgrade cmake - cmake --version - fi - - - if [[ "$CROSS_COMPILE" == 1 ]] ; then - if [[ "$ARM_ARCH_DIR" == "aarch64-linux-gnu" ]] ; then - mkdir $HOME/linker_bin ; - ln -s /usr/bin/aarch64-linux-gnu-ld $HOME/linker_bin/ld ; - echo "SETTING GNU LINKER DIR" ; - ls -al $HOME/linker_bin/ld ; - cmake . -DBUILD_TESTS=ON -DCROSS_COMPILE_ARM=ON -DDOWNLOAD_GTEST=ON -DARM_ARCH_DIRECTORY="$ARM_ARCH_DIR" -DARM_GCC_VER="$GCC_VER" -DTARGET_ARCH="$ARM_SETTINGS --prefix=$HOME/linker_bin/" ${CMAKE_OPTIONS}; - else - cmake . -DBUILD_TESTS=ON -DCROSS_COMPILE_ARM=ON -DDOWNLOAD_GTEST=ON -DARM_ARCH_DIRECTORY="$ARM_ARCH_DIR" -DARM_GCC_VER="$GCC_VER" -DTARGET_ARCH="$ARM_SETTINGS" ${CMAKE_OPTIONS}; - fi - else - cmake . -DBUILD_TESTS=ON ${CMAKE_OPTIONS}; - fi - - make -j2 test_jsoncons - - cd tests -script: - - if [[ "$JSONCONS_VALGRIND" == 1 ]]; then - ctest -T memcheck; - fi - - if [[ "$CROSS_COMPILE" == 1 ]]; then - if [[ "$ARM_ARCH_DIR" == "aarch64-linux-gnu" ]]; then - qemu-aarch64 -L /usr/aarch64-linux-gnu/ ./test_jsoncons ; - else - qemu-arm -L /usr/arm-linux-gnueabi/ ./test_jsoncons ; - fi - else - ./test_jsoncons; - fi diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/CMakeLists.txt b/cpp/src/core/thirdparty/jsoncons-0.126.0/CMakeLists.txt deleted file mode 100644 index 1c067413a7..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/CMakeLists.txt +++ /dev/null @@ -1,73 +0,0 @@ -cmake_minimum_required(VERSION 3.1) - -project(jsoncons) - -set(JSONCONS_PROJECT_DIR ${PROJECT_SOURCE_DIR}) -set(JSONCONS_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include) - -# Versioning -# ========== - -file(STRINGS "${JSONCONS_INCLUDE_DIR}/jsoncons/config/version.hpp" jsoncons_version_defines - REGEX "#define JSONCONS_VERSION_(MAJOR|MINOR|PATCH)") -foreach(ver ${jsoncons_version_defines}) - if(ver MATCHES "#define JSONCONS_VERSION_(MAJOR|MINOR|PATCH) +([^ ]+)$") - set(JSONCONS_VERSION_${CMAKE_MATCH_1} "${CMAKE_MATCH_2}" CACHE INTERNAL "") - endif() -endforeach() -set(${PROJECT_NAME}_VERSION - ${JSONCONS_VERSION_MAJOR}.${JSONCONS_VERSION_MINOR}.${JSONCONS_VERSION_PATCH}) -message(STATUS "jsoncons v${${PROJECT_NAME}_VERSION}") - -# Build -# ===== - - -file(GLOB_RECURSE JSONCONS_HEADERS ${JSONCONS_INCLUDE_DIR}/*.hpp) - -add_library(jsoncons INTERFACE) -target_include_directories(jsoncons INTERFACE $ - $) - -OPTION(BUILD_TESTS "jsoncons test suite" ON) - -if(BUILD_TESTS) - add_subdirectory(tests) -endif() - -# Installation -# ============ - -include(GNUInstallDirs) -include(CMakePackageConfigHelpers) - -install(TARGETS jsoncons - EXPORT ${PROJECT_NAME}-targets) - -# Makes the project importable from the build directory -export(EXPORT ${PROJECT_NAME}-targets - FILE "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Targets.cmake") - -install(DIRECTORY ${JSONCONS_INCLUDE_DIR}/jsoncons - ${JSONCONS_INCLUDE_DIR}/jsoncons_ext - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) - -# GNUInstallDirs "DATADIR" wrong here; CMake search path wants "share". -set(JSONCONS_CMAKECONFIG_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}" CACHE STRING "install path for jsonconsConfig.cmake") - -configure_package_config_file(build_files/cmake/config.cmake.in - "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake" - INSTALL_DESTINATION ${JSONCONS_CMAKECONFIG_INSTALL_DIR}) - -# jsoncons is header-only and does not depend on the architecture. - -write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake - VERSION ${${PROJECT_NAME}_VERSION} - COMPATIBILITY AnyNewerVersion) -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake - ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake - DESTINATION ${JSONCONS_CMAKECONFIG_INSTALL_DIR}) -install(EXPORT ${PROJECT_NAME}-targets - FILE ${PROJECT_NAME}Targets.cmake - DESTINATION ${JSONCONS_CMAKECONFIG_INSTALL_DIR}) - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/ChangeLog.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/ChangeLog.md deleted file mode 100644 index 2f9175c98b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/ChangeLog.md +++ /dev/null @@ -1,2129 +0,0 @@ -master ------- - -Enhancements - -- The `json_reader` and `csv_reader` constructors have been generalized to take either a value -from which a `jsoncons::string_view` is constructible (e.g. std::string), or a value from which -a `source_type` is constructible (e.g. std::istream). - -- With this enhancement, the convenience typedefs `json_string_reader` and `csv_string_reader` are no -longer needed, and have been deprecated. - -Name change - -- The name `json_pull_reader` has been deprecated (still works) and renamed to `json_cursor` - -Changes to bigfloat mapping - -In previous versions, jsoncons arrays that contained - -- an int64_t or a uint64_t (defines base-2 exponent) -- an int64_t or a uint64_t or a string tagged with `semantic_tag::bigint` (defines the mantissa) - -and that were tagged with `semantic_tag::bigfloat`, were encoded into CBOR bigfloats. -This behaviour has been deprecated. - -CBOR bigfloats are now decoded into a jsoncons string that consists of the following parts - -- (optional) minus sign -- 0x -- nonempty sequence of hexadecimal digits (defines mantissa) -- p followed with optional minus or plus sign and nonempty sequence of hexadecimal digits (defines base-2 exponent) - -and tagged with `semantic_tag::bigfloat` (before they were decoded into a jsoncons array and tagged with `semantic_tag::bigfloat`) - -jsoncons strings that consist of the following parts - -- (optional) plus or minus sign -- 0x or 0X -- nonempty sequence of hexadecimal digits optionally containing a decimal-point character -- (optional) p or P followed with optional minus or plus sign and nonempty sequence of decimal digits, - -and tagged with `semantic_tag::bigfloat` are now encoded into CBOR bignums. - -v0.125.0 --------- - -CMake build fix - -- The CMAKE_BUILD_TYPE variable is now left alone if already set - -Non-breaking changes to lighten `semantic_tag` names - -- The `semantic_tag` enum values `big_integer`, `big_decimal`, `big_float` and `date_time` - have been deprecated (still work) and renamed to `bigint`, `bigdec`, `bigfloat` and `datetime`. - -- The `json_content_handler` functions `big_integer_value`, `big_decimal_value`, `date_time_value` - and `timestamp_value` have been deprecated (still work.) Calls to these functions should be replaced - by calls to `string_value` with `semantic_tag::bigint`, `semantic_tag::bigdec`, and `semantic_tag::datetime`, - and by calls to `int64_vaue` with `semantic_tag::timestamp`. - -- The enum type `big_integer_chars_format` has been deprecated (still works) and renamed to - `bigint_chars_format`. - -- The `json_options` modifier `big_integer_format` has been deprecated (still works) and renamed to - `bigint_format`. - -Non-breaking changes to `ser_context`, `ser_error` and `jsonpath_error` - -- The function names `line_number` and `column_number` have been deprecated (still work) and - renamed to `line` and `column`. - -v0.124.0 --------- - -- Fixed bug in `json_encoder` with `pad_inside_array_brackets` and `pad_inside_object_braces` options - -- Fixed issue with escape character in jsonpath quoted names. - -- Fixed pedantic level compiler warnings - -- Added doozer tests for CentOS 7.6 and Fedora release 24 - -- New macro `JSONCONS_GETTER_CTOR_TRAITS_DECL` that can be used to generate the `json_type_traits` boilerplate -from getter functions and a constructor. - -v0.123.2 --------- - -defect fix: - -- Fixed name of JSONCONS_MEMBER_TRAITS_DECL - -v0.123.1 --------- - -jsonpath bug fixes: - -- Fixed bug in construction of normalized paths [#177](https://github.com/danielaparker/jsoncons/issues/177). - -- Fixed bug in jsonpath recursive descent with filters which could result in too many values being returned - -v0.123.0 --------- - -Warning fix: - -- Removed redundant macro continuation character [#176](https://github.com/danielaparker/jsoncons/pull/176) - -Non-breaking change to names (old name still works) - -- The name JSONCONS_TYPE_TRAITS_DECL has been deprecated and changed to JSONCONS_MEMBER_TRAITS_DECL - -Changes to jsonpath: - -- jsonpath unions now return distinct values (no duplicates) -- a single dot immediately followed by a left bracket now results in an error (illegal JSONPath) - -Enhancements to jsonpath - -- Union of completely separate paths are allowed, e.g. - - $[firstName,address.city] - -- Names in the dot notation may be single or double quoted - -Other enhancements: - -- `basic_json` now supports operators `<`, `<=`, `>`, `>=` - -v0.122.0 --------- - -Changes: - -- The template parameter `CharT` has been removed from the binary encoders - `basic_bson_encoder`, `basic_cbor_encoder`, `basic_msgpack_encoder`, - and `basic_ubjson_encoder`. - -Enhancements: - -- Added macro JSONCONS_TYPE_TRAITS_FRIEND - -- Generalized the `csv_encode`, `bson_encode`, `cbor_encode`, `msgpack_encode` and `ubjson_encode` functions to convert from any type T that implements `json_type_traits` - -- Generalized the `csv_decode`, `bson_decode`, `cbor_decode`, `msgpack_decode` and `ubjson_decode` functions to convert to any type T that implements `json_type_traits` - -v0.121.1 --------- - -Bug fix: - -- Fixed issue with cbor_reader only reading tag values 0 through 23 - -Name change - -- The name `json::semantic_tag()` has been renamed to `json::get_semantic_tag()` - -Non-breaking changes (old names still work) - -- The name `semantic_tag_type` has been deprecated and renamed to `semantic_tag` -- The names `json_serializer`, `bson_serializer`, `cbor_serializer`, `csv_serializer`, `msgpack_serializer`, - and `ubjson_serializer` have been deprecated and renamed to `json_encoder`, `bson_encoder`, - `cbor_encoder`, `csv_encoder`, `msgpack_encoder`, and `ubjson_encoder` -- The names `bson_buffer_serializer`, `cbor_buffer_serializer`, `msgpack_buffer_serializer`, - and `ubjson_buffer_serializer` have been deprecated and renamed to `bson_bytes_encoder`, - `cbor_bytes_encoder`, `msgpack_bytes_encoder`, and `ubjson_bytes_encoder` -- The names `bson_buffer_reader`, `cbor_buffer_reader`, `msgpack_buffer_reader`, - and `ubjson_buffer_reader` have been deprecated and renamed to `bson_bytes_reader`, - `cbor_bytes_reader`, `msgpack_bytes_reader`, and `ubjson_bytes_reader` - -Enhancements - -- Cleanup of `encode_json` and `decode_json` functions and increased test coverage -- Rewrote cbor_reader to avoid recursive function call -- CBOR reader supports [stringref extension to CBOR](http://cbor.schmorp.de/stringref) -- New `cbor_options` has `packed_strings` option - -v0.120.0 --------- - -Bug fix: - -- Fixed issue with `j.as()` - -Non-breaking changes - -- The name `is_json_type_traits_impl` has been deprecated and renamed to `is_json_type_traits_declared` -- The name `serializing_context` has been deprecated and renamed to `ser_context` -- The name `serialization_error` has been deprecated and renamed to `ser_error` - -Enhancements - -- json `as_byte_string` attempts to decode string values if `semantic_tag_type` is `base64`, `base64url`, or `base16`. - -- New macro `JSONCONS_TYPE_TRAITS_DECL` that can be used to generate the `json_type_traits` boilerplate -for your own types. - -- New `basic_json` member function `get_allocator` - -v0.119.1 --------- - -Bug fix: - -Fixed issue wjson dump() not formatting booleans correctly #174 - -v0.119.0 --------- - -Name change: - -- The name `json_staj_reader` has been deprecated and renamed to `json_pull_reader` - -Bug fix: - -- Fixed a bug in json function `empty()` when type is `byte_string`. -- Fixed a bug with preserving semantic_tag_type when copying json values of long string type. - -Changes: - -- Removed deprecated feature `cbor_view` - -- CBOR decimal fraction and bigfloat string formatting now consistent with double string formatting - -Enhancements: - -- json `to_string()` and `to_double()` now work with CBOR bigfloat - -- JSONPath operators in filter expressions now work with `big_integer`, `big_decimal`, and `big_float` - tagged json values - -- json `is_number()` function now returns `true` if string value is tagged with `big_integer` or - `big_decimal`, or if array value is tagged with `big_float`. - -- json `as_string()` function now converts arrays tagged with `big_float` to decimal strings - -- json `as_double()` function now converts arrays tagged with `big_float` to double values - -v0.118.0 --------- - -New features - -- New csv option `lossless_number`. If set to `true`, parse numbers with exponent and fractional parts as strings with - semantic tagging `semantic_tag_type::big_decimal` (instead of double.) Defaults to `false`. - -- A class `jsonpointer::address` has been introduced to make it simpler to construct JSON Pointer addresses - -Name change - -- The `json_options` name `dec_to_str` has been deprecated and renamed to `lossless_number`. - -v0.117.0 --------- - -Deprecated features: - -- cbor_view has been deprecated. Rationale: The complexity of supporting and documenting this component - exceeded its benefits. - -New features - -- New `json_options` option `dec_to_str`. If set to `true`, parse decimal numbers as strings with - semantic tagging `semantic_tag_type::big_decimal` instead of double. Defaults to `false`. - -- The `ojson` (order preserving) implementation now has an index to support binary search - for retrieval. - -- Added `std::string_view` detection - -- jsoncons-CBOR semantic tagging supported for CBOR tags 32 (uri) - -Name changes (non-breaking) - -- The json options name `bignum_chars_format` has been deprecated and replaced with `big_integer_chars_format`. -- `big_integer_chars_format::integer` (`bignum_chars_format::integer`) has been deprecated and replaced with - `big_integer_chars_format::number` -- The `json_options function` `bignum_format` has been deprecated and replaced with `big_integer_format` - -Changes to floating-point printing - -- If the platform supports the IEEE 754 standard, jsoncons now uses the Grisu3 algorithm for printing floating-point numbers, - falling back to a safe method using C library functions for the estimated 0.5% of floating-point numbers that might be rejected by Grisu3. - The Grisu3 implementation follows Florian Loitsch's [grisu3_59_56](https://www.cs.tufts.edu/~nr/cs257/archive/florian-loitsch/printf.pdf) - implementation. If the platform does not support the IEEE 754 standard, the fall back method is used. - -- In previous versions, jsoncons preserved information about the format, precision, and decimal places - of the floating-point numbers that it read, and used that information when printing them. With the - current strategy, that information is no longer needed. Consequently, the `floating_point_options` - parameter in the `do_double_value` and `double_value` functions of the SAX-style interface have - been removed. - -- The `json` functions `precision()` and `decimal_places()` have been deprecated and return 0 - (as this information is no longer preserved.) - -- The constructor `json(double val, uint8_t precision)` has been deprecated. - -- Note that it is still possible to set precision as a json option when serializing. - -v0.116.0 --------- - -New features: - -- New jsonpath functions `keys` and `tokenize`. - -- jsoncons-CBOR data item mappings supported for CBOR tags 33 (string base64url) and 34 (string base64) - -v0.115.0 --------- - -New features: - -- bson UTC datetime associated with jsoncons `semantic_tag_type::timestamp` - -- New traits class `is_json_type_traits_impl` that addresses issues - [#133](https://github.com/danielaparker/jsoncons/issues/133) and - [#115](https://github.com/danielaparker/jsoncons/issues/115) (duplicates) - -- Following a proposal from soberich, jsonpath functions on JSONPath expressions - are no longer restricted to filter expressions. - -- New jsonpath functions `sum`, `count`, `avg`, and `prod` - -- Added `semantic_tag_type::base16`, `semantic_tag_type::base64`, `semantic_tag_type::base64url` - -Non-breaking changes: - -- The json constructor that takes a `byte_string` and a `byte_string_chars_format` - has been deprecated, use a `semantic_tag_type` to supply an encoding hint for a byte string, if any. - -- The content handler `byte_string_value` function that takes a `byte_string` and a `byte_string_chars_format` - has been deprecated, use a `semantic_tag_type` to supply an encoding hint for a byte string, if any. - -Changes: - -- The `byte_string_chars_format` parameter in the content handler `do_byte_string_value` function - has been removed, the `semantic_tag_type` parameter is now used to supply an encoding hint for a byte string, if any. - -v0.114.0 --------- - -Bug fixes: - -- On Windows platforms, fixed issue with macro expansion of max when - including windows.h (also in 0.113.1) - -- Fixed compile issue with `j = json::make_array()` (also in 0.113.2) - -Breaking changes to jsoncons semantic tag type names: - -- semantic_tag_type::bignum to semantic_tag_type::big_integer -- semantic_tag_type::decimal_fraction to semantic_tag_type::big_decimal -- semantic_tag_type::epoch_time to semantic_tag_type::timestamp - -Non-breaking name changes: - -The following names have been deprecated and renamed (old names still work) - -- `bignum_value` to `big_integer_value` in `json_content_handler` -- `decimal_value` to `big_decimal_value` in `json_content_handler` -- `epoch_time_value` to `timestamp_value` in `json_content_handler` - -- `cbor_bytes_serializer` to `cbor_buffer_serializer` -- `msgpack_bytes_serializer` to `msgpack_buffer_serializer` - -- `json_serializing_options` to `json_options` -- `csv_serializing_options` to `csv_options` - -- `parse_error` to `serialization_error` - -The rationale for renaming `parse_error` to `serialization_error` -is that we need to use error category codes for serializer -errors as well as parser errors, so we need a more general name -for the exception type. - -Message Pack enhancements - -- New `msgpack_serializer` that supports Message Pack bin formats - -- New `msgpack_parser` that supports Message Pack bin formats - -- `encode_msgpack` and `decode_msgpack` have been - rewritten using `msgpack_serializer` and `msgpack_parser`, - and also now support bin formats. - -New features: - -- decode from and encode to the [Universal Binary JSON Specification (ubjson)](http://bsonspec.org/) data format -- decode from and encode to the [Binary JSON (bson)](http://bsonspec.org/) data format - -- The cbor, msgpack and ubjson streaming serializers now validate that the expected number of - items have been supplied in an object or array of pre-determined length. - -version 0.113.0 ---------------- - -Bug fix - -- Fixed issue with indefinite length byte strings, text strings, arrays, - and maps nested inside other CBOR items (wasn't advancing the - input pointer past the "break" indicator.) - -Changes - -- __FILE__ and __LINE__ macros removed from JSONCONS_ASSERT - if not defined _DEBUG (contributed by zhskyy.) - -- semantic_tag_type name `decimal` changed to `decimal_fraction` - -New CBOR feature - -- CBOR semantic tagging of expected conversion of byte strings - to base64, base64url and base16 are preserved and respected in JSON - serialization (unless overridden in `json_serializing_options`.) - -- CBOR semantic tagging of bigfloat preserved with `semantic_tag_type::bigfloat` - -- CBOR non text string keys converted to strings when decoding - to json values - -Changes to `json_serializing_options` - -New options - -- spaces_around_colon (defaults to `space_after`) -- spaces_around_comma (defaults to `space_after`) -- pad_inside_object_braces (defaults to `false`) -- pad_inside_array_brackets (defaults to `false`) -- line_length_limit (defaults to '120`) -- new_line_chars (for json serialization, defaults to `\n`) - -`nan_replacement`, `pos_inf_replacement`, and `neg_inf_replacement` are deprecated (still work) -These have been replaced by - -- nan_to_num/nan_to_str -- inf_to_num/inf_to_str -- neginf_to_num/neginf_to_str (default is `-` followed by inf_to_num/inf_to_str) - -`nan_to_str`, `inf_to_str` and `neginf_to_str` are also used to substitute back to `nan`, `inf` and `neginf` in the parser. - -- Long since deprecated options `array_array_block_option`, - `array_object_block_option`, `object_object_block_option` and - `object_array_block_option` have been removed. - -- The names `object_array_split_lines`, `object_object_split_lines`, - `array_array_split_lines` and `array_object_split_lines` have - been deprecated (still work) and renamed to `object_array_line_splits`, - `object_object_line_splits`, `array_array_line_splits` and `array_object_line_splits`. - Rationale: consistency with `line_split_kind` name. - -Changes to json_serializer - -- Previously the constructor of `json_serializer` took an optional argument to - indicate whether "indenting" was on. `json_serializer` now always produces - indented output, so this argument has been removed. - -- A new class `json_compressed_serializer` produces compressed json without - indenting. - - The jsoncons functions that perform serialization including `json::dump`, - `pretty_print` and the output stream operator are unaffected. - -v0.112.0 --------- - -Changes to `json_content_handler` - -- The function `byte_string_value` no longer supports passing a byte string as - - handler.byte_string_value({'b','a','r'}); - -(shown in some of the examples.) Instead use - - handler.byte_string_value(byte_string({'b','a','r'})); - -(or a pointer to utf8_t data and a size.) - -- The function `bignum_value` no longer supports passing a CBOR signum and - byte string, `bignum_value` now accepts only a string view. If you - have a CBOR signum and byte string, you can use the bignum class to - convert it into a string. - -Name changes (non breaking) - -- The name `json_stream_reader` has been deprecated and replaced with `json_staj_reader`. -- The name `stream_event_type` has been deprecated and replaced with `staj_event_type` -- The names `basic_stream_event` (`stream_event`) have been deprecated and replaced with `basic_staj_event` (`staj_event`) -- The names `basic_stream_filter` (`stream_filter`) have been deprecated and replaced with `basic_staj_filter` (`staj_filter`) -(staj stands for "streaming API for JSON, analagous to StAX in XML) - -- The `json_parser` function `end_parse` has been deprecated and replaced with `finish_parse`. - -Enhancements - -- json double values convert to CBOR float if double to float - round trips. - -- csv_parser `ignore_empty_values` option now applies to - `m_columns` style json output. - -- json_reader and json_staj_reader can be initialized with strings - in addition to streams. - -Extension of semantic tags to other values - - - The `json_content_handler` functions `do_null_value`, `do_bool_value`, - `do_begin_array` and `do_begin_object` have been given the - semantic_tag_type parameter. - - - New tag type `semantic_tag_type::undefined` has been added - - - The `cbor_parser` encodes a CBOR undefined tag to a json null - value with a `semantic_tag_type::undefined` tag, and the - `cbor_serializer` maps that combination back to a CBOR undefined tag. - -Removed: - -- Long since deprecated `value()` functions have been removed from `json_content_handler` - -v0.111.1 --------- - -Enhancements - -- Improved efficiency of json_decoder - -- Improved efficiency of json_proxy - -- Conversion of CBOR decimal fraction to string uses exponential - notation if exponent is positive or if the exponent plus the - number of digits in the mantissa is negative. - -Bug fix - -- Fixed issue with conversion of CBOR decimal fraction to string - when mantissa is negative - -v0.111.0 --------- - -Bug and warning fixes: - -- A case where the json parser performed validation on a string - before all bytes of the string had been read, and failed if - missing part of a multi-byte byte sequence, is fixed. - -- An issue with reading a bignum with the pull parser -`json_stream_reader` (in the case that an integer value -overflows) has been fixed. - -- GCC and clang warnings about switch fall through have - been fixed - -Non-breaking changes: - -- The functions `json::has_key` and `cbor::has_key` have been - deprecated (but still work) and renamed to `json::contains` - and `cbor::contains`. Rationale: consistency with C++ 20 - associative map `contains` function. - -- The json function `as_integer()` is now a template function, - -```c++ -template -T as_integer(); -``` - -where T can be any integral type, signed or unsigned. The -default parameter is for backwards compatability, but is -a depreated feature, and may be removed in a future version. -Prefer j.as(). - -- The json functions `is_integer()` and `is_uinteger()` -have been deprecated and renamed to `is_int64()`, `is_uint64()`. -Prefer j.is() and j.is(). - -- The json function `as_uinteger()` has been deprecated. -Prefer j.as(). - -and `as_uinteger()` have been deprecated and renamed to -`is_int64()`, `is_uint64()`, `as_int64()` and `as_uint64()`. - -Change to pull parser API: - -- The `stream_filter` function `accept` has been changed to - take a `const stream_event&` and a `const serializing_context&`. - -- `stream_event_type::bignum_value` has been removed. `stream_event` - now exposes information about optional semantic tagging through - the `semantic_tag()` function. - -Enhancements: - -- `j.as()` has been enhanced to return a bignum value -if j is an integer, floating point value, or any string that -contains an optional minus sign character followed by a sequence -of digits. - -- `j.as()` has been enhanced to support extended integer -types that have `std::numeric_limits` specializations. In particular, -it supports GCC `__int128` and `unsigned __int128` when code is -compiled with `std=gnu++NN`, allowing a `bignum` to be returned as -an `__int128` or `unsigned __int128`. (when code is compiled with -`-std=c++NN`, `__int128` and `unsigned __int128` do not have -`std::numeric_limits` specializations.) - -New feature: - -This release accomodate the additional semantics for the -CBOR data items date-time (a string), and epoch time (a positive or -negative integer or floating point value), and decimal fraction -(converted in the jsoncons data model to a string). - -But first, some of the virtual functions in `json_content_handler` -have to be modified to preserve these semantics. Consequently, -the function signatures - - bool do_int64_value(int64_t, const serializing_context&) - - bool do_uint64_value(uint64_t, const serializing_context&) - - bool do_double_value(double, const floating_point_options&, const serializing_context&) - - bool do_string_value(const string_view_type&, const serializing_context&) - - bool do_byte_string_value(const uint8_t*, size_t, const serializing_context&) - -have been given an additonal parameter, a `semantic_tag_type`, - - bool do_int64_value(int64_t, semantic_tag_type, const serializing_context&) - - bool do_uint64_value(uint64_t, semantic_tag_type, const serializing_context&) - - bool do_double_value(double, const floating_point_options&, semantic_tag_type, const serializing_context&) - - bool do_string_value(const string_view_type&, semantic_tag_type, const serializing_context&) - - bool do_byte_string_value(const uint8_t*, size_t, semantic_tag_type, const serializing_context&) - -For consistency, the virtual function - - bool do_bignum_value(const string_view_type&, const serializing_context&) - -has been removed, and in its place `do_string_value` will be called -with semantic_tag_type::bignum_type. - -For users who have written classes that implement all or part of -`json_content_handler`, including extensions to `json_filter`, -these are breaking changes. But otherwise users should be unaffected. - -v0.110.2 --------- - -Continuous integration: - -- jsoncons is now cross compiled for ARMv8-A architecture on Travis using - clang and executed using the emulator qemu. - -- UndefinedBehaviorSanitizer (UBSan) is enabled for selected gcc and clang builds. - -Maintenance: - -- Removed compiler warnings - -v0.110.1 --------- - -Bug fixes contributed by Cebtenzzre - -- Fixed a case where `as_double()`, `as_integer()` etc on a `basic_json` - value led to an infinite recursion when the value was a bignum - -- Fixed undefined behavior in bignum class - -v0.110.0 --------- - -### New features - -- A JSON pull parser, `json_stream_reader`, has been added. This - required a change to the `json_content_handler` function signatures, - the return values has been changed to bool, to indicate whether - parsing is to continue. (An earlier version on master was - called `json_event_reader`.) - -- `json_parser` has new member function `stopped()`. - -- The `json::is` function now supports `is()`, - and if your compiler has `std::string_view`, `is()` - as well. It returns `true` if the json value is a string, otherwise - `false`. - -- The `json::as` function now supports `as()`, - and if your compiler has `std::string_view`, `as()` - as well. - -### Changes to `json_content_handler` and related streaming classes - -#### Non-breaking changes - -These changes apply to users that call the public functions defined by -`json_content_handler`, e.g. begin_object, end_object, etc., but are -non-breaking because the old function signatures, while deprecated, -have been preserved. Going forward, however, users should remove -calls to `begin_document`, replace `end_document` with `flush`, -and replace `integer_value` and `end_integer_value` with `int64_value` -and `uint64_value`. - -- The public functions defined by `json_content_handler` have been changed - to return a bool value, to indicate whether parsing is to continue. - -- The function names `integer_value` and `uinteger_value` have been - changed to `int64_value` and `uint64_value`. - -- The function names `begin_document` and `end_document` have been - deprecated. The deprecated `begin_document` does nothing, and the - deprecated `end_document` calls `do_flush`. - -- The function `flush` has been added, which calls `do_flush`. - -- The `json` member function `dump_fragment` has been deprecated, as with - the dropping of `begin_document` and `end_document`, it is now - equivalent to `dump`. - -- The function `encode_fragment` has been deprecated, as with - the dropping of `begin_document` and `end_document`, it is now - equivalent to `encode_json`. - -- The `json_filter` member function `downstream_handler` has been - renamed to `destination_handler`. - -#### Breaking changes - -These changes will affect users who have written classes that implement -all or part of `json_content_handler`, including extensions to `json_filter`. - -- The virtual functions defined for `json_content_handler`, `do_begin_object`, - `do_end_object`, etc. have been changed to return a bool value, to indicate - whether serializing or deserializing is to continue. - -- The virtual functions `do_begin_document` and `do_end_document` have been removed. - A virtual function `do_flush` has been added to allow producers of json events to - flush whatever they've buffered. - -- The function names `do_integer_value` and `do_uinteger_value` have been changed to - `do_int64_value` and `do_uint64_value`. - -- The signature of `do_bignum_value` has been changed to -``` - bool do_bignum_value(const string_view_type& value, - const serializing_context& context) -``` - -v0.109.0 --------- - -Enhancements - -- Added byte string formatting option `byte_string_chars_format::base16` - -Changes - -- Scons dropped as a build system for tests and examples, use CMake - -- Tests no longer depend on boost, boost test framework replaced by Catch2. - boost code in tests moved to `examples_boost` directory. - -- Previously, if `json_parser` encountered an unopened object or array, e.g. "1]", - this would cause a JSONCONS_ASSERT failure, resulting in an `std::runtime_error`. This has been - changed to cause a `json_parse_errc::unexpected_right_brace` or `json_parse_errc::unexpected_right_bracket` error code. - -Warning fixes - -- Eliminated vs2017 level 3 and level 4 warnings - -v0.108.0 --------- - -Enhancements - -- `bignum_chars_format::base64` is supported - -- The incremental parser `json_parser` has been documented - -Changes (non-breaking) - -- Previously, jsonpointer::get returned values (copies) - Now, jsonpointer::get returns references if applied to `basic_json`, and values if applied to `cbor_view` - -- `bignum_chars_format::string` has been deprecated (still works) and replaced with `bignum_chars_format::base10` - -- `json_parser_errc`, `cbor_parser_errc`, and `csv_parser_errc` have been deprecated (still work) and renamed to - `json_parse_errc`, `cbor_parse_errc`, and `csv_parse_errc` - -v0.107.2 --------- - -Bug fixes: - -- Fixed issue with `UINT_MAX` not declared in `bignum.hpp` - -v0.107.1 --------- - -Bug fixes: - -- Fixed issue with cbor_view iterators over indefinite length arrays and maps - -Enhancements: - -- csv_serializer recognizes byte strings and bignums. - -v0.107.0 --------- - -Enhancements - -- Support for CBOR bignums -- Added json serializing options for formatting CBOR bignums as integer, string, or base64url encoded byte string -- Added json serializing options for formatting CBOR bytes strings as base64 or base64url -- Enhanced interface for `cbor_view` including `dump`, `is`, and `as` functions - -Changes - -- If the json parser encounters an integer overflow, the value is now handled as a bignum rather than a double value. - -- The `json_content_handler` names `begin_json` and `end_json` have been - deprecated and replaced with `begin_document` and `end_document`, and the - names `do_begin_json` and `do_end_json` have been removed and replaced with - `do_begin_document`, and `do_end_document`. - Rationale: meaningfullness across JSON and other data formats including - CBOR. - -Bug fixes: - -- Fixed bug in base64url encoding of CBOR byte strings -- Fixed bug in parsing indefinite length CBOR arrays and maps - -v0.106.0 --------- - -Changes - -- If a fractional number is read in in fixed format, serialization now preserves - that fixed format, e.g. if 0.000071 is read in, serialization gives 0.000071 - and not 7.1e-05. In previous versions, the floating point format, whether - fixed or scientific, was determined by the behavior of snprintf using the g - conversion specifier. - -Bug fix: - -- Fixed issue with parsing cbor indefinite length arrays and maps - -Warning fix: - -- Use memcpy in place of reinterpret_cast in binary data format utility - `from_big_endian` - -Compiler fix: - -- Fixed issues with g++ 4.8 - -v0.105.0 --------- - -Enhancements - -- The CSV extension now supports multi-valued fields separated by subfield delimiters - -- New functions `decode_json` and `encode_json` convert JSON - formatted strings to C++ objects and back. These functions attempt to - perform the conversion by streaming using `json_convert_traits`, and if - streaming is not supported, fall back to using `json_type_traits`. `decode_json` - and `encode_json` will work for all types that have `json_type_traits` defined. - -- The json::parse functions and the json_parser and json_reader constructors - optionally take a json_serializing_options parameter, which allows replacing - a string that matches nan_replacement(), pos_inf_replacement(), and neg_inf_replacement(). - -Changes to Streaming - -- The `basic_json_input_handler` and `basic_json_output_handler` interfaces have been - combined into one class `basic_json_content_handler`. This greatly simplifies the - implementation of `basic_json_filter`. Also, the name `parsing_context` has been - deprecated and renamed to `serializing_context`, as it now applies to both - serializing and deserializing. - - If you have subclassed `json_filter` or have fed JSON events directlty to a - `json_serializer`, you shouldn't have to make any changes. In the less likely - case that you've implemented the `basic_json_input_handler` or - `basic_json_output_handler` interfaces, you'll need to change that to - `json_content_handler`. - -Other Changes - -- `serialization_traits` and the related `dump` free functions have been deprecated, - as their functionality has been subsumed by `json_convert_traits` and the - `encode_json` functions. - -- The option bool argument to indicate pretty printing in the `json` `dump` functions - and the `json_serializer` class has been deprecated. It is replaced by the enum class - `indenting` with enumerators `indenting::no_indent` and `indenting::indent`. - -- The name `serialization_options` has been deprecated (still works) and renamed to - `json_serializing_options`. Rationale: naming consistency. - -- The `json_reader` `max_nesting_depth` getter and setter functions have been deprecated. - Use the `json_serializing_options` `max_nesting_depth` getter and setter functions instead. - -- The name `csv_parameters` has been deprecated (still works) and renamed to - `csv_serializing_options`. Rationale: naming consistency. - -v0.104.0 --------- - -Changes - -- `decode_csv` by default now attempts to infer null, true, false, integer and floating point values - in the CSV source. In previous versions the default was to read everything as strings, - and other types had to be specified explicitly. If the new default behavior is not desired, the - `csv_parameters` option `infer_types` can be set to `false`. Column types can still be set explicitly - if desired. - -v0.103.0 --------- - -Changes - -- Default `string_view_type` `operator std::basic_string() const` made explicit - to be consistent with `std::string_view` - -- The virtual method `do_double_value` of `json_input_handler` and `json_output_handler` takes a `number_format` parameter - -Performance improvements - -- Faster json dump to string (avoids streams) -- Faster floating point conversions for linux and MacOSX -- Memory allocation decoding larger string values reduced by half -- Optimization to json_parser parse_string -- Improvements to json_decoder - -v0.102.1 --------- - -Bug fix: - -Fixed an off-by-one error that could lead to an out of bounds read. Reported by mickcollyer (issue #145) - -v0.102.0 --------- - -Bug fixes: - -Fixed issue with how jsonpath filters are applied to arrays in the presence of recursion, resulting in -duplicate results. - -Changes: - -The signatures of `jsonpointer::get`, `jsonpointer::insert`, `jsonpointer::insert_or_assign`, -`jsonpointer::remove` and `jsonpointer::replace` have been changed to be consistent -with other functions in the jsoncons library. Each of these functions now has two overloads, -one that takes an `std::error_code` parameter and uses it to report errors, and one that -throws a `jsonpointer_error` exception to report errors. - -The function `jsonpatch::patch` has been replaced by `jsonpatch::apply_patch`, which takes -a json document, a patch, and a `std::error_code&` to report errors. The function -`jsonpatch::diff` has been renamed to `jsonpatch::from_diff` - -The old signatures for `encode_cbor` and `encode_msgpack` that returned a `std::vector` -have been deprecated and replaced by new signatures that have void return values and have -an output parameter 'std::vector&'. The rationale for this change is consistency -with other functions in the jsoncons library. - -v0.101.0 --------- - -Fixes: - -- Fixes to `string_view` code when `JSONCONS_HAS_STRING_VIEW` is defined in `jsoncons_config.hpp` - -Changes: - -- `as_double` throws if `json` value is null (previously returned NaN) - -Enhancements: - -- Added convenience functions `decode_csv` and `encode_csv` -- Support custom allocaor (currently stateless only) in `json_decoder`, `json_reader`, - `csv_reader`, `csv_parameters` - -v0.100.2 -------- - -Resolved warnings on GCC Issue #127 - -v0.100.1 -------- - -Fix for platform issue with vs2017: - -- Renamed label `minus` to `minus_sign` in `json_parser.hpp` - -Enhancements: - -- New classes `byte_string` and `byte_string_view`, to augment support for cbor byte strings in `json` values - -v0.100.0 -------- - -Changes: - -- `template json_traits` replaced with `sorted_policy` -- `template o_json_traits` replaced with `preserve_order_policy` - -- The return type for the json::get_with_default function overload for `const char*` has been - changed from `const char*` to `json::string_view_type`, which is assignable to `std::string`. - -- New functions `byte_string_value` and `do_byte_string_value` have been added to - `basic_json_input_handler` and `basic_json_output_handler` - -- json::is() and json::as() specializations (supported but never - documented) have been deprecated - -- In android specific `string_to_double`, `strtod_l` changed to `strtold_l` - -Enhancements: - -- The `json` class and the `decode_cbor` and `encode_cbor` functions now support byte strings - A `json` byte string value will, when serialized to JSON, be converted to a base64url string. - -- `version.hpp` added to `include` directory - -v0.99.9.2 --------- - -Bug fixes: - -- Fixed issue with jsonpatch::diff (fix contributed by Alexander (rog13)) - -Enhancements: - -- New class `cbor_view` for accessing packed `cbor` values. A `cbor_view` satisfies the requirements for `jsonpointer::get`. - -Changes (non breaking) ----------------------- - -- `jsonpointer::erase` renamed to `jsonpointer::remove`, old name deprecated - -v0.99.9.1 --------- - -New features ------------- - -- JSON Pointer implementation - -- JSON Patch implementation, includes patch and diff - -- `json::insert` function for array that inserts values from range [first, last) before pos. - -Bug fixes - -- Fixed issue with serialization of json array of objects to csv file - -Changes (non breaking) ----------------------- - -- The member function name `json::dump_body` has been deprecated and replaced with `json::dump_fragment`. -- The non member function name `dump_body` has been deprecated and replaced with `dump_fragment`. - -- The class name `rename_name_filter` has been deprecated and replaced with `rename_object_member_filter`. - -- In the documentation and examples, the existing function `json::insert_or_assign` - is now used instead of the still-supported `json::set`. The reason is that - `insert_or_assign` follows the naming convention of the C++ standard library. - -Changes -------- - -- The recently introduced class `json_stream_traits` has been renamed to `serialization_traits` - -- Removed template parameter `CharT` from class `basic_parsing_context` and renamed it to `parsing_context` - -- Removed template parameter `CharT` from class `basic_parse_error_handler` and renamed it to `parse_error_handler` - -v0.99.8.2 --------- - -New features - -- Added `json` functions `push_back` and `insert` for appending values - to the end of a `json` array and inserting values at a specifed position - -Rationale: While these functions provide the same functionality as the existing -`json::add` function, they have the advantage of following the naming conventions -of the C++ library, and have been given prominence in the examples and documentation -(`add` is still supported.) - -v0.99.8.1 --------- - -New features - -- cbor extension supports encoding to and decoding from the cbor binary serialization format. - -- `json_type_traits` supports `std::valarray` - -Documentation - -- Documentation is now in the repository itself. Please see the documentation - link in the README.md file - -Changed - -- Removed `CharT` template parameter from `json_stream_traits` - -v0.99.8 ------- - -Changes - -- Visual Studio 2013 is no longer supported (jsonpath uses string initilizer lists) - -- `json_input_handler` overloaded functions value(value,context)` have been deprecated. - Instead use `string_value(value,context)`, `integer_value(value,context)`, - `uinteger_value(value,context)`, `double_value(value,precision,context)`, - `bool_value(value,context)` and `null_value(context)` - -- `json_output_handler` overloaded functions value(value)` have been deprecated. - Instead use `string_value(value)`, `integer_value(value)`, `uinteger_value(value)`, - `double_value(value,precision=0)`, `bool_value(value)` and `null_value(context)` - -- For consistency, the names `jsoncons_ext/msgpack/message_pack.hpp`, - `encode_message_pack` and `decode_message_pack` have been deprecated and - replaced with `jsoncons_ext/msgpack/msgpack.hpp`, `encode_msgpack` and `decode_msg_pack` - -Bug fixes - -- Fixed operator== throws when comparing a string against an empty object - -- Fixed jsonpath issue with array 'length' (a.length worked but not a['length']) - -- `msgpack` extension uses intrinsics for determing whether to swap bytes - -New features - -- Stream supported C++ values directly to JSON output, governed by `json_stream_traits` - -- json::is() and json::as() accept template packs, which they forward to the - `json_type_traits` `is` and `as` functions. This allows user defined `json_type_traits` - implementations to resolve, for instance, a name into a C++ object looked up from a - registry. See [Type Extensibility](https://github.com/danielaparker/jsoncons), Example 2. - -- jsonpath `json_query` now supports returning normalized paths (with - optional `return_type::path` parameter) - -- New jsonpath `max` and `min` aggregate functions over numeric values - -- New `json::merge` function that inserts another json object's key-value pairs - into a json object, if they don't already exist. - -- New `json::merge_or_update` function that inserts another json object's key-value - pairs into a json object, or assigns them if they already exist. - -v0.99.7.3 --------- - -- `json_type_traits` supports `std::pair` (convert to/from json array of size 2) - -- `parse_stream` renamed to `parse` (backwards compatible) - -- `kvp_type` renamed to `key_value_pair_type` (backwards compatible) - -- The `_json` and `_ojson` literal operators have been moved to the namespace `jsoncons::literals`. - Access to these literals now requires -```c++ - using namespace jsoncons::literals; -``` -Rationale: avoid name clashes with other `json` libraries - -- The name `owjson` has been deprecated (still works) and changed to `wojson`. Rationale: naming consistency - -- Added json array functions `emplace_back` and `emplace`, and json object functions `try_emplace` - and `insert_or_assign`, which are analagous to the standard library vector and map functions. - -v0.99.7.2 --------- - -Bug fix - -- A bug was introduced in v0.99.7 causing the values of existing object members to not be changed wiht set or assignment operations. This has been fixed. - -Change - -- jsoncons_ext/binary changed to jsoncons_ext/msgpack -- namespace jsoncons::binary changed to jsoncons::msgpack - -v0.99.7.1 --------- - -- Workarounds in unicode_traits and jsonpath to maintain support for vs2013 -- Added `mapping_type::n_rows`, `mapping_type::n_objects`, and `mapping_type::m_columns` options for csv to json - -v0.99.7 ------- - -Bug fixes - -- Issues with precedence in JsonPath filter evaluations have been fixed -- An issue with (a - expression) in JsonPath filter evaluations has been fixed - -New feature - -- The new binary extension supports encoding to and decoding from the MessagePack binary serialization format. -- An extension to JsonPath to allow filter expressions over a single object. -- Added support for `*` and `/` operators to jsonpath filter -- literal operators _json and _ojson have been introduced - -Non-breaking changes - -- The `json` `write` functions have been renamed to `dump`. The old names have been deprecated but still work. -- Support for stateful allocators -- json function object_range() now returns a pair of RandomAccessIterator (previously BidirectionalIterator) -- json operator [size_t i] applied to a json object now returns the ith object (previously threw) - -Breaking change (if you've implemented your own input and output handlers) - -In basic_json_input_handler, the virtual functions -```c++ -virtual void do_name(const CharT* value, size_t length, - const basic_parsing_context& context) - -virtual void do_string_value(const CharT* value, size_t length, - const basic_parsing_context& context) -``` -have been changed to -```c++ -virtual void do_name(string_view_type val, - const basic_parsing_context& context) - -virtual void do_string_value(string_view_type val, - const basic_parsing_context& context) -``` - -In basic_json_output_handler, the virtual functions -```c++ -virtual void do_name(const CharT* value, size_t length) - -virtual void do_string_value(const CharT* value, size_t length) -``` -have been changed to -```c++ -virtual void do_name(string_view_type val) - -virtual void do_string_value(string_view_type val) -``` - -Removed features: - -- The jsonx extension has been removed - -v0.99.5 ------- - -- Validations added to utf8 and utf16 string parsing to pass all [JSONTestSuite](https://github.com/nst/JSONTestSuite) tests -- The name `json_encoder` introduced in v0.99.4 has been changed to `json_decoder`. Rationale: consistencty with common usage (encoding and serialization, decoding and deserialization) - -v0.99.4a -------- - -Fixes Issue #101, In json.hpp, line 3376 change "char__type" to "char_type" -Fixes Issue #102, include cstring and json_error_category.hpp in json.hpp - -v0.99.4 ------- - -Changes - -- The deprecated class `json::any` has been removed. -- The jsoncons `boost` extension has been removed. That extension contained a sample `json_type_traits` specialization for `boost::gregorian::date`, which may still be found in the "Type Extensibility" tutorial. -- The member `json_type_traits` member function `assign` has been removed and replaced by `to_json`. if you have implemented your own type specializations, you will also have to change your `assign` function to `to_json`. -- `json_type_traits` specializations no longer require the `is_assignable` data member - -Non-breaking name changes - -- The names `json_deserializer`,`ojson_deserializer`,`wjson_deserializer`,`owjson_deserializer` have been deprecated (they still work) and replaced by `json_encoder`, `json_encoder`, `json_encoder` and `json_encoder`. -- The name `output_format` has been deprecated (still works) and renamed to `serialization_options`. -- The name `wojson` has been deprecated (still works) and renamed to `owjson`. -- The `json_filter` member function `input_handler` has been deprecated (still works) and renamed to `downstream_handler`. -- The name `elements` has been deprecated (still works) and renamed to `owjson`. -- The `json` member function `members()` has been deprecated (still works) and renamed to `object_range()`. -- The `json` member function `elements()` has been deprecated (still works) and renamed to `array_range()`. -- The `json` member_type function `name()` has been deprecated (still works) and renamed to `key()`. Rationale: consistency with more general underlying storage classes. - -New features - -- `json_filter` instances can be passed to functions that take a `json_output_handler` argument (previously only a `json_input_handler` argument) -- New `jsonpath` function `json_replace` that searches for all values that match a JsonPath expression and replaces them with a specified value. -- `json` class has new method `has_key()`, which returns `true` if a `json` value is an object and has a member with that key -- New filter class `rename_name` allows search and replace of JSON object names - -v0.99.3a -------- - -Changes - -The `json` initializer-list constructor has been removed, it gives inconsistent results when an initializer has zero elements, or one element of the type being initialized (`json`). Please replace - -`json j = {1,2,3}` with `json j = json::array{1,2,3}`, and - -`json j = {{1,2,3},{4,5,6}}` with `json j = json::array{json::array{1,2,3},json::array{4,5,6}}` - -- Initializer-list constructors are now supported in `json::object` as well as `json::array`, e.g. -```c++ -json j = json::object{{"first",1},{"second",json::array{1,2,3}}}; -``` - -- json::any has been deprecated and will be removed in the future - -- The json method `to_stream` has been renamed to `write`, the old name is still supported. - -- `output_format` `object_array_block_option`, `array_array_block_option` functions have been deprecated and replaced by - `object_array_split_lines`, `array_array_split_lines` functions. - -Enhancements - -- A new method `get_with_default`, with return type that of the default, has been added to `json` - -- A new template parameter, `JsonTraits`, has been added to the `basic_json` class template. - -- New instantiations of `basic_json`, `ojson` and `wojson`, have been added for users who wish to preserve the alphabetical sort of parsed json text and to insert new members in arbitrary name order. - -- Added support for `json` `is`, `as`, constructor, and assignment operator for any sequence container (`std::array`, `std::vector`, `std::deque`, `std::forward_list`, `std::list`) whose values are assignable to JSON types (e.g., ints, doubles, bools, strings, STL containers of same) and for associative containers (`std::set`, `std::multiset`, `std::unordered_set`, `std::unordered_multiset`.) - -- Added static method `null()` to `json` class to return null value - -- A new extension jsonx that supports serializing JSON values to [JSONx](http://www.ibm.com/support/knowledgecenter/SS9H2Y_7.5.0/com.ibm.dp.doc/json_jsonx.html) (XML) - -- json parser will skip `bom` in input if present - -Fixes: - -- Fixes to the `jsonpath` extension, including the union operator and applying index operations to string values - -- Fixes to remove warnings and issues reported by VS2015 with 4-th warnings level, PVS-Studio static analyzer tool, and UBSAN. - -v0.99.2 ------- - -- Included workaround for a C++11 issue in GCC 4.8, contributed by Alex Merry -- Fixed operator== so that json() == json(json::object()) -- Fixed issue with `json` assignment to initializer list -- Fixed issue with assignment to empty json object with multiple keys, e.g. - - json val; - val["key1"]["key2"] = 1; - -v0.99.1 ------- - -- Fix to json_filter class -- Fix to readme_examples - -v0.99 ----- - -- Fixes to deprecated json parse functions (deprecated, but still supposed to work) -- The Visual C++ specific implementation for reading floating point numbers should have freed a `_locale_t` object, fixed -- Added `json_type_traits` specialization to support assignment from non-const strings -- When parsing fractional numbers in text, floating point number precision is retained, and made available to serialization to preserve round-trip. The default output precision has been changed from 15 to 16. -- Added json `std::initializer_list` constructor for constructing arrays -- The deprecated json member constants null, an_object, and an_array have been removed -- Microsoft VC++ versions earlier than 2013 are no longer supported - -v0.98.4 ------- - -- Fixes issues with compilation with clang - -v0.98.3 ------- - -New features - -- Supports [Stefan Goessner's JsonPath](http://goessner.net/articles/JsonPath/). -- json member function `find` added -- json member function `count` added -- json array range accessor `elements()` added, which supports range-based for loops over json arrays, and replaces `begin_elements` and `end_elements` -- json object range accessor `members()` added, which supports range-based for loops over json objects, and replaces `begin_members` and `end_members` -- New version of json `add` member function that takes a parameter `array_iterator` -- json member function `shrink_to_fit` added - -API Changes - -- The json internal representation of signed and unsigned integers has been changed from `long long` and `unsigned long long` to `int64_t` and `uint64_t`. This should not impact you unless you've implemented your own `json_input_handler` or `json_output_handler`, in which case you'll need to change your `json_input_handler` function signatures - - void do_longlong_value(long long value, const basic_parsing_context& context) override - void do_ulonglong_integer_value(unsigned long long value, const basic_parsing_context& context) override - -to - - void do_integer_value(int64_t value, const basic_parsing_context& context) override - void do_uinteger_value(uint64_t value, const basic_parsing_context& context) override - -and your `json_output_handler` function signatures from - - void do_longlong_value(long long value) override - void do_ulonglong_integer_value(unsigned long long value) override - -to - - void do_integer_value(int64_t value) override - void do_uinteger_value(uint64_t value) override - -- `output_format` drops support for `floatfield` property - -Non-beaking API Changes -- remove_range has been deprecated, use erase(array_iterator first, array_iterator last) instead -- remove has been deprecated, use erase(const std::string& name ) instead -- `json::parse_string` has been renamed to `json::parse`, `parse_string` is deprecated but still works -- `json member function `is_empty` has been renamed to `empty`, `is_empty` is deprecated but still works. Rationale: consistency with C++ containers -- json member functions `begin_elements` and `end_elements` have been deprecated, instead use `elements().begin()` and `elements.end()` -- json member functions `begin_members` and `end_members` have been deprecated, instead use `members().begin()` and `members.end()` -- json member function `has_member` has been deprecated, instead use `count`. Rationale: consistency with C++ containers -- json member function `remove_member` has been deprecated, instead use `remove`. Rationale: only member function left with _element or _member suffix -- json_parse_exception renamed to parse_error, json_parse_exception typedef to parse_error -- json::parse(std::istream& is) renamed to json::parse_stream. json::parse(std::istream is) is deprecated but still works. - -v0.98.2 Release --------------- - -- `json` constructor is now templated, so constructors now accept extended types -- Following [RFC7159](http://www.ietf.org/rfc/rfc7159.txt), `json_parser` now accepts any JSON value, removing the constraint that it be an object or array. -- The member `json_type_traits` member functions `is`, `as`, and `assign` have been changed to static functions. if you have implemented your own type specializations, you will also have to change your `is`, `as` and `assign` functions to be static. -- Removed json deprecated functions `custom_data`, `set_custom_data`, `add_custom_data` -- `json_reader` member function `max_depth` has been renamed to `max_nesting_depth`, the former name is still supported. -- `json` member function `resize_array` has been renamed to `resize`, the former name is still supported. - -jsoncons supports alternative ways for constructing `null`, `object`, and `array` values. - -null: - - json a = jsoncons::null_type(); // Using type constructor - json b = json::null_type(); // Using alias - json c(json::null); // From static data member prototype - -object: - - json a(); // Default is empty object - json b = json::object(); // Using type constructor - json c(json::an_object); // From static data member prototype - -array: - - json a = json::array(); // Using type constructor - json b = json::make_array(); // Using factory method - json c(json::an_array); // From static data member prototype - -Since C++ has possible order issues with static data members, the jsoncons examples and documentation have been changed to consistently use the other ways, and `json::null`, `json::an_object` and `json::an_array` have been, while still usable, deprecated. - -v0.98.1 Release --------------- - -- Enhances parser for CSV files that outputs JSON, see example below. -- Adds `get_result` member function to `json_deserializer`, which returns the json value `v` stored in a `json_deserializer` as `std::move(v)`. The `root()` member function has been deprecated but is still supported. -- Adds `is_valid` member function to `json_deserializer` -- Enhances json::any class, adds type checks when casting back to original value -- Fixes some warning messages - -v0.98 Release --------------- - -Bug fixes: - -- Fixes the noexcept specification (required for Visual Studio 2015 and later.) Fix - contributed by Rupert Steel. -- Fixes bug with proxy operator== when comparing object member values, - such as in val["field"] == json("abc") - -Enhancements: - -- Refines error codes and improves error messages - -- Renames `json_reader` method `read` to `read_next`, reflecting the fact that it supports reading a sequence of JSON texts from a stream. The - former name is deprecated but still works. - -- Adds `json_reader` method `check_done` that throws if there are unconsumed non-whitespace characters after one or more calls to `read_next`. - -- Adds getter and setter `max_depth` methods to allow setting the maximum JSON parse tree depth if desired, by default -it is arbitrarily large (limited by heap memory.) - -- Modifies `json` static methods `parse_string`, `parse_file`, and `parse` behaviour to throw if there are unconsumed non-whitespace characters after reading one JSON text. - -Changes to extensions: - -- Changes the top level namespace for the extensions from `jsoncons_ext` to `jsoncons`, e.g. `jsoncons_ext::csv::csv_reader` becomes `jsoncons::csv::csv_reader` -- Modifies csv_reader and csv_serializer so that the constructors are passed parameters in a `csv_parameters` object rather than a `json` object. -- Adds more options to csv_reader - -v0.97.2 Release --------------- - -- Incorporates test suite files from http://www.json.org/JSON_checker/ into test suite -- The `jsoncons` parser accepts all of the JSON_checker files that its supposed to accept. -- Failures to reject incorrect exponential notation (e.g. [0e+-1]) have been fixed. -- The `jsoncons` parser now rejects all of the JSON_checker files that its supposed to reject except ones with stuff after the end of the document, e.g. - - ["Extra close"]] - - (Currently the `jsoncons` parser stops after reading a complete JSON text, and supports reading a sequence of JSON texts.) - -- Incorporates a fix to operator== on json objects, contributed by Alex Merry - -v0.97.1 Release --------------- - -- "Transforming JSON with filters" example fixed -- Added a class-specific in-place new to the json class that is implemented in terms of the global version (required to create json objects with placement new operator.) -- Reorganized header files, removing unnecessary includes. -- Incorporates validation contributed by Alex Merry for ensuring that there is an object or array on parse head. -- Incorporates fix contributed by Milan Burda for “Switch case is in protected scope” clang build error - -v0.97 Release ------------- - -- Reversion of v0.96 change: - -The virtual methods `do_float_value`, `do_integer_value`, and `do_unsigned_value` of `json_input_handler` and `json_output_handler` have been restored to `do_double_value`, `do_longlong_value` and `do_ulonglong_value`, and their typedefed parameter types `float_type`, `integer_type`, and `unsigned_type` have been restored to `double`, `long long`, and `unsigned long long`. - -The rationale for this reversion is that the change doesn't really help to make the software more flexible, and that it's better to leave out the typedefs. There will be future enhancements to support greater numeric precision, but these will not affect the current method signatures. - -- Fix for "unused variable" warning message - -v0.96 Release ------------- - -This release includes breaking changes to interfaces. Going forward, the interfaces are expected to be stable. - -Breaking changes: - -- Renamed `error_handler` to `parse_error_handler`. - -- Renamed namespace `json_parser_error` to `json_parser_errc` - -- Renamed `value_adapter` to `json_type_traits`, if you have implemented your own type specializations, - you will have to rename `value_adapter` also. - -- Only json arrays now support `operator[](size_t)` to loop over values, this is no longer supported for `json` objects. Use a json object iterator instead. - -- The virtual methods `do_double_value`, `do_integer_value` and `do_uinteger_value` of `json_input_handler` and `json_output_handler` have been renamed to `do_float_value`, `do_integer_value`, and `do_unsigned_value`, - and their parameters have been changed from `double`, `long long`, and `unsigned long long` to typedefs `float_type`, `integer_type`, and `unsigned_type`. - The rationale for this change is to allow different configurations for internal number types (reversed in v0.97.) - -General changes - -- `json` member function `begin_object` now returns a bidirectional iterator rather than a random access iterator. - -- Static singleton `instance` methods have been added to `default_parse_error_handler` - and `empty_json_input_handler`. - -- Added to the `json` class overloaded static methods parse, parse_string - and parse_file that take a `parse_error_handler` as a parameter. - -- Added methods `last_char()` and `eof()` to `parsing_context`. - -- Enhancements to json parsing and json parse event error notification. - -- Added to `json_input_handler` and `json_output_handler` a non virtual method `value` that takes a null terminated string. - -- Added methods `is_integer`, `is_unsigned` and `is_float` to `json` to replace `is_longlong`, `is_ulonglong` and `is_double`, which have been deprecated. - -- Added methods `as_integer`, `as_unsigned` and `as_float` to `json` to replace `is_longlong`, `is_ulonglong` and `is_double`, which have been deprecated. - -Bug fixes: - -- Fixed issue with column number reported by json_reader - -- Where &s[0] and s.length() were passed to methods, &s[0] has been replaced with s.c_str(). - While this shouldn't be an issue on most implementations, VS throws an exception in debug modes when the string has length zero. - -- Fixes two issues in v0.95 reported by Alex Merry that caused errors with GCC: a superfluous typename has been removed in csv_serializer.hpp, and a JSONCONS_NOEXCEPT specifier has been added to the json_error_category_impl name method. - -- Fixed a number of typename issues in the v0.96 candidate identifed by Ignatov Serguei. - -- Fixes issues with testsuite cmake and scons reported by Alex Merry and Ignatov Serguei - -v0.95 ----- - -Enhancements: - -- Added template method `any_cast` to `json` class. - -- The allocator type parameter in basic_json is now supported, it allows you to supply a - custom allocator for dynamically allocated, fixed size small objects in the json container. - The allocator type is not used for structures including vectors and strings that use large - or variable amounts of memory, these always use the default allocators. - -Non-breaking Change: - -- `json_filter` method `parent` has been renamed to `input_handler` (old name still works) - -Breaking change (if you've implemented your own input and output handlers, or if you've -passed json events to input and output handlers directly): - -- The input handler virtual method - `name(const std::string& name, const parsing_context& context)` - has been changed to - `do_name(const char* p, size_t length, const parsing_context& context)` - -- The output handler virtual method - `name(const std::string& name)` - has been changed to - `do_name(const char* p, size_t length)` - -- The input handler virtual method - `string_value(const std::string& value, const parsing_context& context)` - has been changed to - `do_string_value(const char* p, size_t length, const parsing_context& context)` - -- The output handler virtual method - `string_value(const std::string& value)` - has been changed to - `do_string_value(const char* p, size_t length)` - -The rationale for the method parameter changes is to allow different internal -representations of strings but preserve efficiency. - -- The input and output handler virtual implementation methods begin_json, end_json, - begin_object, end_object, begin_array, end_array, name, string_value, - longlong_value, ulonglong_value, double_value, bool_value and null_value - have been renamed to do_begin_json, do_end_json, do_begin_object, do_end_object, - do_begin_array, do_end_array, do_name, do_string_value, do_longlong_value, - do_ulonglong_value, do_double_value, do_bool_value and do_null_value and have been - made private. - -- Public non-virtual interface methods begin_json, end_json, - begin_object, end_object, begin_array, end_array, name - have been added to json_input_handler and json_output_handler. - -The rationale for these changes is to follow best C++ practices by making the -json_input_handler and json_output_handler interfaces public non-virtual and -the implementations private virtual. Refer to the documentation and tutorials for details. - -- The error_handler virtual implementation methods have been renamed to `do_warning` and - `do_error`, and made private. Non virtual public interface methods `warning` and `error` - have been added. Error handling now leverages `std::error_code` to communicate parser - error events in an extendable way. - -Bug fixes: - -- Fixed bug in csv_reader - -v0.94.1 ------- - -Bug fixes: - -- Incorporates fix from Alex Merry for comparison of json objects - -v0.94 ----- - -Bug fixes - -- Incorporates contributions from Cory Fields for silencing some compiler warnings -- Fixes bug reported by Vitaliy Gusev in json object operator[size_t] -- Fixes bug in json is_empty method for empty objects - -Changes - -- json constructors that take string, double etc. are now declared explicit (assignments and defaults to get and make_array methods have their own implementation and do not depend on implicit constructors.) -- make_multi_array renamed to make_array (old name is still supported) -- Previous versions supported any type values through special methods set_custom_data, add_custom_data, and custom_data. This version introduces a new type json::any that wraps any values and works with the usual accessors set, add and as, so the specialized methods are no longer required. - -Enhancements - -- json get method with default value now accepts extended types as defaults -- json make_array method with default value now accepts extended types as defaults - -New extensions - -- Added jsoncons_ext/boost/type_extensions.hpp to collect - extensions traits for boost types, in particular, for - boost::gregorian dates. - -v0.93 Release ------------- - -New features - -- Supports wide character strings and streams with wjson, wjson_reader etc. Assumes UTF16 encoding if sizeof(wchar_t)=2 and UTF32 encoding if sizeof(wchar_t)=4. -- The empty class null_type is added to the jsoncons namespace, it replaces the member type json::null_type (json::null_type is typedefed to jsoncons::null_type for backward compatibility.) - -Defect fixes: - -- The ascii character 0x7f (del) was not being considered a control character to be escaped, this is fixed. -- Fixed two issues with serialization when the output format property escape_all_non_ascii is enabled. One, the individual bytes were being checked if they were non ascii, rather than first converting to a codepoint. Two, continuations weren't being handled when decoding. - -v0.92a Release -------------- - -Includes contributed updates for valid compilation and execution in gcc and clang environments - -v0.92 Release ------------- - -Breaking change (but only if you have subclassed json_input_handler or json_output_handler) - -- For consistency with other names, the input and output handler methods new to v0.91 - value_string, value_double, value_longlong, value_ulonglong and value_bool - have been renamed to string_value, double_value, longlong_value, ulonglong_value and bool_value. - -Non breaking changes (previous features are deprecated but still work) - -- name_value_pair has been renamed to member_type (typedefed to previous name.) - -- as_string(output_format format) has been deprecated, use the existing to_string(output_format format) instead - -Enhancements: - -- json now has extensibilty, you can access and modify json values with new types, see the tutorial Extensibility - -Preparation for allocator support: - -- The basic_json and related classes now have an Storage template parameter, which is currently just a placeholder, but will later provide a hook to allow users to control how json storage is allocated. This addition is transparent to users of the json and related classes. - -v0.91 Release ------------- - -This release should be largely backwards compatible with v0.90 and 0.83 with two exceptions: - -1. If you have used object iterators, you will need to replace uses of std::pair with name_value_pair, in particular, first becomes name() and second becomes value(). - -2. If you have subclassed json_input_handler, json_output_handler, or json_filter, and have implemented value(const std::string& ..., value(double ..., etc., you will need to modify the names to value_string(const std::string& ..., value_double(double ... (no changes if you are feeding existing implementations.) - -The changes are - -- Replaced std::pair with name_value_pair that has accessors name() and value() - -- In json_input_handler and json_output_handler, allowed for overrides of the value methods by making them non-virtual and adding virtual methods value_string, value_double, value_longlong, value_ulonglong, and value_bool - -Other new features: - -- Changed implementation of is and as, the current implementation should be user extensible - -- make_multi_array makes a multidimensional array with the number of dimensions specified as a template parameter. Replaces make_2d_array and make_3d_array, which are now deprecated. - -- Added support for is> and as> - -- Removed JSONCONS_NO_CXX11_RVALUE_REFERENCES, compiler must support move semantics - -Incorporates a number of contributions from Pedro Larroy and the developers of the clearskies_core project: - -- build system for posix systems -- GCC to list of supported compilers -- Android fix -- fixed virtual destructors missing in json_input_handler, json_output_handler and parsing_context -- fixed const_iterator should be iterator in json_object implementation - -To clean up the interface and avoid too much duplicated functionality, we've deprecated some json methods (but they still work) - - make_array -Use json val(json::an_array) or json::make_multi_array<1>(...) instead (but make_array will continue to work) - - make_2d_array - make_3d_array -Use make_multi_array<2> and make_multi_array<3> instead - - as_vector -Use as> etc. instead - - as_int - as_uint - as_char -Use as, as, and as instead - -Release v0.90a -------------- - -Fixed issue affecting clang compile - -Release v0.90 ------------- - -This release should be fully backwards compatible with 0.83. - -Includes performance enhancements to json_reader and json_deserializer - -Fixes issues with column numbers reported with exceptions - -Incorporates a number of patches contributed by Marc Chevrier: - -- Fixed issue with set member on json object when a member with that name already exists -- clang port -- cmake build files for examples and test suite -- json template method is for examining the types of json values -- json template method as for accessing json values - -v0.83 ------------- - -Optimizations (very unlikely to break earlier code) - -- get(const std::name& name) const now returns const json& if keyed value exists, otherwise a const reference to json::null - -- get(const std::string& name, const json& default_val) const now returns const json (actually a const proxy that evaluates to json if read) - -Bug fixes - -- Line number not incremented within multiline comment - fixed - -Deprecated features removed - -- Removed deprecated output_format properties (too much bagage to carry around) - -v0.82a -------------- - -- The const version of the json operator[](const std::string& name) didn't need to return a proxy, the return value has been changed to const json& (this change is transparent to the user.) - -- get(const std::name& name) has been changed to return a copy (rather than a reference), and json::null if there is no member with that name (rather than throw.) This way both get methods return default values if no keyed value exists. - -- non-const and const methods json& at(const std::name& name) have been added to replace the old single argument get method. These have the same behavior as the corresponding operator[] functions, but the non-const at is more efficient. - -v0.81 ------------- - -- Added accessor and modifier methods floatfield to output_format to provide a supported way to set the floatfield format flag to fixed or scientific with a specified number of decimal places (this can be done in older versions, but only with deprecated methods.) - -- The default constructor now constructs an empty object (rather than a null object.) While this is a change, it's unlikely to break exisitng code (all test cases passed without modification.) - -This means that instead of - - json obj(json::an_object); - obj["field"] = "field"; - -you can simply write - - json obj; - obj["field"] = "field"; - -The former notation is still supported, though. - -- Added a version of 'resize_array' to json that resizes the array to n elements and initializes them to a specified value. - -- Added a version of the static method json::make_array that takes no arguments and makes an empty json array - -Note that - - json arr(json::an_array); - -is equivalent to - - json arr = json::make_array(); - -and - - json arr(json::an_array); - arr.resize_array(10,0.0); - -is equivalent to - - json arr = json::make_array(10,0.0); - -For consistency the json::make_array notation is now favored in the documentation. - -v0.71 -------------- - -- Added resize_array method to json for resizing json arrays - -- Fixed issue with remove_range method (templated code failed to compile if calling this method.) - -- Added remove_member method to remove a member from a json object - -- Fixed issue with multiline line comments, added test case - -- Fixed issue with adding custom data to a json array using add_custom_data, added examples. - -v0.70 -------------- - -- Since 0.50, jsoncons has used snprintf for default serialization of double values to string values. This can result in invalid json output when running on a locale like German or Spanish. The period character (‘.’) is now always used as the decimal point, non English locales are ignored. - -- The output_format methods that support alternative floating point formatting, e.g. fixed, have been deprecated. - -- Added a template method as_vector to the json class. If a json value is an array and conversion is possible to the template type, returns a std::vector of that type, otherwise throws an std::exception. Specializations are provided for std::string, bool, char, int, unsigned int, long, unsigned long, long long, unsigned long long, and double. For example - - std::string s("[0,1,2,3]"); - - json val = json::parse_string(s); - - std::vector v = val.as_vector(); - -- Undeprecated the json member function precision - -v0.60b -------------- - -This release (0.60b) is fully backwards compatible with 0.50. - -A change introduced with 0.60 has been reversed. 0.60 introduced an alternative method of constructing a json arrray or object with an initial default constructor, a bug with this was fixed in 0.60a, but this feature and related documentation has been removed because it added complexity but no real value. - -### Enhancements - -- Added swap member function to json - -- Added add and add_custom_data overrides to json that take an index value, for adding a new element at the specified index and shifting all elements currently at or above that index to the right. - -- Added capacity member functions to json - -### 0.60 extensions - -- csv_serializer has been added to the csv extension - -v0.50 ------------- - -This release is fully backwards compatible with 0.4*, and mostly backwards compatible to 0.32 apart from the two name changes in 0.41 - -Bug fixes - -- When reading the escaped characters "\\b", "\\f", "\\r" and "\\t" appearing in json strings, json_reader was replacing them with the linefeed character, this has been fixed. - -Deprecated - -- Deprecated modifiers precision and fixed_decimal_places from output_format. Use set_floating_point_format instead. -- Deprecated constructor that takes indenting parameter from output_format. For pretty printing with indenting, use the pretty_print function or pass the indenting parameter in json_serializer. - -Changes - -- When serializing floating point values to a stream, previous versions defaulted to default floating point precision with a precision of 16. This has been changed to truncate trailing zeros but keep one if immediately after a decimal point. - -New features - -- For line reporting in parser error messages, json_reader now recognizes \\r\\n, \\n alone or \\r alone (\\r alone is new.) -- Added set_floating_point_format methods to output_format to give more control over floating point notation. - -Non functional enhancements - -- json_reader now estimates the minimum capacity for arrays and objects, and reports that information for the begin_array and begin_object events. This greatly reduces reallocations. - -v0.42 ------------- - -- Fixed another bug with multi line /**/ comments -- Minor fixes to reporting line and column number of errors -- Added fixed_decimal_places setter to output_format -- Added version of as_string to json that takes output_format as a parameter -- Reorganization of test cases and examples in source tree - -v0.41 ------------- - -- Added non-member overload swap(json& a, json& b) -- Fixed bug with multi line /**/ comments -- Added begin_json and end_json methods to json_output_handler -- json_deserializer should now satisfy basic exception safety (no leak guarantee) -- Moved csv_reader.hpp to jsoncons_ext/csv directory -- Changed csv_reader namespace to jsoncons::csv -- json::parse_file no longer reads the entire file into memory before parsing - (it now uses json_reader default buffering) - -v0.40 ------------- - -- json_listener renamed to json_input_handler -- json_writer renamed to json_output_handler - -- Added json_filter class - -- json get method that takes default argument now returns a value rather than a reference -- Issue in csv_reader related to get method issue fixed -- Issue with const json operator[] fixed -- Added as_char method to json -- Improved exception safety, some opportunites for memory leaks in the presence of exceptions removed - -v0.33 ------------- - -Added reserve method to json - -Added static make_3d_array method to json - -json_reader now configured for buffered reading - -Added csv_reader class for reading CSV files and producing JSON events - -Fixed bug with explicitly passing output_format in pretty_print. - -v0.32 ------------- - -Added remove_range method, operator== and operator!= to proxy and json objects - -Added static methods make_array and make_2d_array to json - -v0.31 ------------- - -error_handler method content_error renamed to error - -Added error_code to warning, error and fatal_error methods of error_handler - -v0.30 ------------- - -json_in_stream renamed to json_listener - -json_out_stream renamed to json_writer - -Added buffer accessor method to parsing_context - -v0.20 ------------- - -Added parsing_context class for providing information about the -element being parsed. - -error_handler methods take message and parsing_context parameters - -json_in_stream handlers take parsing_context parameter - -v0.19 ------------- - -Added error_handler class for json_reader - -Made json_exception a base class for all json exceptions - -Added root() method to json_deserializer to get a reference to the json value - -Removed swap_root() method from json_deserializer - -v0.18 ------------- - -Renamed serialize() class method to to_stream() in json - -Custom data serialization supported through template function specialization of serialize -(reverses change in 0.17) - - -v0.17 ------------- - -Added is_custom() method to json and proxy - -get_custom() method renamed to custom_data() in json and proxy - -Added clear() method to json and proxy - -set_member() method renamed to set() - -set_custom() method renamed to set_custom_data() - -push_back() method renamed to add() in json and proxy - -Added add_custom_data method() in json and proxy - -Custom data serialization supported through template class specialization of custom_serialization -(replaces template function specialization of serialize) - -v0.16 ------------- - -Change to json_out_stream and json_serializer: - - void value(const custom_data& value) - -removed. - -Free function serialize replaces free function to_stream for -serializing custom data. - -pretty print tidied up for nested arrays - -v0.15 ------------- - -Made eof() method on json_reader public, to support reading -multiple JSON objects from a stream. - -v0.14 ------------- - -Added pretty_print class - -Renamed json_stream_writer to json_serializer, -implements pure virtual class json_out_stream - -Renamed json_stream_listener to json_deserializer -implements pure virtual class json_in_stream - -Renamed json_parser to json_reader, parse to read. - -Changed indenting so object and array members start on new line. - -Added support for storing user data in json object, and -serializing to JSON. - -v0.13 ------------- - -Replaced simple_string union member with json_string that -wraps std::basic_string - -name() and value() event handler methods on -basic_json_stream_writer take const std::basic_string& -rather than const Char* and length. - -v0.12 ------------- - -Implemented operator<< for json::proxy - -Added to_stream methods to json::proxy - -v0.11 ------------- - -Added members to json_parser to access and modify the buffer capacity - -Added checks when parsing integer values to determine overflow for -long long and unsigned long long, and if overflow, parse them as -doubles. - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/LICENSE b/cpp/src/core/thirdparty/jsoncons-0.126.0/LICENSE deleted file mode 100644 index d508ab08e2..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/LICENSE +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Daniel Parker 2013 - 2017. -// Distributed under the Boost Software License, Version 1.0. -// (See accompanying file LICENSE or copy at -// http://www.boost.org/LICENSE_1_0.txt) - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/README.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/README.md deleted file mode 100644 index 9c48e800aa..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/README.md +++ /dev/null @@ -1,951 +0,0 @@ -# JSONCONS - -jsoncons is a C++, header-only library for constructing [JSON](http://www.json.org) and JSON-like -data formats such as [CBOR](http://cbor.io/). It supports - -- Parsing JSON-like text or binary formats into a tree model - that defines an interface for accessing and modifying that data. - -- Serializing the tree model into different JSON-like text or binary formats. - -- Converting from JSON-like text or binary formats to C++ data structures and back via [json_type_traits](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/json_type_traits.md). - -- Streaming JSON read and write events, somewhat analogously to SAX (push parsing) and StAX (pull parsing) in the XML world. - -Compared to other JSON libraries, jsoncons has been designed to handle very large JSON texts. At its heart are -SAX style parsers and serializers. Its [json parser](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/json_parser.md) is an -incremental parser that can be fed its input in chunks, and does not require an entire file to be loaded in memory at one time. -Its tree model is more compact than most, and can be made more compact still with a user-supplied -allocator. It also supports memory efficient parsing of very large JSON texts with a [pull parser](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/json_cursor.md), -built on top of its incremental parser. - -The [jsoncons data model](doc/ref/data-model.md) supports the familiar JSON types - nulls, -booleans, numbers, strings, arrays, objects - plus byte strings. In addition, jsoncons -supports semantic tagging of date-time values, timestamp values, big integers, -big decimals, bigfloats and binary encodings. This allows it to preserve these type semantics when parsing -JSON-like data formats such as CBOR that have them. - -jsoncons is distributed under the [Boost Software License](http://www.boost.org/users/license.html). - -## Extensions - -- [jsonpointer](doc/ref/jsonpointer/jsonpointer.md) implements the IETF standard [JavaScript Object Notation (JSON) Pointer](https://tools.ietf.org/html/rfc6901) -- [jsonpatch](doc/ref/jsonpatch/jsonpatch.md) implements the IETF standard [JavaScript Object Notation (JSON) Patch](https://tools.ietf.org/html/rfc6902) -- [jsonpath](doc/ref/jsonpath/jsonpath.md) implements [Stefan Goessner's JSONPath](http://goessner.net/articles/JsonPath/). It also supports search and replace using JSONPath expressions. -- [cbor](doc/ref/cbor/cbor.md) implements decode from and encode to the IETF standard [Concise Binary Object Representation](http://cbor.io/) data format. -- [msgpack](doc/ref/msgpack/msgpack.md) implements decode from and encode to the [MessagePack](http://msgpack.org/index.html) data format. -- [ubjson](doc/ref/ubjson/ubjson.md) implements decode from and encode to the [Universal Binary JSON Specification](http://ubjson.org/) data format. -- [bson](doc/ref/bson/bson.md) implements decode from and encode to the [Binary JSON](http://bsonspec.org/) data format. -- [csv](doc/ref/csv/csv.md) implements reading (writing) JSON values from (to) CSV files - -## What users say - -_"I am so happy I have come across your json c++ library!"_ - -_"Im using your library for an external interface to pass data, as well as using the conversions from csv to json, which are really helpful for converting data for use in javascript ... it's a great library."_ - -_"this software is great and the ability to have an xpath like facility is so useful."_ - -_"really good"_ _"awesome project"_ _"very solid and very dependable"_ _"great job"_ _"Your repo rocks!!!!!"_ - -## Supported compilers - -jsoncons uses some features that are new to C++ 11, including [move semantics](http://thbecker.net/articles/rvalue_references/section_02.html) and the [AllocatorAwareContainer](http://en.cppreference.com/w/cpp/concept/AllocatorAwareContainer) concept. It is tested in continuous integration on [AppVeyor](https://ci.appveyor.com/project/danielaparker/jsoncons), [Travis](https://travis-ci.org/danielaparker/jsoncons), and [doozer](https://doozer.io/). - -| Compiler | Version |Architecture | Operating System | Notes | -|-------------------------|---------------------------|-------------|-------------------|-------| -| Microsoft Visual Studio | vs2015 (MSVC 19.0.24241.7)| x86,x64 | Windows 10 | | -| | vs2017 | x86,x64 | Windows 10 | | -| g++ | 4.8 and above | x64 | Ubuntu |`std::regex` isn't fully implemented in 4.8, so `jsoncons::jsonpath` regular expression filters aren't supported in 4.8 | -| | 4.8.5 | x64 | CentOS 7.6 |`std::regex` isn't fully implemented in 4.8, so `jsoncons::jsonpath` regular expression filters aren't supported in 4.8 | -| | 6.3.1 (Red Hat 6.3.1-1) | x64 | Fedora release 24 | | -| clang | 3.8 and above | x64 | Ubuntu | | -| clang xcode | 6.4 and above | x64 | OSX | | - -It is also cross compiled for ARMv8-A architecture on Travis using clang and executed using the emulator qemu. - -[UndefinedBehaviorSanitizer (UBSan)](http://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html) diagnostics are enabled for selected gcc and clang builds. - -## Get jsoncons - -Download the [latest release](https://github.com/danielaparker/jsoncons/releases) and unpack the zip file. Copy the directory `include/jsoncons` to your `include` directory. If you wish to use extensions, copy `include/jsoncons_ext` as well. - -Or, download the latest code on [master](https://github.com/danielaparker/jsoncons/archive/master.zip). - -## How to use it - -- [Quick guide](http://danielaparker.github.io/jsoncons) -- [Examples](doc/Examples.md) -- [Reference](doc/Home.md) - -As the `jsoncons` library has evolved, names have sometimes changed. To ease transition, jsoncons deprecates the old names but continues to support many of them. See the [deprecated list](doc/ref/deprecated.md) for the status of old names. The deprecated names can be suppressed by defining macro `JSONCONS_NO_DEPRECATED`, which is recommended for new code. - -## Benchmarks - -[json_benchmarks](https://github.com/danielaparker/json_benchmarks) provides some measurements about how `jsoncons` compares to other `json` libraries. - -- [JSONTestSuite and JSON_checker test suites](https://danielaparker.github.io/json_benchmarks/) - -- [Performance benchmarks with text and integers](https://github.com/danielaparker/json_benchmarks/blob/master/report/performance.md) - -- [Performance benchmarks with text and doubles](https://github.com/danielaparker/json_benchmarks/blob/master/report/performance_fp.md) - -### A simple example - -```c++ -#include -#include -#include - -// For convenience -using jsoncons::json; - -int main() -{ - json color_spaces = json::array(); - color_spaces.push_back("sRGB"); - color_spaces.push_back("AdobeRGB"); - color_spaces.push_back("ProPhoto RGB"); - - json image_sizing; // empty object - image_sizing["Resize To Fit"] = true; // a boolean - image_sizing["Resize Unit"] = "pixels"; // a string - image_sizing["Resize What"] = "long_edge"; // a string - image_sizing["Dimension 1"] = 9.84; // a double - - json export_settings; - - // create "File Format Options" as an object and put "Color Spaces" in it - export_settings["File Format Options"]["Color Spaces"] = std::move(color_spaces); - - export_settings["Image Sizing"] = std::move(image_sizing); - - // Write to stream - std::ofstream os("export_settings.json"); - os << export_settings; - - // Read from stream - std::ifstream is("export_settings.json"); - json j = json::parse(is); - - // Pretty print - std::cout << "(1)\n" << pretty_print(j) << "\n\n"; - - // Does object member exist? - std::cout << "(2) " << std::boolalpha << j.contains("Image Sizing") << "\n\n"; - - // Get reference to object member - const json& val = j["Image Sizing"]; - - // Access member as double - std::cout << "(3) " << "Dimension 1 = " << val["Dimension 1"].as() << "\n\n"; - - // Try access member with default - std::cout << "(4) " << "Dimension 2 = " << val.get_with_default("Dimension 2",0.0) << "\n"; -} -``` -Output: -```json -(1) -{ - "File Format Options": { - "Color Spaces": ["sRGB","AdobeRGB","ProPhoto RGB"], - "Image Formats": ["JPEG","PSD","TIFF","DNG"] - }, - "File Settings": { - "Color Space": "sRGB", - "Image Format": "JPEG", - "Limit File Size": true, - "Limit File Size To": 10000 - }, - "Image Sizing": { - "Dimension 1": 9.84, - "Resize To Fit": true, - "Resize Unit": "pixels", - "Resize What": "long_edge" - } -} - -(2) true - -(3) Dimension 1 = 9.8 - -(4) Dimension 2 = 0 -``` - -## About jsoncons::basic_json - -The jsoncons library provides a `basic_json` class template, which is the generalization of a `json` value for different -character types, different policies for ordering name-value pairs, etc. A `basic_json` provides a tree model -of JSON-like data formats, and defines an interface for accessing and modifying that data. -Despite its name, it is not JSON specific. - -```c++ -typedef basic_json> json; -``` -The library includes four instantiations of `basic_json`: - -- [json](doc/ref/json.md) constructs a utf8 character json value that sorts name-value members alphabetically - -- [ojson](doc/ref/ojson.md) constructs a utf8 character json value that preserves the original name-value insertion order - -- [wjson](doc/ref/wjson.md) constructs a wide character json value that sorts name-value members alphabetically - -- [wojson](doc/ref/wojson.md) constructs a wide character json value that preserves the original name-value insertion order - -## More examples - -[Encode C++ data structures to JSON, decode JSON to C++ data structures](#E1) - -[Playing around with CBOR, JSON, and CSV](#E2) - -[Query CBOR with JSONPath](#E3) - -[Pull parser example](#E4) - -[Iterate over a json stream with staj iterators](#E5) - -[Dump json content into a larger document](#E6) - -
- -### Encode C++ data structures to JSON, decode JSON to C++ data structures - -jsoncons supports conversion between C++ data structures and JSON. The functions [encode_json](doc/ref/encode_json.md) -and [decode_json](doc/ref/decode_json.md) convert C++ data structures to JSON formatted strings or streams and back. -Encode and decode work for all C++ classes that have -[json_type_traits](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/json_type_traits.md) -defined. The standard library containers are already supported, and you can specialize `json_type_traits` -for your own types in the `jsoncons` namespace. - -`JSONCONS_MEMBER_TRAITS_DECL` is a macro that simplifies the creation of the necessary boilerplate -from member data. It must be placed outside any namespace blocks. - -```c++ -#include -#include -#include - -namespace ns { - - struct reputon - { - std::string rater; - std::string assertion; - std::string rated; - double rating; - - friend bool operator==(const reputon& lhs, const reputon& rhs) - { - return lhs.rater == rhs.rater && lhs.assertion == rhs.assertion && - lhs.rated == rhs.rated && lhs.rating == rhs.rating; - } - - friend bool operator!=(const reputon& lhs, const reputon& rhs) - { - return !(lhs == rhs); - }; - }; - - class reputation_object - { - std::string application; - std::vector reputons; - - // Make json_type_traits specializations friends to give accesses to private members - JSONCONS_TYPE_TRAITS_FRIEND; - public: - reputation_object() - { - } - reputation_object(const std::string& application, const std::vector& reputons) - : application(application), reputons(reputons) - {} - - friend bool operator==(const reputation_object& lhs, const reputation_object& rhs) - { - return (lhs.application == rhs.application) && (lhs.reputons == rhs.reputons); - } - - friend bool operator!=(const reputation_object& lhs, const reputation_object& rhs) - { - return !(lhs == rhs); - }; - }; - - -} // namespace ns - -// Declare the traits. Specify which data members need to be serialized. -JSONCONS_MEMBER_TRAITS_DECL(ns::reputon, rater, assertion, rated, rating) -JSONCONS_MEMBER_TRAITS_DECL(ns::reputation_object, application, reputons) - -using namespace jsoncons; // for convenience - -int main() -{ - ns::reputation_object val("hiking", { ns::reputon{"HikingAsylum.example.com","strong-hiker","Marilyn C",0.90} }); - - std::string s; - encode_json(val, s, indenting::indent); - std::cout << s << "\n"; - - auto val2 = decode_json(s); - - assert(val2 == val); -} -``` -Output: -``` -{ - "application": "hiking", - "reputons": [ - { - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rater": "HikingAsylum.example.com", - "rating": 0.9 - } - ] -} -``` - -See [examples](https://github.com/danielaparker/jsoncons/blob/master/doc/Examples.md#G1) - -
- -### Playing around with CBOR, JSON, and CSV - -```c++ -#include -#include -#include -#include - -// For convenience -using namespace jsoncons; - -int main() -{ - // Construct some CBOR using the streaming API - std::vector b; - cbor::cbor_bytes_encoder encoder(b); - encoder.begin_array(); // indefinite length outer array - encoder.begin_array(3); // a fixed length array - encoder.string_value("foo"); - encoder.byte_string_value(byte_string{'P','u','s','s'}); // no suggested conversion - encoder.string_value("-18446744073709551617", semantic_tag::bigint); - encoder.end_array(); - encoder.end_array(); - encoder.flush(); - - // Print bytes - std::cout << "(1) "; - for (auto c : b) - { - std::cout << std::hex << std::setprecision(2) << std::setw(2) - << std::setfill('0') << static_cast(c); - } - std::cout << "\n\n"; -/* - 9f -- Start indefinte length array - 83 -- Array of length 3 - 63 -- String value of length 3 - 666f6f -- "foo" - 44 -- Byte string value of length 4 - 50757373 -- 'P''u''s''s' - c3 -- Tag 3 (negative bignum) - 49 -- Byte string value of length 9 - 010000000000000000 -- Bytes content - ff -- "break" -*/ - // Unpack bytes into a json variant value, and add some more elements - json j = cbor::decode_cbor(b); - - // Loop over the rows - std::cout << "(2)\n"; - for (const json& row : j.array_range()) - { - std::cout << row << "\n"; - } - std::cout << "\n"; - - // Get bignum value at position 0/2 using jsonpointer - json& v = jsonpointer::get(j, "/0/2"); - std::cout << "(3) " << v.as() << "\n\n"; - - // Print JSON representation with default options - std::cout << "(4)\n"; - std::cout << pretty_print(j) << "\n\n"; - - // Print JSON representation with different options - json_options options; - options.byte_string_format(byte_string_chars_format::base64) - .bigint_format(bigint_chars_format::base64url); - std::cout << "(5)\n"; - std::cout << pretty_print(j, options) << "\n\n"; - - // Add some more elements - - json another_array = json::array(); - another_array.emplace_back(byte_string({'P','u','s','s'}), - semantic_tag::base64); // suggested conversion to base64 - another_array.emplace_back("273.15", semantic_tag::bigdec); - another_array.emplace(another_array.array_range().begin(),"bar"); // place at front - - j.push_back(std::move(another_array)); - std::cout << "(6)\n"; - std::cout << pretty_print(j) << "\n\n"; - - // Get big decimal value at position /1/2 using jsonpointer - json& ref = jsonpointer::get(j, "/1/2"); - std::cout << "(7) " << ref.as() << "\n\n"; - -#if (defined(__GNUC__) || defined(__clang__)) && (!defined(__STRICT_ANSI__) && defined(_GLIBCXX_USE_INT128)) - // e.g. if code compiled with GCC and std=gnu++11 (rather than std=c++11) - __int128 i = j[1][2].as<__int128>(); -#endif - - // Get byte string value at position /1/1 as a byte_string - byte_string bs = j[1][1].as(); - std::cout << "(8) " << bs << "\n\n"; - - // or alternatively as a std::vector - std::vector u = j[1][1].as>(); - - // Repack bytes - std::vector b2; - cbor::encode_cbor(j, b2); - - // Print the repacked bytes - std::cout << "(9) "; - for (auto c : b2) - { - std::cout << std::hex << std::setprecision(2) << std::setw(2) - << std::setfill('0') << static_cast(c); - } - std::cout << "\n\n"; -/* - 82 -- Array of length 2 - 83 -- Array of length 3 - 63 -- String value of length 3 - 666f6f -- "foo" - 44 -- Byte string value of length 4 - 50757373 -- 'P''u''s''s' - c3 -- Tag 3 (negative bignum) - 49 -- Byte string value of length 9 - 010000000000000000 -- Bytes content - 83 -- Another array of length 3 - 63 -- String value of length 3 - 626172 -- "bar" - d6 - Expected conversion to base64 - 44 -- Byte string value of length 4 - 50757373 -- 'P''u''s''s' - c4 -- Tag 4 (decimal fraction) - 82 -- Array of length 2 - 21 -- -2 - 19 6ab3 -- 27315 -*/ - // Encode to CSV - csv::csv_options csv_options; - csv_options.column_names("Column 1,Column 2,Column 3"); - - std::cout << "(10)\n"; - csv::encode_csv(j, std::cout, csv_options); -} - -``` -Output: -``` -(1) 9f8363666f6f4450757373c349010000000000000000ff - -(2) -["foo","UHVzcw","-18446744073709551617"] - -(3) -18446744073709551617 - -(4) -[ - ["foo", "UHVzcw", "-18446744073709551617"] -] - -(5) -[ - ["foo", "UHVzcw==", "~AQAAAAAAAAAA"] -] - -(6) -[ - ["foo", "UHVzcw", "-18446744073709551617"], - ["bar", "UHVzcw==", "273.15"] -] - -(7) 273.15 - -(8) 50757373 - -(9) 828363666f6f4450757373c3490100000000000000008363626172d64450757373c48221196ab3 - -(10) -Column 1,Column 2,Column 3 -foo,UHVzcw,-18446744073709551617 -bar,UHVzcw==,273.15 -``` - -
- -### Query CBOR with JSONPath -```c++ -#include -#include -#include -#include - -using namespace jsoncons; // For convenience - -int main() -{ - // Construct a json array of numbers - json j = json::array(); - - j.emplace_back(5.0); - - j.emplace_back(0.000071); - - j.emplace_back("-18446744073709551617",semantic_tag::bigint); - - j.emplace_back("1.23456789012345678901234567890", semantic_tag::bigdec); - - j.emplace_back("0x3p-1", semantic_tag::bigfloat); - - // Encode to JSON - std::cout << "(1)\n"; - std::cout << pretty_print(j); - std::cout << "\n\n"; - - // as() and as() - std::cout << "(2)\n"; - std::cout << std::dec << std::setprecision(15); - for (const auto& item : j.array_range()) - { - std::cout << item.as() << ", " << item.as() << "\n"; - } - std::cout << "\n"; - - // Encode to CBOR - std::vector v; - cbor::encode_cbor(j,v); - - std::cout << "(3)\n"; - for (auto c : v) - { - std::cout << std::hex << std::setprecision(2) << std::setw(2) - << std::setfill('0') << static_cast(c); - } - std::cout << "\n\n"; -/* - 85 -- Array of length 5 - fa -- float - 40a00000 -- 5.0 - fb -- double - 3f129cbab649d389 -- 0.000071 - c3 -- Tag 3 (negative bignum) - 49 -- Byte string value of length 9 - 010000000000000000 - c4 -- Tag 4 (decimal fraction) - 82 -- Array of length 2 - 38 -- Negative integer of length 1 - 1c -- -29 - c2 -- Tag 2 (positive bignum) - 4d -- Byte string value of length 13 - 018ee90ff6c373e0ee4e3f0ad2 - c5 -- Tag 5 (bigfloat) - 82 -- Array of length 2 - 20 -- -1 - 03 -- 3 -*/ - - // Decode back to json - json other = cbor::decode_cbor(v); - assert(other == j); - - // Query with JSONPath - std::cout << "(4)\n"; - json result = jsonpath::json_query(other,"$[?(@ < 1.5)]"); - std::cout << pretty_print(result) << "\n\n"; -``` -Output: -``` -(1) -[ - 5.0, - 7.1e-05, - "-18446744073709551617", - "1.23456789012345678901234567890", - [-1, 3] -] - -(2) -5.0, 5 -7.1e-05, 7.1e-05 --18446744073709551617, -1.84467440737096e+19 -1.23456789012345678901234567890, 1.23456789012346 -1.5, 1.5 - -(3) -85fa40a00000fb3f129cbab649d389c349010000000000000000c482381cc24d018ee90ff6c373e0ee4e3f0ad2c5822003 - -(4) -[ - 7.1e-05, - "-18446744073709551617", - "1.23456789012345678901234567890" -] -``` - -
- -### Pull parser example - -A typical pull parsing application will repeatedly process the `current()` -event and call `next()` to advance to the next event, until `done()` -returns `true`. - -The example JSON text, `book_catalog.json`, is used by the examples below. - -```json -[ - { - "author" : "Haruki Murakami", - "title" : "Hard-Boiled Wonderland and the End of the World", - "isbn" : "0679743464", - "publisher" : "Vintage", - "date" : "1993-03-02", - "price": 18.90 - }, - { - "author" : "Graham Greene", - "title" : "The Comedians", - "isbn" : "0099478374", - "publisher" : "Vintage Classics", - "date" : "2005-09-21", - "price": 15.74 - } -] -``` - -#### Reading the JSON stream -```c++ -std::ifstream is("book_catalog.json"); - -json_cursor reader(is); - -for (; !reader.done(); reader.next()) -{ - const auto& event = reader.current(); - switch (event.event_type()) - { - case staj_event_type::begin_array: - std::cout << "begin_array\n"; - break; - case staj_event_type::end_array: - std::cout << "end_array\n"; - break; - case staj_event_type::begin_object: - std::cout << "begin_object\n"; - break; - case staj_event_type::end_object: - std::cout << "end_object\n"; - break; - case staj_event_type::name: - // If underlying type is string, can return as string_view - std::cout << "name: " << event.as() << "\n"; - break; - case staj_event_type::string_value: - std::cout << "string_value: " << event.as() << "\n"; - break; - case staj_event_type::null_value: - std::cout << "null_value: " << event.as() << "\n"; - break; - case staj_event_type::bool_value: - std::cout << "bool_value: " << event.as() << "\n"; - break; - case staj_event_type::int64_value: - std::cout << "int64_value: " << event.as() << "\n"; - break; - case staj_event_type::uint64_value: - std::cout << "uint64_value: " << event.as() << "\n"; - break; - case staj_event_type::double_value: - // Return as string, could also use event.as() - std::cout << "double_value: " << event.as() << "\n"; - break; - default: - std::cout << "Unhandled event type\n"; - break; - } -} -``` -Output: -``` -begin_array -begin_object -name: author -string_value: Haruki Murakami -name: title -string_value: Hard-Boiled Wonderland and the End of the World -name: isbn -string_value: 0679743464 -name: publisher -string_value: Vintage -name: date -string_value: 1993-03-02 -name: price -double_value: 18.90 -end_object -begin_object -name: author -string_value: Graham Greene -name: title -string_value: The Comedians -name: isbn -string_value: 0099478374 -name: publisher -string_value: Vintage Classics -name: date -string_value: 2005-09-21 -name: price -double_value: 15.74 -end_object -end_array -``` - -
- -#### Implementing a staj_filter - -```c++ -// A stream filter to filter out all events except name -// and restrict name to "author" - -class author_filter : public staj_filter -{ - bool accept_next_ = false; -public: - bool accept(const staj_event& event, const ser_context&) override - { - if (event.event_type() == staj_event_type::name && - event.as() == "author") - { - accept_next_ = true; - return false; - } - else if (accept_next_) - { - accept_next_ = false; - return true; - } - else - { - accept_next_ = false; - return false; - } - } -}; -``` - -#### Filtering the JSON stream - -```c++ -std::ifstream is("book_catalog.json"); - -author_filter filter; -json_cursor reader(is, filter); - -for (; !reader.done(); reader.next()) -{ - const auto& event = reader.current(); - switch (event.event_type()) - { - case staj_event_type::string_value: - std::cout << event.as() << "\n"; - break; - } -} -``` -Output: -``` -Haruki Murakami -Graham Greene -``` - -See [json_cursor](doc/ref/json_cursor.md) - -### Iterate over a json stream with staj iterators - -```c++ -const std::string example = R"( -[ - { - "employeeNo" : "101", - "name" : "Tommy Cochrane", - "title" : "Supervisor" - }, - { - "employeeNo" : "102", - "name" : "Bill Skeleton", - "title" : "Line manager" - } -] -)"; - -int main() -{ - std::istringstream is(example); - - json_cursor reader(is); - - staj_array_iterator it(reader); - - for (const auto& j : it) - { - std::cout << pretty_print(j) << "\n"; - } - std::cout << "\n\n"; -} -``` -Output: -``` -{ - "employeeNo": "101", - "name": "Tommy Cochrane", - "title": "Supervisor" -} -{ - "employeeNo": "102", - "name": "Bill Skeleton", - "title": "Line manager" -} -``` -See [staj_array_iterator](doc/ref/staj_array_iterator.md) and [staj_object_iterator](doc/ref/staj_object_iterator.md) - -
- -### Dump json content into a larger document - -```c++ -#include - -using namespace jsoncons; - -int main() -{ - const json some_books = json::parse(R"( - [ - { - "title" : "Kafka on the Shore", - "author" : "Haruki Murakami", - "price" : 25.17 - }, - { - "title" : "Women: A Novel", - "author" : "Charles Bukowski", - "price" : 12.00 - } - ] - )"); - - const json more_books = json::parse(R"( - [ - { - "title" : "A Wild Sheep Chase: A Novel", - "author" : "Haruki Murakami", - "price" : 9.01 - }, - { - "title" : "Cutter's Way", - "author" : "Ivan Passer", - "price" : 8.00 - } - ] - )"); - - json_encoder encoder(std::cout, jsoncons::indenting::indent); // pretty print - serializer.begin_array(); - for (const auto& book : some_books.array_range()) - { - book.dump(encoder); - } - for (const auto& book : more_books.array_range()) - { - book.dump(encoder); - } - serializer.end_array(); - serializer.flush(); -} -``` -Output: -```json -[ - { - "author": "Haruki Murakami", - "price": 25.17, - "title": "Kafka on the Shore" - }, - { - "author": "Charles Bukowski", - "price": 12.0, - "title": "Women: A Novel" - }, - { - "author": "Haruki Murakami", - "price": 9.01, - "title": "A Wild Sheep Chase: A Novel" - }, - { - "author": "Ivan Passer", - "price": 8.0, - "title": "Cutter's Way" - } -] -``` - -## Building the test suite and examples with CMake - -[CMake](https://cmake.org/) is a cross-platform build tool that generates makefiles and solutions for the compiler environment of your choice. On Windows you can download a [Windows Installer package](https://cmake.org/download/). On Linux it is usually available as a package, e.g., on Ubuntu, -``` -sudo apt-get install cmake -``` -Once cmake is installed, you can build the tests: -``` -mkdir build -cd build -cmake ../ -DBUILD_TESTS=ON -cmake --build . --target test_jsoncons --config Release -``` -Run from the jsoncons tests directory: - -On Windows: -``` -..\build\tests\Release\test_jsoncons -``` - -On UNIX: -``` -../build/tests/Release/test_jsoncons -``` - -## Acknowledgements - -The jsoncons platform dependent binary configuration draws on to the excellent MIT licensed [tinycbor](https://github.com/intel/tinycbor). - -A big thanks to Milo Yip, author of [RapidJSON](http://rapidjson.org/), for raising the quality of JSON libraries across the board, by publishing [the benchmarks](https://github.com/miloyip/nativejson-benchmark), and contacting this project (among others) to share the results. - -The jsoncons implementation of the Grisu3 algorithm for printing floating-point numbers follows Florian Loitsch's MIT licensed [grisu3_59_56 implementation](http://florian.loitsch.com/publications), with minor modifications. - -The macro `JSONCONS_MEMBER_TRAITS_DECL` was inspired by Martin York's [ThorsSerializer](https://github.com/Loki-Astari/ThorsSerializer) - -Special thanks to our [contributors](https://github.com/danielaparker/jsoncons/blob/master/acknowledgements.md) - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/Roadmap.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/Roadmap.md deleted file mode 100644 index f78cdc9698..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/Roadmap.md +++ /dev/null @@ -1,15 +0,0 @@ -# Roadmap - -### For later releases - -- Generaliztion and enhancement of encode and decode functions - -At this point we'll slap a Version 1.0.0 Full Release stamp on `jsoncons` -(we've been leading up to this since 2013.) - -### Post 1.0.0 - -- Support more error recovery and introduce optional `lenient_error_handler`. - -- Implement [Concise data definition language (CDDL)](https://tools.ietf.org/html/draft-ietf-cbor-cddl-08) for schema validation in `jsoncons_ext` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/acknowledgements.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/acknowledgements.md deleted file mode 100644 index 638caefcfd..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/acknowledgements.md +++ /dev/null @@ -1,76 +0,0 @@ -A big thanks to the following individuals for contributing: - -- Andrew Hutko (early code review) - -- [Marc Chevrier](https://github.com/MarkaPola) (contributed clang port, build files, json is and as methods, -and make_array template implementation.) - -- [Pedro Larroy](https://github.com/larroy) and the developers of the clearskies_core project (contributed build -system for posix systems, adding GCC to list of supported compilers, bug fixes, -Android fix) - -- [Cory Fields](https://github.com/theuni) for fixing warnings about unused variables - -- [Vitaliy Gusev](https://github.com/gusev-vitaliy) (reported error in json object operator[size_t i]) - -- [Alex Merry](https://github.com/amerry) for reporting errors with "typename" keyword experienced with gcc and providing -workaround for gcc 4.8 regex issues. - -- [Ignatov Serguei](https://github.com/sergign60) (reported issues experienced with gcc for 0.95 and -0.96 candidate and helped fix them) - -- [Milan Burda](https://github.com/miniak) for fix for clang build error - -- [Peter Tissen](https://github.com/Bigpet), for reporting and suggesting a fix for get(name,default_val) - -- [Tom Bass](https://github.com/tbass) for assistance with clang build errors - -- [Andrey Alifanov](https://github.com/AndreyAlifanov) and [Amit Naik](https://github.com/amitnaik1) for failing test cases for JSON Path - -- [Yuri Plaksyuk](https://github.com/yplaksyuk) for contributing an extension to JsonPath to allow filter -expressions over a single object. - -- [Nikolay Amiantov](https://github.com/abbradar) for fixing compilation errors and warnings by GCC and -Clang, adding read support for std::array and, most appreciated, -adding Travis CI configuration. - -- [jakalx](https://github.com/jakalx) contributed fix for operator== throws when comparing a string -against an empty object - -- [Alexander](https://github.com/rog13) for contributing fix to jsonpatch::diff - -- [Stefano Sinigardi](https://github.com/cenit) for contributing workaround for vs2017 platform issue - -- [xezon](https://github.com/danielaparker/jsoncons/pull/140) for proposing decode_csv and encode_csv functions, the -ignore_empty_lines option, and fixes to mismatched allocator types. Also for fixes and improvements in string_view code. - -- Vojtech Fried for contributing patches to JSONCONS_DEFINE_LITERAL -and to json::as_string to remove warnings - -- [Joshua Pritikin](https://github.com/jpritikin), for reporting gcc ubsan runtime warnings about -load of misaligned addresses, and verifying fix - -- [Tobias Hermann](https://github.com/Dobiasd), for reporting issue with `UINT_MAX` not declared -in `bignum.hpp`, and proposing fix. - -- [Cebtenzzre](https://github.com/Cebtenzzre), for finding and fixing an issue with conversions on -a basic_json value leading to an infinite recursion when the -value is a bignum, and for fixing undefined behavior in the bignum -class. - -- [massimo morara](https://github.com/massimomorara) for reporting numerous issues - -- [Alexander B](https://github.com/bas524), for uncovering a bug in how json_parser validated -UTF-8 strings. - -- [zhskyy](https://github.com/zhskyy), for contributing __FILE__ and __LINE__ macros removed -from JSONCONS_ASSERT if not defined _DEBUG. - -- [soberich](https://github.com/soberich), for contributing the jsonpath sum and prod functions, -and a proposal for aggregation functions that work outside a filter. - -- [patternoia](https://github.com/patternoia) for fixing the installation script -to include copying the jsoncons_ext directory into the installation place - -- [mikewallis](https://github.com/mikewallis) for removing redundant macro continuation character in JSONCONS_TYPE_TRAITS_DECL - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/appveyor.yml b/cpp/src/core/thirdparty/jsoncons-0.126.0/appveyor.yml deleted file mode 100644 index e89d60c9f1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/appveyor.yml +++ /dev/null @@ -1,86 +0,0 @@ -build: false - -environment: - vsversion: none - arch: default - matrix: - - platform: vs - vsversion: 2015 - arch: x86 - FLAGS: "" - - - platform: vs - vsversion: 2015 - arch: x86 - FLAGS: "" - - - platform: vs - vsversion: 2015 - arch: x86 - FLAGS: "/permissive- /std:c++latest /utf-8" - - - platform: vs - vsversion: 2015 - arch: x64 - FLAGS: "" - - - platform: vs - vsversion: 2017 - arch: x64 - FLAGS: "" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - - - platform: vs - vsversion: 2017 - arch: x64 - FLAGS: "/permissive- /std:c++latest /utf-8" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - - - platform: vs - vsversion: 2017 - arch: ARM - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 - -init: - - git config --global core.autocrlf input - -before_build: - # Remove the following from the path, as it will interfere with - # the MinGW builds - - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% - - if %platform%==msvc call "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" - - if %platform%==msvc cmake -G "NMake Makefiles" -DCMAKE_INSTALL_PREFIX=%P% - - if %platform%==vs ( - set "makecommand=Visual Studio" - ) - - set "vcx=false" - - set "vcs=false" - - if %platform%==vs ( - set "vcx=true" - ) - - if %vsversion%==2015 ( - set "makecommand=%makecommand% 14 %vsversion%" - ) - - if %vsversion%==2017 ( - set "makecommand=%makecommand% 15 %vsversion%" - ) - - if %arch%==x64 ( - set "makecommand=%makecommand% Win64" - ) - - if %arch%==ARM ( - set "makecommand=%makecommand% ARM" - ) - - - cmake -G "%makecommand%" -D BUILD_TESTS=1 . - -build_script: - - cmake --build . --target test_jsoncons --config Release - - cd tests - -test_script: - - set "testplatform=%platform%" - # Can not run ARM builds on x86/x64 build images - - if %arch%==ARM ( - set "testplatform=none" - ) - - if %testplatform%==vs .\Release\test_jsoncons diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/build.sh b/cpp/src/core/thirdparty/jsoncons-0.126.0/build.sh deleted file mode 100755 index e31140ac4a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -x - -INSTALL_PREFIX=$(pwd)/../build - -cp -rf include/* ${INSTALL_PREFIX}/include diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/build_files/cmake/Darwin.cmake b/cpp/src/core/thirdparty/jsoncons-0.126.0/build_files/cmake/Darwin.cmake deleted file mode 100644 index e1687258a3..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/build_files/cmake/Darwin.cmake +++ /dev/null @@ -1,7 +0,0 @@ -# -# Global Configuration for MacOS platform -# - -# customize compiler flags -## Add new flags - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/build_files/cmake/Linux.cmake b/cpp/src/core/thirdparty/jsoncons-0.126.0/build_files/cmake/Linux.cmake deleted file mode 100644 index 2aa24c210e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/build_files/cmake/Linux.cmake +++ /dev/null @@ -1,16 +0,0 @@ -# -# Global Configuration for linux platform -# - -# -# GNU libstdc++ runtime is not supported because not yet C++11 compliant -# - -# customize compiler flags -## Add new flags -add_definitions (-pthread) - -set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pthread") -set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pthread") -set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pthread") - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/build_files/cmake/Windows.cmake b/cpp/src/core/thirdparty/jsoncons-0.126.0/build_files/cmake/Windows.cmake deleted file mode 100644 index deb42645cc..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/build_files/cmake/Windows.cmake +++ /dev/null @@ -1,6 +0,0 @@ -# -# Global Configuration for windows platform -# - -# define some preprocessor flags -add_definitions(/DWIN32_LEAN_AND_MEAN /D_UNICODE /DUNICODE /W4) diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/build_files/cmake/config.cmake.in b/cpp/src/core/thirdparty/jsoncons-0.126.0/build_files/cmake/config.cmake.in deleted file mode 100644 index f5e390b030..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/build_files/cmake/config.cmake.in +++ /dev/null @@ -1,13 +0,0 @@ -# jsoncons cmake module -# This module sets the following variables in your project:: -# -# jsoncons_FOUND - true if jsoncons found on the system -# jsoncons_INCLUDE_DIRS - the directory containing jsoncons headers -# jsoncons_LIBRARY - empty - -@PACKAGE_INIT@ - -if(NOT TARGET @PROJECT_NAME@) - include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake") - get_target_property(@PROJECT_NAME@_INCLUDE_DIRS jsoncons INTERFACE_INCLUDE_DIRECTORIES) -endif() diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/css/jsoncons.css b/cpp/src/core/thirdparty/jsoncons-0.126.0/css/jsoncons.css deleted file mode 100644 index 8b13789179..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/css/jsoncons.css +++ /dev/null @@ -1 +0,0 @@ - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Examples.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Examples.md deleted file mode 100644 index 5dd154e578..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Examples.md +++ /dev/null @@ -1,1554 +0,0 @@ -# Examples - -### Parse and decode - -[Parse JSON from a string](#A1) -[Parse JSON from a file](#A2) -[Parse numbers without loosing precision](#A8) -[Validate JSON without incurring parse exceptions](#A3) -[How to allow comments? How not to?](#A4) -[Set a maximum nesting depth](#A5) -[Prevent the alphabetic sort of the outputted JSON, retaining the original insertion order](#A6) -[Parse a very large JSON file with json_cursor](#A7) - -### Encode - -[Encode a json value to a string](#B1) -[Encode a json value to a stream](#B2) -[Escape all non-ascii characters](#B3) -[Replace the representation of NaN, Inf and -Inf when serializing. And when reading in again.](#B4) - -### Decode JSON to C++ data structures, encode C++ data structures to JSON - -[Convert JSON to/from C++ data structures by specializing json_type_traits](#G1) -[A simple example using JSONCONS_MEMBER_TRAITS_DECL to generate the json_type_traits](#G2) -[A polymorphic example using JSONCONS_GETTER_CTOR_TRAITS_DECL to generate the json_type_traits](#G3) -[Convert JSON numbers to/from boost multiprecision numbers](#G4) - -### Construct - -[Construct a json object](#C1) -[Construct a json array](#C2) -[Insert a new value in an array at a specific position](#C3) -[Create arrays of arrays of arrays of ...](#C4) -[Merge two json objects](#C5) -[Construct a json byte string](#C6) - -### Access - -[Use string_view to access the actual memory that's being used to hold a string](#E1) -[Given a string in a json object that represents a decimal number, assign it to a double](#E2) -[Retrieve a big integer that's been parsed as a string](#E3) -[Look up a key, if found, return the value converted to type T, otherwise, return a default value of type T](#E4) -[Retrieve a value in a hierarchy of JSON objects](#E5) -[Retrieve a json value as a byte string](#E6) - -### Iterate - -[Iterate over a json array](#D1) -[Iterate over a json object](#D2) - -### Search and Replace - -[Search for and repace an object member key](#F1) -[Search for and replace a value](#F2) - -### Parse and decode - -
- -#### Parse JSON from a string - -``` -std::string s = R"({"first":1,"second":2,"fourth":3,"fifth":4})"; - -json j = json::parse(s); -``` - -or - -```c++ -using namespace jsoncons::literals; - -json j = R"( -{ - "StartDate" : "2017-03-01", - "MaturityDate" : "2020-12-30" -} -)"_json; -``` - -
- -#### Parse JSON from a file - -``` -std::ifstream is("myfile.json"); - -json j = json::parse(is); -``` - -
- -#### Parse numbers without loosing precision - -By default, jsoncons parses a number with an exponent or fractional part -into a double precision floating point number. If you wish, you can -keep the number as a string with semantic tagging `bigdec`, -using the `lossless_number` option. You can then put it into a `float`, -`double`, a boost multiprecision number, or whatever other type you want. - -```c++ -#include - -int main() -{ - std::string s = R"( - { - "a" : 12.00, - "b" : 1.23456789012345678901234567890 - } - )"; - - // Default - json j = json::parse(s); - - std::cout.precision(15); - - // Access as string - std::cout << "(1) a: " << j["a"].as() << ", b: " << j["b"].as() << "\n"; - // Access as double - std::cout << "(2) a: " << j["a"].as() << ", b: " << j["b"].as() << "\n\n"; - - // Using lossless_number option - json_options options; - options.lossless_number(true); - - json j2 = json::parse(s, options); - // Access as string - std::cout << "(3) a: " << j2["a"].as() << ", b: " << j2["b"].as() << "\n"; - // Access as double - std::cout << "(4) a: " << j2["a"].as() << ", b: " << j2["b"].as() << "\n\n"; -} -``` -Output: -``` -(1) a: 12.0, b: 1.2345678901234567 -(2) a: 12, b: 1.23456789012346 - -(3) a: 12.00, b: 1.23456789012345678901234567890 -(4) a: 12, b: 1.23456789012346 -``` - -
- -#### Validate JSON without incurring parse exceptions -```c++ -std::string s = R"( -{ - "StartDate" : "2017-03-01", - "MaturityDate" "2020-12-30" -} -)"; - -json_reader reader(s); - -// or, -// std::stringstream is(s); -// json_reader reader(is); - -std::error_code ec; -reader.read(ec); - -if (ec) -{ - std::cout << ec.message() - << " on line " << reader.line() - << " and column " << reader.column() - << std::endl; -} -``` -Output: -``` -Expected name separator ':' on line 4 and column 20 -``` - -
- -#### How to allow comments? How not to? - -jsoncons, by default, accepts and ignores C-style comments - -```c++ -std::string s = R"( -{ - // Single line comments - /* - Multi line comments - */ -} -)"; - -// Default -json j = json::parse(s); -std::cout << "(1) " << j << std::endl; - -// Strict -try -{ - strict_parse_error_handler err_handler; - json j = json::parse(s, err_handler); -} -catch (const ser_error& e) -{ - std::cout << "(2) " << e.what() << std::endl; -} -``` -Output: -``` -(1) {} -(2) Illegal comment at line 3 and column 10 -``` - -
- -#### Set a maximum nesting depth - -Like this, -```c++ -std::string s = "[[[[[[[[[[[[[[[[[[[[[\"Too deep\"]]]]]]]]]]]]]]]]]]]]]"; -try -{ - json_options options; - options.max_nesting_depth(20); - json j = json::parse(s, options); -} -catch (const ser_error& e) -{ - std::cout << e.what() << std::endl; -} -``` -Output: -``` -Maximum JSON depth exceeded at line 1 and column 21 -``` - -
- -#### Prevent the alphabetic sort of the outputted JSON, retaining the original insertion order - -Use `ojson` instead of `json` (or `wojson` instead of `wjson`) to retain the original insertion order. - -```c++ -ojson j = ojson::parse(R"( -{ - "street_number" : "100", - "street_name" : "Queen St W", - "city" : "Toronto", - "country" : "Canada" -} -)"); -std::cout << "(1)\n" << pretty_print(j) << std::endl; - -// Insert "postal_code" at end -j.insert_or_assign("postal_code", "M5H 2N2"); -std::cout << "(2)\n" << pretty_print(j) << std::endl; - -// Insert "province" before "country" -auto it = j.find("country"); -j.insert_or_assign(it,"province","Ontario"); -std::cout << "(3)\n" << pretty_print(j) << std::endl; -``` -Output: -``` -(1) -{ - "street_number": "100", - "street_name": "Queen St W", - "city": "Toronto", - "country": "Canada" -} -(2) -{ - "street_number": "100", - "street_name": "Queen St W", - "city": "Toronto", - "country": "Canada", - "postal_code": "M5H 2N2" -} -(3) -{ - "street_number": "100", - "street_name": "Queen St W", - "city": "Toronto", - "province": "Ontario", - "country": "Canada", - "postal_code": "M5H 2N2" -} -``` - -
- -### Parse a very large JSON file with json_cursor - - -A typical pull parsing application will repeatedly process the `current()` -event and call `next()` to advance to the next event, until `done()` -returns `true`. - -The example JSON text, `book_catalog.json`, is used by the examples below. - -```json -[ - { - "author" : "Haruki Murakami", - "title" : "Hard-Boiled Wonderland and the End of the World", - "isbn" : "0679743464", - "publisher" : "Vintage", - "date" : "1993-03-02", - "price": 18.90 - }, - { - "author" : "Graham Greene", - "title" : "The Comedians", - "isbn" : "0099478374", - "publisher" : "Vintage Classics", - "date" : "2005-09-21", - "price": 15.74 - } -] -``` - -#### Reading the JSON stream -```c++ -std::ifstream is("book_catalog.json"); - -json_cursor reader(is); - -for (; !reader.done(); reader.next()) -{ - const auto& event = reader.current(); - switch (event.event_type()) - { - case staj_event_type::begin_array: - std::cout << "begin_array\n"; - break; - case staj_event_type::end_array: - std::cout << "end_array\n"; - break; - case staj_event_type::begin_object: - std::cout << "begin_object\n"; - break; - case staj_event_type::end_object: - std::cout << "end_object\n"; - break; - case staj_event_type::name: - // If underlying type is string, can return as string_view - std::cout << "name: " << event.as() << "\n"; - break; - case staj_event_type::string_value: - std::cout << "string_value: " << event.as() << "\n"; - break; - case staj_event_type::null_value: - std::cout << "null_value: " << event.as() << "\n"; - break; - case staj_event_type::bool_value: - std::cout << "bool_value: " << event.as() << "\n"; - break; - case staj_event_type::int64_value: - std::cout << "int64_value: " << event.as() << "\n"; - break; - case staj_event_type::uint64_value: - std::cout << "uint64_value: " << event.as() << "\n"; - break; - case staj_event_type::double_value: - // Return as string, could also use event.as() - std::cout << "double_value: " << event.as() << "\n"; - break; - default: - std::cout << "Unhandled event type\n"; - break; - } -} -``` -Output: -``` -begin_array -begin_object -name: author -string_value: Haruki Murakami -name: title -string_value: Hard-Boiled Wonderland and the End of the World -name: isbn -string_value: 0679743464 -name: publisher -string_value: Vintage -name: date -string_value: 1993-03-02 -name: price -double_value: 18.90 -end_object -begin_object -name: author -string_value: Graham Greene -name: title -string_value: The Comedians -name: isbn -string_value: 0099478374 -name: publisher -string_value: Vintage Classics -name: date -string_value: 2005-09-21 -name: price -double_value: 15.74 -end_object -end_array -``` - -#### Implementing a staj_filter - -```c++ -// A stream filter to filter out all events except name -// and restrict name to "author" - -class author_filter : public staj_filter -{ - bool accept_next_ = false; -public: - bool accept(const staj_event& event, const ser_context&) override - { - if (event.event_type() == staj_event_type::name && - event.as() == "author") - { - accept_next_ = true; - return false; - } - else if (accept_next_) - { - accept_next_ = false; - return true; - } - else - { - accept_next_ = false; - return false; - } - } -}; -``` - -#### Filtering the JSON stream - -```c++ -std::ifstream is("book_catalog.json"); - -author_filter filter; -json_cursor reader(is, filter); - -for (; !reader.done(); reader.next()) -{ - const auto& event = reader.current(); - switch (event.event_type()) - { - case staj_event_type::string_value: - std::cout << event.as() << "\n"; - break; - } -} -``` -Output: -``` -Haruki Murakami -Graham Greene -``` - -See [json_cursor](doc/ref/json_cursor.md) - -
- -### Decode JSON to C++ data structures, encode C++ data structures to JSON - -
- -#### Convert JSON to/from C++ data structures by specializing json_type_traits -jsoncons supports conversion between JSON text and C++ data structures. The functions [decode_json](doc/ref/decode_json.md) -and [encode_json](doc/ref/encode_json.md) convert JSON formatted strings or streams to C++ data structures and back. -Decode and encode work for all C++ classes that have -[json_type_traits](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/json_type_traits.md) -defined. The standard library containers are already supported, and you can specialize `json_type_traits` -for your own types in the `jsoncons` namespace. - -```c++ -#include -#include -#include -#include - -namespace ns { - struct book - { - std::string author; - std::string title; - double price; - }; -} // namespace ns - -namespace jsoncons { - - template - struct json_type_traits - { - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - return j.is_object() && j.contains("author") && - j.contains("title") && j.contains("price"); - } - static ns::book as(const Json& j) - { - ns::book val; - val.author = j.at("author").template as(); - val.title = j.at("title").template as(); - val.price = j.at("price").template as(); - return val; - } - static Json to_json(const ns::book& val, - allocator_type allocator=allocator_type()) - { - Json j(allocator); - j.try_emplace("author", val.author); - j.try_emplace("title", val.title); - j.try_emplace("price", val.price); - return j; - } - }; -} // namespace jsoncons -``` - -To save typing and enhance readability, the jsoncons library defines macros, -so we could have written - -```c++ -JSONCONS_MEMBER_TRAITS_DECL(ns::book, author, title, price) -``` - -which expands to the code above. - -```c++ -using namespace jsoncons; // for convenience - -int main() -{ - const std::string s = R"( - [ - { - "author" : "Haruki Murakami", - "title" : "Kafka on the Shore", - "price" : 25.17 - }, - { - "author" : "Charles Bukowski", - "title" : "Pulp", - "price" : 22.48 - } - ] - )"; - - std::vector book_list = decode_json>(s); - - std::cout << "(1)\n"; - for (const auto& item : book_list) - { - std::cout << item.author << ", " - << item.title << ", " - << item.price << "\n"; - } - - std::cout << "\n(2)\n"; - encode_json(book_list, std::cout, indenting::indent); - std::cout << "\n\n"; -} -``` -Output: -``` -(1) -Haruki Murakami, Kafka on the Shore, 25.17 -Charles Bukowski, Pulp, 22.48 - -(2) -[ - { - "author": "Haruki Murakami", - "price": 25.17, - "title": "Kafka on the Shore" - }, - { - "author": "Charles Bukowski", - "price": 22.48, - "title": "Pulp" - } -] -``` - -
- -#### A simple example using JSONCONS_MEMBER_TRAITS_DECL to generate the json_type_traits - -`JSONCONS_MEMBER_TRAITS_DECL` is a macro that can be used to generate the `json_type_traits` boilerplate -from member data. - -```c++ -#include -#include -#include - -namespace ns { - - struct reputon - { - std::string rater; - std::string assertion; - std::string rated; - double rating; - - friend bool operator==(const reputon& lhs, const reputon& rhs) - { - return lhs.rater == rhs.rater && lhs.assertion == rhs.assertion && - lhs.rated == rhs.rated && lhs.rating == rhs.rating; - } - - friend bool operator!=(const reputon& lhs, const reputon& rhs) - { - return !(lhs == rhs); - }; - }; - - class reputation_object - { - std::string application; - std::vector reputons; - - // Make json_type_traits specializations friends to give accesses to private members - JSONCONS_TYPE_TRAITS_FRIEND; - public: - reputation_object() - { - } - reputation_object(const std::string& application, const std::vector& reputons) - : application(application), reputons(reputons) - {} - - friend bool operator==(const reputation_object& lhs, const reputation_object& rhs) - { - return (lhs.application == rhs.application) && (lhs.reputons == rhs.reputons); - } - - friend bool operator!=(const reputation_object& lhs, const reputation_object& rhs) - { - return !(lhs == rhs); - }; - }; - -} // namespace ns - -// Declare the traits. Specify which data members need to be serialized. -JSONCONS_MEMBER_TRAITS_DECL(ns::reputon, rater, assertion, rated, rating) -JSONCONS_MEMBER_TRAITS_DECL(ns::reputation_object, application, reputons) - -using namespace jsoncons; // for convenience - -int main() -{ - ns::reputation_object val("hiking", { ns::reputon{"HikingAsylum.example.com","strong-hiker","Marilyn C",0.90} }); - - std::string s; - encode_json(val, s, indenting::indent); - std::cout << s << "\n"; - - auto val2 = decode_json(s); - - assert(val2 == val); -} -``` -Output: -``` -{ - "application": "hiking", - "reputons": [ - { - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rater": "HikingAsylum.example.com", - "rating": 0.9 - } - ] -} -``` - -
- -#### A polymorphic example using JSONCONS_GETTER_CTOR_TRAITS_DECL to generate the json_type_traits - -`JSONCONS_GETTER_CTOR_TRAITS_DECL` is a macro that can be used to generate the `json_type_traits` boilerplate -from getter functions and a constructor. - -```c++ -#include -#include -#include -#include - -using namespace jsoncons; - -namespace ns { - -class Employee -{ - std::string firstName_; - std::string lastName_; -public: - Employee(const std::string& firstName, const std::string& lastName) - : firstName_(firstName), lastName_(lastName) - { - } - virtual ~Employee() = default; - - virtual double calculatePay() const = 0; - - const std::string& firstName() const {return firstName_;} - const std::string& lastName() const {return lastName_;} -}; - -class HourlyEmployee : public Employee -{ - double wage_; - unsigned hours_; -public: - HourlyEmployee(const std::string& firstName, const std::string& lastName, - double wage, unsigned hours) - : Employee(firstName, lastName), - wage_(wage), hours_(hours) - { - } - HourlyEmployee(const HourlyEmployee&) = default; - HourlyEmployee(HourlyEmployee&&) = default; - HourlyEmployee& operator=(const HourlyEmployee&) = default; - HourlyEmployee& operator=(HourlyEmployee&&) = default; - - double wage() const {return wage_;} - - unsigned hours() const {return hours_;} - - double calculatePay() const override - { - return wage_*hours_; - } -}; - -class CommissionedEmployee : public Employee -{ - double baseSalary_; - double commission_; - unsigned sales_; -public: - CommissionedEmployee(const std::string& firstName, const std::string& lastName, - double baseSalary, double commission, unsigned sales) - : Employee(firstName, lastName), - baseSalary_(baseSalary), commission_(commission), sales_(sales) - { - } - CommissionedEmployee(const CommissionedEmployee&) = default; - CommissionedEmployee(CommissionedEmployee&&) = default; - CommissionedEmployee& operator=(const CommissionedEmployee&) = default; - CommissionedEmployee& operator=(CommissionedEmployee&&) = default; - - double baseSalary() const - { - return baseSalary_; - } - - double commission() const - { - return commission_; - } - - unsigned sales() const - { - return sales_; - } - - double calculatePay() const override - { - return baseSalary_ + commission_*sales_; - } -}; -} // ns - -JSONCONS_GETTER_CTOR_TRAITS_DECL(ns::HourlyEmployee, firstName, lastName, wage, hours) -JSONCONS_GETTER_CTOR_TRAITS_DECL(ns::CommissionedEmployee, firstName, lastName, baseSalary, commission, sales) - -namespace jsoncons { - -template -struct json_type_traits> -{ - static bool is(const Json& j) noexcept - { - return j.is() || j.is(); - } - static std::shared_ptr as(const Json& j) - { - if (j.at("type").as() == "Hourly") - { - return std::make_shared(j.as()); - } - else if (j.at("type").as() == "Commissioned") - { - return std::make_shared(j.as()); - } - else - { - throw std::runtime_error("Not an employee"); - } - } - static Json to_json(const std::shared_ptr& ptr) - { - if (ns::HourlyEmployee* p = dynamic_cast(ptr.get())) - { - Json j(*p); - j.try_emplace("type","Hourly"); - return j; - } - else if (ns::CommissionedEmployee* p = dynamic_cast(ptr.get())) - { - Json j(*p); - j.try_emplace("type","Commissioned"); - return j; - } - else - { - throw std::runtime_error("Not an employee"); - } - } -}; - -} // jsoncons - -int main() -{ - std::vector> v; - - v.push_back(std::make_shared("John", "Smith", 40.0, 1000)); - v.push_back(std::make_shared("Jane", "Doe", 30000, 0.25, 1000)); - - json j(v); - std::cout << pretty_print(j) << "\n\n"; - - assert(j[0].is()); - assert(!j[0].is()); - assert(!j[1].is()); - assert(j[1].is()); - - - for (size_t i = 0; i < j.size(); ++i) - { - auto p = j[i].as>(); - assert(p->firstName() == v[i]->firstName()); - assert(p->lastName() == v[i]->lastName()); - assert(p->calculatePay() == v[i]->calculatePay()); - } -} -``` -Output: -``` -[ - { - "firstName": "John", - "hours": 1000, - "lastName": "Smith", - "type": "Hourly", - "wage": 40.0 - }, - { - "baseSalary": 30000.0, - "commission": 0.25, - "firstName": "Jane", - "lastName": "Doe", - "sales": 1000, - "type": "Commissioned" - } -] -``` - -
- -#### Convert JSON numbers to/from boost multiprecision numbers - -``` -#include -#include - -namespace jsoncons -{ - template - struct json_type_traits> - { - typedef boost::multiprecision::number multiprecision_type; - - static bool is(const Json& val) noexcept - { - if (!(val.is_string() && val.semantic_tag() == semantic_tag::bigdec)) - { - return false; - } - else - { - return true; - } - } - - static multiprecision_type as(const Json& val) - { - return multiprecision_type(val.template as()); - } - - static Json to_json(multiprecision_type val) - { - return Json(val.str(), semantic_tag::bigdec); - } - }; -} - -int main() -{ - typedef boost::multiprecision::number multiprecision_type; - - std::string s = "[100000000000000000000000000000000.1234]"; - json_options options; - options.lossless_number(true); - json j = json::parse(s, options); - - multiprecision_type x = j[0].as(); - - std::cout << "(1) " << std::setprecision(std::numeric_limits::max_digits10) - << x << "\n"; - - json j2 = json::array{x}; - std::cout << "(2) " << j2[0].as() << "\n"; -} -``` -Output: -``` -(1) 100000000000000000000000000000000.1234 -(2) 100000000000000000000000000000000.1234 -``` - -### Encode - -
- -#### Encode a json value to a string - -``` -std::string s; - -j.dump(s); // compressed - -j.dump(s, indenting::indent); // pretty print -``` - -
- -#### Encode a json value to a stream - -``` -j.dump(std::cout); // compressed - -j.dump(std::cout, indenting::indent); // pretty print -``` -or -``` -std::cout << j << std::endl; // compressed - -std::cout << pretty_print(j) << std::endl; // pretty print -``` - -
- -#### Escape all non-ascii characters - -``` -json_options options; -options.escape_all_non_ascii(true); - -j.dump(std::cout, options); // compressed - -j.dump(std::cout, options, indenting::indent); // pretty print -``` -or -``` -std::cout << print(j, options) << std::endl; // compressed - -std::cout << pretty_print(j, options) << std::endl; // pretty print -``` - -
- -#### Replace the representation of NaN, Inf and -Inf when serializing. And when reading in again. - -Set the serializing options for `nan` and `inf` to distinct string values. - -```c++ -json j; -j["field1"] = std::sqrt(-1.0); -j["field2"] = 1.79e308 * 1000; -j["field3"] = -1.79e308 * 1000; - -json_options options; -options.nan_to_str("NaN") - .inf_to_str("Inf"); - -std::ostringstream os; -os << pretty_print(j, options); - -std::cout << "(1)\n" << os.str() << std::endl; - -json j2 = json::parse(os.str(),options); - -std::cout << "\n(2) " << j2["field1"].as() << std::endl; -std::cout << "(3) " << j2["field2"].as() << std::endl; -std::cout << "(4) " << j2["field3"].as() << std::endl; - -std::cout << "\n(5)\n" << pretty_print(j2,options) << std::endl; -``` - -Output: -```json -(1) -{ - "field1": "NaN", - "field2": "Inf", - "field3": "-Inf" -} - -(2) nan -(3) inf -(4) -inf - -(5) -{ - "field1": "NaN", - "field2": "Inf", - "field3": "-Inf" -} -``` - -### Construct - -
- -#### Construct a json object - -Start with an empty json object and insert some name-value pairs, - -```c++ -json image_sizing; -image_sizing.insert_or_assign("Resize To Fit",true); // a boolean -image_sizing.insert_or_assign("Resize Unit", "pixels"); // a string -image_sizing.insert_or_assign("Resize What", "long_edge"); // a string -image_sizing.insert_or_assign("Dimension 1",9.84); // a double -image_sizing.insert_or_assign("Dimension 2",json::null()); // a null value -``` - -or use an object initializer-list, - -```c++ -json file_settings = json::object{ - {"Image Format", "JPEG"}, - {"Color Space", "sRGB"}, - {"Limit File Size", true}, - {"Limit File Size To", 10000} -}; -``` - -
- -#### Construct a json array - -```c++ -json color_spaces = json::array(); // an empty array -color_spaces.push_back("sRGB"); -color_spaces.push_back("AdobeRGB"); -color_spaces.push_back("ProPhoto RGB"); -``` - -or use an array initializer-list, -```c++ -json image_formats = json::array{"JPEG","PSD","TIFF","DNG"}; -``` - -
- -#### Insert a new value in an array at a specific position - -```c++ -json cities = json::array(); // an empty array -cities.push_back("Toronto"); -cities.push_back("Vancouver"); -// Insert "Montreal" at beginning of array -cities.insert(cities.array_range().begin(),"Montreal"); - -std::cout << cities << std::endl; -``` -Output: -``` -["Montreal","Toronto","Vancouver"] -``` - -
- -#### Create arrays of arrays of arrays of ... - -Like this: - -```c++ -json j = json::make_array<3>(4, 3, 2, 0.0); -double val = 1.0; -for (size_t i = 0; i < a.size(); ++i) -{ - for (size_t j = 0; j < j[i].size(); ++j) - { - for (size_t k = 0; k < j[i][j].size(); ++k) - { - j[i][j][k] = val; - val += 1.0; - } - } -} -std::cout << pretty_print(j) << std::endl; -``` -Output: -```json -[ - [ - [1.0,2.0], - [3.0,4.0], - [5.0,6.0] - ], - [ - [7.0,8.0], - [9.0,10.0], - [11.0,12.0] - ], - [ - [13.0,14.0], - [15.0,16.0], - [17.0,18.0] - ], - [ - [19.0,20.0], - [21.0,22.0], - [23.0,24.0] - ] -] -``` - -
- -#### Merge two json objects - -[json::merge](ref/json/merge.md) inserts another json object's key-value pairs into a json object, -unless they already exist with an equivalent key. - -[json::merge_or_update](ref/json/merge_or_update.md) inserts another json object's key-value pairs -into a json object, or assigns them if they already exist. - -The `merge` and `merge_or_update` functions perform only a one-level-deep shallow merge, -not a deep merge of nested objects. - -```c++ -json another = json::parse(R"( -{ - "a" : "2", - "c" : [4,5,6] -} -)"); - -json j = json::parse(R"( -{ - "a" : "1", - "b" : [1,2,3] -} -)"); - -j.merge(std::move(another)); -std::cout << pretty_print(j) << std::endl; -``` -Output: -```json -{ - "a": "1", - "b": [1,2,3], - "c": [4,5,6] -} -``` - -
- -#### Construct a json byte string - -```c++ -#include - -namespace jc=jsoncons; - -int main() -{ - byte_string bs = {'H','e','l','l','o'}; - - // default suggested encoding (base64url) - json j1(bs); - std::cout << "(1) "<< j1 << "\n\n"; - - // base64 suggested encoding - json j2(bs, semantic_tag::base64); - std::cout << "(2) "<< j2 << "\n\n"; - - // base16 suggested encoding - json j3(bs, semantic_tag::base16); - std::cout << "(3) "<< j3 << "\n\n"; -} -``` - -Output: -``` -(1) "SGVsbG8" - -(2) "SGVsbG8=" - -(3) "48656C6C6F" -``` - -### Iterate - -
- -#### Iterate over a json array - -```c++ -json j = json::array{1,2,3,4}; - -for (auto val : j.array_range()) -{ - std::cout << val << std::endl; -} -``` - -
- -#### Iterate over a json object - -```c++ -json j = json::object{ - {"author", "Haruki Murakami"}, - {"title", "Kafka on the Shore"}, - {"price", 25.17} -}; - -for (const auto& member : j.object_range()) -{ - std::cout << member.key() << "=" - << member.value() << std::endl; -} -``` - -### Access - -
- -#### Use string_view to access the actual memory that's being used to hold a string - -You can use `j.as()`, e.g. -```c++ -json j = json::parse("\"Hello World\""); -auto sv = j.as(); -``` -`jsoncons::string_view` supports the member functions of `std::string_view`, including `data()` and `size()`. - -If your compiler supports `std::string_view`, you can also use `j.as()`. - -
- -#### Given a string in a json object that represents a decimal number, assign it to a double - -```c++ -json j = json::object{ - {"price", "25.17"} -}; - -double price = j["price"].as(); -``` - -
- -#### Retrieve a big integer that's been parsed as a string - -If an integer exceeds the range of an `int64_t` or `uint64_t`, jsoncons parses it as a string -with semantic tagging `bigint`. - -```c++ -#include -#include -#include - -using jsoncons::json; - -int main() -{ - std::string input = "-18446744073709551617"; - - json j = json::parse(input); - - // Access as string - std::string s = j.as(); - std::cout << "(1) " << s << "\n\n"; - - // Access as double - double d = j.as(); - std::cout << "(2) " << std::setprecision(17) << d << "\n\n"; - - // Access as jsoncons::bignum - jsoncons::bignum bn = j.as(); - std::cout << "(3) " << bn << "\n\n"; - - // If your compiler supports extended integral types for which std::numeric_limits is specialized -#if (defined(__GNUC__) || defined(__clang__)) && (!defined(__STRICT_ANSI__) && defined(_GLIBCXX_USE_INT128)) - __int128 i = j.as<__int128>(); - std::cout << "(4) " << i << "\n\n"; -#endif -} -``` -Output: -``` -(1) -18446744073709551617 - -(2) -1.8446744073709552e+19 - -(3) -18446744073709551617 - -(4) -18446744073709551617 -``` - -
- -#### Look up a key, if found, return the value converted to type T, otherwise, return a default value of type T. - -```c++ -json j = json::object{ - {"price", "25.17"} -}; - -double price = j.get_with_default("price", 25.00); // returns 25.17 - -double sale_price = j.get_with_default("sale_price", 22.0); // returns 22.0 -``` - -
- -#### Retrieve a value in a hierarchy of JSON objects - -```c++ -#include -#include -#include - -int main() -{ - json j = json::parse(R"( - { - "application": "hiking", - "reputons": [ - { - "rater": "HikingAsylum.example.com", - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rating": 0.90 - } - ] - } - )"); - - // Using index or `at` accessors - std::string result1 = j["reputons"][0]["rated"].as(); - std::cout << "(1) " << result1 << std::endl; - std::string result2 = j.at("reputons").at(0).at("rated").as(); - std::cout << "(2) " << result2 << std::endl; - - // Using JSON Pointer - std::string result3 = jsonpointer::get(j, "/reputons/0/rated").as(); - std::cout << "(3) " << result3 << std::endl; - - // Using JSONPath - json result4 = jsonpath::json_query(j, "$.reputons.0.rated"); - if (result4.size() > 0) - { - std::cout << "(4) " << result4[0].as() << std::endl; - } - json result5 = jsonpath::json_query(j, "$..0.rated"); - if (result5.size() > 0) - { - std::cout << "(5) " << result5[0].as() << std::endl; - } -} -``` - -
- -#### Retrieve a json value as a byte string - -```c++ -#include - -namespace jc=jsoncons; - -int main() -{ - json j; - j["ByteString"] = byte_string({'H','e','l','l','o'}); - j["EncodedByteString"] = json("SGVsbG8=", semantic_tag::base64); - - std::cout << "(1)\n"; - std::cout << pretty_print(j) << "\n\n"; - - // Retrieve a byte string as a jsoncons::byte_string - byte_string bs1 = j["ByteString"].as(); - std::cout << "(2) " << bs1 << "\n\n"; - - // or alternatively as a std::vector - std::vector v = j["ByteString"].as>(); - - // Retrieve a byte string from a text string containing base64 character values - byte_string bs2 = j["EncodedByteString"].as(); - std::cout << "(3) " << bs2 << "\n\n"; - - // Retrieve a byte string view to access the memory that's holding the byte string - byte_string_view bsv3 = j["ByteString"].as(); - std::cout << "(4) " << bsv3 << "\n\n"; - - // Can't retrieve a byte string view of a text string - try - { - byte_string_view bsv4 = j["EncodedByteString"].as(); - } - catch (const std::exception& e) - { - std::cout << "(5) "<< e.what() << "\n\n"; - } -} -``` -Output: -``` -(1) -{ - "ByteString": "SGVsbG8", - "EncodedByteString": "SGVsbG8=" -} - -(2) 48656c6c6f - -(3) 48656c6c6f - -(4) 48656c6c6f - -(5) Not a byte string -``` - -### Search and Replace - -
- -#### Search for and repace an object member key - -You can rename object members with the built in filter [rename_object_member_filter](ref/rename_object_member_filter.md) - -```c++ -#include -#include -#include - -using namespace jsoncons; - -int main() -{ - std::string s = R"({"first":1,"second":2,"fourth":3,"fifth":4})"; - - json_encoder encoder(std::cout); - - // Filters can be chained - rename_object_member_filter filter2("fifth", "fourth", encoder); - rename_object_member_filter filter1("fourth", "third", filter2); - - // A filter can be passed to any function that takes a json_content_handler ... - std::cout << "(1) "; - std::istringstream is(s); - json_reader reader(is, filter1); - reader.read(); - std::cout << std::endl; - - // or - std::cout << "(2) "; - ojson j = ojson::parse(s); - j.dump(filter1); - std::cout << std::endl; -} -``` -Output: -```json -(1) {"first":1,"second":2,"third":3,"fourth":4} -(2) {"first":1,"second":2,"third":3,"fourth":4} -``` - -
- -#### Search for and replace a value - -You can use [json_replace](ref/jsonpath/json_replace.md) in the `jsonpath` extension - -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - json j = json::parse(R"( - { "store": { - "book": [ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - }, - { "category": "fiction", - "author": "Herman Melville", - "title": "Moby Dick", - "isbn": "0-553-21311-3", - "price": 8.99 - } - ] - } - } - )"); - - // Change the price of "Moby Dick" from $8.99 to $10 - jsonpath::json_replace(j,"$.store.book[?(@.isbn == '0-553-21311-3')].price",10.0); - std::cout << pretty_print(booklist) << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Home.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Home.md deleted file mode 100644 index 87e082e5b5..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Home.md +++ /dev/null @@ -1,64 +0,0 @@ -All core jsoncons classes and functions are in namespace `jsoncons`. - -#### Unpacked Representation - -[json](ref/json.md) -[json_parser](ref/json_parser.md) -[json_reader](ref/json_reader.md) -[json_decoder](ref/json_decoder.md) - -[ojson](ref/ojson.md) - -[wjson](ref/wjson.md) -[wjson_reader](ref/wjson_reader.md) - -[wojson](ref/wojson.md) - -#### C++/JSON Conversion - -[encode_json](ref/encode_json.md) -[decode_json](ref/decode_json.md) - -#### Streaming - -[json_content_handler](ref/json_content_handler.md) -[json_encoder](ref/json_encoder.md) -[json_options](ref/json_options.md) - -[wjson_encoder](ref/wjson_encoder.md) -[wjson_options](ref/wjson_options.md) - -[json_filter](ref/json_filter.md) -[rename_object_member_filter](ref/rename_object_member_filter.md) - -[json_cursor](ref/json_cursor.md) -[staj_reader](ref/staj_reader.md) -[staj_object_iterator](ref/staj_object_iterator.md) -[staj_array_iterator](ref/staj_array_iterator.md) - -### Extensions - -#### [jsonpointer](ref/jsonpointer/jsonpointer.md) - -#### [jsonpatch](ref/jsonpatch/jsonpatch.md) - -#### [jsonpath](ref/jsonpath/jsonpath.md) - -#### [cbor](ref/cbor/cbor.md) - -#### [msgpack](ref/msgpack/msgpack.md) - -#### [ubjson](ref/ubjson/ubjson.md) - -#### [msgpack](ref/msgpack/msgpack.md) - -#### [bson](ref/bson/bson.md) - -### Tutorials - -[Basics](Tutorials/Basics.md) - -[Custom Allocators](Tutorials/Custom%20Allocators.md) - -[Unicode support](Tutorials/Unicode%20support.md) - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Pages/index.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Pages/index.md deleted file mode 100644 index f5d894358d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Pages/index.md +++ /dev/null @@ -1,812 +0,0 @@ -# jsoncons: a C++ library for json construction - -[Preliminaries](#A1) - -[Reading JSON text from a file](#A2) - -[Constructing json values in C++](#A3) - -[Conversion between JSON and C++ data structures](#A4) - -[Converting CSV files to json](#A5 ) - -[Pretty print](#A6) - -[Filters](#A7) - -[JSONPath](#A8) - -[About jsoncons::json](#A9) - -[Wide character support](#A10) - -[ojson and wojson](#A11) - -
-### Preliminaries - -jsoncons is a C++, header-only library for constructing [JSON](http://www.json.org) and JSON-like -data formats such as [CBOR](http://cbor.io/). It supports - -- Parsing JSON-like text or binary formats into a tree model - that defines an interface for accessing and modifying that data (covers bignum and byte string values.) - -- Serializing the tree model into different JSON-like text or binary formats. - -- Converting from JSON-like text or binary formats to C++ data structures and back via [json_type_traits](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/json_type_traits.md). - -- Streaming JSON read and write events, somewhat analogously to SAX (push parsing) and StAX (pull parsing) in the XML world. - -The jsoncons library is header-only: it consists solely of header files containing templates and inline functions, and requires no separately-compiled library binaries when linking. It has no dependence on other libraries. - -To install the librray, download the [latest release](https://github.com/danielaparker/jsoncons/releases) and unpack the zip file. Copy the directory `include/jsoncons` to your `include` directory. If you wish to use extensions, copy `include/jsoncons_ext` as well. - -Or, download the latest code on [master](https://github.com/danielaparker/jsoncons/archive/master.zip). - -Compared to other JSON libraries, jsoncons has been designed to handle very large JSON texts. At its heart are -SAX style parsers and serializers. Its [json parser](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/json_parser.md) is an -incremental parser that can be fed its input in chunks, and does not require an entire file to be loaded in memory at one time. -Its tree model is more compact than most, and can be made more compact still with a user-supplied -allocator. It also supports memory efficient parsing of very large JSON texts with a [pull parser](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/json_cursor.md), -built on top of its incremental parser. - -The [jsoncons data model](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/data-model.md) supports the familiar JSON types - nulls, -booleans, numbers, strings, arrays, objects - plus byte strings. In addition, jsoncons -supports semantic tagging of date-time values, timestamp values, big integers, -big decimals, bigfloats and binary encodings. This allows it to preserve these type semantics when parsing -JSON-like data formats such as CBOR that have them. - -The jsoncons classes and functions are in namespace `jsoncons`. You need to include the header file -```c++ -#include -``` -and, for convenience, -```c++ -using jsoncons::json; -``` - -
-### Reading JSON text from a file - -Example file (`books.json`): -```c++ -[ - { - "title" : "Kafka on the Shore", - "author" : "Haruki Murakami", - "price" : 25.17 - }, - { - "title" : "Women: A Novel", - "author" : "Charles Bukowski", - "price" : 12.0 - }, - { - "title" : "Cutter's Way", - "author" : "Ivan Passer" - } -] -``` -It consists of an array of book elements, each element is an object with members title, author, and price. - -Read the JSON text into a `json` value, -```c++ -std::ifstream is("books.json"); -json books = json::parse(is); -``` -Loop through the book array elements, using a range-based for loop -```c++ -for (const auto& book : books.array_range()) -{ - std::string author = book["author"].as(); - std::string title = book["title"].as(); - std::cout << author << ", " << title << std::endl; -} -``` -or begin-end iterators -```c++ -for (auto it = books.array_range().begin(); - it != books.array_range().end(); - ++it) -{ - std::string author = (*it)["author"].as(); - std::string title = (*it)["title"].as(); - std::cout << author << ", " << title << std::endl; -} -``` -or a traditional for loop -```c++ -for (size_t i = 0; i < books.size(); ++i) -{ - json& book = books[i]; - std::string author = book["author"].as(); - std::string title = book["title"].as(); - std::cout << author << ", " << title << std::endl; -} -``` -Output: -``` -Haruki Murakami, Kafka on the Shore -Charles Bukowski, Women: A Novel -Ivan Passer, Cutter's Way -``` - -Loop through the members of the third book element, using a range-based for loop - -```c++ -for (const auto& member : books[2].object_range()) -{ - std::cout << member.key() << "=" - << member.value() << std::endl; -} -``` - -or begin-end iterators: - -```c++ -for (auto it = books[2].object_range().begin(); - it != books[2].object_range().end(); - ++it) -{ - std::cout << (*it).key() << "=" - << (*it).value() << std::endl; -} -``` -Output: -``` -author=Ivan Passer -title=Cutter's Way -``` - -Note that the third book, Cutter's Way, is missing a price. - -You have a choice of object member accessors: - -- `book["price"]` will throw `std::out_of_range` if there is no price -- `book.get_with_default("price",std::string("n/a"))` will return the price converted to the - default's data type, `std::string`, or `"n/a"` if there is no price. - -So if you want to show "n/a" for the missing price, you can use this accessor -```c++ -std::string price = book.get_with_default("price","n/a"); -``` -Or you can check if book has a member "price" with the method `contains`, and output accordingly, -```c++ -if (book.contains("price")) -{ - double price = book["price"].as(); - std::cout << price; -} -else -{ - std::cout << "n/a"; -} -``` -
-### Constructing json values in C++ - -The default `json` constructor produces an empty json object. For example -```c++ -json image_sizing; -std::cout << image_sizing << std::endl; -``` -produces -```json -{} -``` -To construct a json object with members, take an empty json object and set some name-value pairs -```c++ -image_sizing.insert_or_assign("Resize To Fit",true); // a boolean -image_sizing.insert_or_assign("Resize Unit", "pixels"); // a string -image_sizing.insert_or_assign("Resize What", "long_edge"); // a string -image_sizing.insert_or_assign("Dimension 1",9.84); // a double -image_sizing.insert_or_assign("Dimension 2",json::null()); // a null value -``` - -Or, use an object initializer-list: -```c++ -json file_settings = json::object{ - {"Image Format", "JPEG"}, - {"Color Space", "sRGB"}, - {"Limit File Size", true}, - {"Limit File Size To", 10000} -}; -``` - -To construct a json array, initialize with the array type -```c++ -json color_spaces = json::array(); -``` -and add some elements -```c++ -color_spaces.push_back("sRGB"); -color_spaces.push_back("AdobeRGB"); -color_spaces.push_back("ProPhoto RGB"); -``` - -Or, use an array initializer-list: -```c++ -json image_formats = json::array{"JPEG","PSD","TIFF","DNG"}; -``` - -The `operator[]` provides another way for setting name-value pairs. -```c++ -json file_export; -file_export["File Format Options"]["Color Spaces"] = - std::move(color_spaces); -file_export["File Format Options"]["Image Formats"] = - std::move(image_formats); -file_export["File Settings"] = std::move(file_settings); -file_export["Image Sizing"] = std::move(image_sizing); -``` -Note that if `file_export["File Format Options"]` doesn't exist, -```c++ -file_export["File Format Options"]["Color Spaces"] = - std::move(color_spaces) -``` -creates `"File Format Options"` as an object and puts `"Color Spaces"` in it. - -Serializing -```c++ -std::cout << pretty_print(file_export) << std::endl; -``` -produces -```json -{ - "File Format Options": { - "Color Spaces": ["sRGB","AdobeRGB","ProPhoto RGB"], - "Image Formats": ["JPEG","PSD","TIFF","DNG"] - }, - "File Settings": { - "Color Space": "sRGB", - "Image Format": "JPEG", - "Limit File Size": true, - "Limit File Size To": 10000 - }, - "Image Sizing": { - "Dimension 1": 9.84, - "Dimension 2": null, - "Resize To Fit": true, - "Resize Unit": "pixels", - "Resize What": "long_edge" - } -} -``` -
-### Conversion between JSON and C++ data structures - -jsoncons supports conversion between JSON text and C++ data structures. The functions [decode_json](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/decode_json.md) -and [encode_json](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/encode_json.md) convert JSON formatted strings or streams to C++ data structures and back. -Decode and encode work for all C++ classes that have -[json_type_traits](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/json_type_traits.md) -defined. The standard library containers are already supported, and you can specialize `json_type_traits` -for your own types in the `jsoncons` namespace. - -`JSONCONS_MEMBER_TRAITS_DECL` is a macro that simplifies the creation of the necessary boilerplate -for your own types. - -```c++ -#include -#include -#include - -using namespace jsoncons; - -namespace ns { - - struct reputon - { - std::string rater; - std::string assertion; - std::string rated; - double rating; - - friend bool operator==(const reputon& lhs, const reputon& rhs) - { - return lhs.rater == rhs.rater && - lhs.assertion == rhs.assertion && - lhs.rated == rhs.rated && - lhs.rating == rhs.rating; - } - - friend bool operator!=(const reputon& lhs, const reputon& rhs) - { - return !(lhs == rhs); - }; - }; - - class reputation_object - { - std::string application; - std::vector reputons; - - // Make json_type_traits specializations friends to give accesses to private members - JSONCONS_TYPE_TRAITS_FRIEND; - public: - reputation_object() - { - } - reputation_object(const std::string& application, const std::vector& reputons) - : application(application), reputons(reputons) - {} - - friend bool operator==(const reputation_object& lhs, const reputation_object& rhs) - { - if (lhs.application != rhs.application) - { - return false; - } - if (lhs.reputons.size() != rhs.reputons.size()) - { - return false; - } - for (size_t i = 0; i < lhs.reputons.size(); ++i) - { - if (lhs.reputons[i] != rhs.reputons[i]) - { - return false; - } - } - return true; - } - - friend bool operator!=(const reputation_object& lhs, const reputation_object& rhs) - { - return !(lhs == rhs); - }; - }; - -} // namespace ns - -// Declare the traits. Specify which data members need to be serialized. -JSONCONS_MEMBER_TRAITS_DECL(ns::reputon, rater, assertion, rated, rating) -JSONCONS_MEMBER_TRAITS_DECL(ns::reputation_object, application, reputons) - -int main() -{ - ns::reputation_object val("hiking", { ns::reputon{"HikingAsylum.example.com","strong-hiker","Marilyn C",0.90} }); - - std::string s; - encode_json(val, s, indenting::indent); - std::cout << s << "\n"; - - auto val2 = decode_json(s); - - assert(val2 == val); -} -``` -Output: -``` -{ - "application": "hiking", - "reputons": [ - { - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rater": "HikingAsylum.example.com", - "rating": 0.9 - } - ] -} -``` - -See [examples](https://github.com/danielaparker/jsoncons/blob/master/doc/Examples.md#G1) - -
-### Converting CSV files to json - -Example CSV file (tasks.csv): - -``` -project_id, task_name, task_start, task_finish -4001,task1,01/01/2003,01/31/2003 -4001,task2,02/01/2003,02/28/2003 -4001,task3,03/01/2003,03/31/2003 -4002,task1,04/01/2003,04/30/2003 -4002,task2,05/01/2003, -``` - -You can read the `CSV` file into a `json` value with the `decode_csv` function. - -```c++ -#include -#include -#include -#include - -using namespace jsoncons; - -int main() -{ - std::ifstream is("input/tasks.csv"); - - csv::csv_options options; - options.assume_header(true) - .trim(true) - .ignore_empty_values(true) - .column_types("integer,string,string,string"); - ojson tasks = csv::decode_csv(is, options); - - std::cout << "(1)\n" << pretty_print(tasks) << "\n\n"; - - std::cout << "(2)\n"; - csv::encode_csv(tasks, std::cout); -} -``` -Output: -```json -(1) -[ - { - "project_id": 4001, - "task_name": "task1", - "task_start": "01/01/2003", - "task_finish": "01/31/2003" - }, - { - "project_id": 4001, - "task_name": "task2", - "task_start": "02/01/2003", - "task_finish": "02/28/2003" - }, - { - "project_id": 4001, - "task_name": "task3", - "task_start": "03/01/2003", - "task_finish": "03/31/2003" - }, - { - "project_id": 4002, - "task_name": "task1", - "task_start": "04/01/2003", - "task_finish": "04/30/2003" - }, - { - "project_id": 4002, - "task_name": "task2", - "task_start": "05/01/2003" - } -] -``` -There are a few things to note about the effect of the parameter settings. -- `assume_header` `true` tells the csv parser to parse the first line of the file for column names, which become object member names. -- `trim` `true` tells the parser to trim leading and trailing whitespace, in particular, to remove the leading whitespace in the column names. -- `ignore_empty_values` `true` causes the empty last value in the `task_finish` column to be omitted. -- The `column_types` setting specifies that column one ("project_id") contains integers and the remaining columns strings. - -
-### Pretty print - -The `pretty_print` function applies stylistic formatting to JSON text. For example - -```c++ - json val; - - val["verts"] = json::array{1, 2, 3}; - val["normals"] = json::array{1, 0, 1}; - val["uvs"] = json::array{0, 0, 1, 1}; - - std::cout << pretty_print(val) << std::endl; -``` -produces - -```json -{ - "normals": [1,0,1], - "uvs": [0,0,1,1], - "verts": [1,2,3] -} -``` -By default, within objects, arrays of scalar values are displayed on the same line. - -The `pretty_print` function takes an optional second parameter, [json_options](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/json_options.md), that allows custom formatting of output. -To display the array scalar values on a new line, set the `object_array_line_splits` property to `line_split_kind::new_line`. The code -```c++ -json_options options; -format.object_array_line_splits(line_split_kind::new_line); -std::cout << pretty_print(val,options) << std::endl; -``` -produces -```json -{ - "normals": [ - 1,0,1 - ], - "uvs": [ - 0,0,1,1 - ], - "verts": [ - 1,2,3 - ] -} -``` -To display the elements of array values on multiple lines, set the `object_array_line_splits` property to `line_split_kind::multi_line`. The code -```c++ -json_options options; -format.object_array_line_splits(line_split_kind::multi_line); -std::cout << pretty_print(val,options) << std::endl; -``` -produces -```json -{ - "normals": [ - 1, - 0, - 1 - ], - "uvs": [ - 0, - 0, - 1, - 1 - ], - "verts": [ - 1, - 2, - 3 - ] -} -``` -
-### Filters - -You can rename object member names with the built in filter [rename_object_member_filter](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/rename_object_member_filter.md) - -```c++ -#include -#include -#include - -using namespace jsoncons; - -int main() -{ - std::string s = R"({"first":1,"second":2,"fourth":3,"fifth":4})"; - - json_encoder encoder(std::cout); - - // Filters can be chained - rename_object_member_filter filter2("fifth", "fourth", encoder); - rename_object_member_filter filter1("fourth", "third", filter2); - - // A filter can be passed to any function that takes - // a json_content_handler ... - std::cout << "(1) "; - std::istringstream is(s); - json_reader reader(is, filter1); - reader.read(); - std::cout << std::endl; - - // or a json_content_handler - std::cout << "(2) "; - ojson j = ojson::parse(s); - j.dump(filter1); - std::cout << std::endl; -} -``` -Output: -```json -(1) {"first":1,"second":2,"third":3,"fourth":4} -(2) {"first":1,"second":2,"third":3,"fourth":4} -``` -Or define and use your own filters. See [json_filter](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/json_filter.md) for details. -
-### JSONPath - -[Stefan Goessner's JSONPath](http://goessner.net/articles/JsonPath/) is an XPATH inspired query language for selecting parts of a JSON structure. - -Example JSON file (booklist.json): -```json -{ "store": { - "book": [ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - }, - { "category": "fiction", - "author": "Herman Melville", - "title": "Moby Dick", - "isbn": "0-553-21311-3", - "price": 8.99 - }, - { "category": "fiction", - "author": "J. R. R. Tolkien", - "title": "The Lord of the Rings", - "isbn": "0-395-19395-8", - "price": 22.99 - } - ] - } -} -``` -JSONPath examples: -```c++ -#include - -using jsoncons::jsonpath::json_query; - -std::ifstream is("./input/booklist.json"); -json booklist = json::parse(is); - -// The authors of books that are cheaper than $10 -json result1 = json_query(booklist, "$.store.book[?(@.price < 10)].author"); -std::cout << "(1) " << result1 << std::endl; - -// The number of books -json result2 = json_query(booklist, "$..book.length"); -std::cout << "(2) " << result2 << std::endl; - -// The third book -json result3 = json_query(booklist, "$..book[2]"); -std::cout << "(3)\n" << pretty_print(result3) << std::endl; - -// All books whose author's name starts with Evelyn -json result4 = json_query(booklist, "$.store.book[?(@.author =~ /Evelyn.*?/)]"); -std::cout << "(4)\n" << pretty_print(result4) << std::endl; - -// The titles of all books that have isbn number -json result5 = json_query(booklist, "$..book[?(@.isbn)].title"); -std::cout << "(5) " << result5 << std::endl; - -// All authors and titles of books -json result6 = json_query(booklist, "$['store']['book']..['author','title']"); -std::cout << "(6)\n" << pretty_print(result6) << std::endl; -``` -Output: -```json -(1) ["Nigel Rees","Herman Melville"] -(2) [4] -(3) -[ - { - "author": "Herman Melville", - "category": "fiction", - "isbn": "0-553-21311-3", - "price": 8.99, - "title": "Moby Dick" - } -] -(4) -[ - { - "author": "Evelyn Waugh", - "category": "fiction", - "price": 12.99, - "title": "Sword of Honour" - } -] -(5) ["Moby Dick","The Lord of the Rings"] -(6) -[ - "Nigel Rees", - "Sayings of the Century", - "Evelyn Waugh", - "Sword of Honour", - "Herman Melville", - "Moby Dick", - "J. R. R. Tolkien", - "The Lord of the Rings" -] -``` -
-### About jsoncons::json - -The [json](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/json.md) class is an instantiation of the `basic_json` class template that uses `char` as the character type -and sorts object members in alphabetically order. -```c++ -typedef basic_json> json; -``` -If you prefer to retain the original insertion order, use [ojson](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/ojson.md) instead. - -The library includes an instantiation for wide characters as well, [wjson](https://github.com/danielaparker/jsoncons/blob/master/ref/doc/wjson.md) -```c++ -typedef basic_json> wjson; -``` -If you prefer to retain the original insertion order, use [wojson](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/wojson.md) instead. - -Note that the allocator type allows you to supply a custom allocator. For example, you can use the boost [fast_pool_allocator](http://www.boost.org/doc/libs/1_60_0/libs/pool/doc/html/boost/fast_pool_allocator.html): -```c++ -#include -#include - -typedef jsoncons::basic_json> myjson; - -myjson o; - -o.insert_or_assign("FirstName","Joe"); -o.insert_or_assign("LastName","Smith"); -``` -This results in a json value being constucted with all memory being allocated from the boost memory pool. (In this particular case there is no improvement in performance over `std::allocator`.) - -Note that the underlying memory pool used by the `boost::fast_pool_allocator` is never freed. - -
-### Wide character support - -jsoncons supports wide character strings and streams with `wjson` and `wjson_reader`. It supports `UTF16` encoding if `wchar_t` has size 2 (Windows) and `UTF32` encoding if `wchar_t` has size 4. You can construct a `wjson` value in exactly the same way as a `json` value, for instance: -```c++ -using jsoncons::wjson; - -wjson root; -root[L"field1"] = L"test"; -root[L"field2"] = 3.9; -root[L"field3"] = true; - -std::wcout << root << L"\n"; -``` -which prints -```c++ -{"field1":"test","field2":3.9,"field3":true} -``` -
-### ojson and wojson - -The [ojson](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/ojson.md) ([wojson](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/wojson.md)) class is an instantiation of the `basic_json` class template that uses `char` (`wchar_t`) as the character type and keeps object members in their original order. -```c++ -ojson o = ojson::parse(R"( -{ - "street_number" : "100", - "street_name" : "Queen St W", - "city" : "Toronto", - "country" : "Canada" -} -)"); - -std::cout << pretty_print(o) << std::endl; -``` -Output: -```json -{ - "street_number": "100", - "street_name": "Queen St W", - "city": "Toronto", - "country": "Canada" -} -``` -Insert "postal_code" at end -```c++ -o.insert_or_assign("postal_code", "M5H 2N2"); - -std::cout << pretty_print(o) << std::endl; -``` -Output: -```json -{ - "street_number": "100", - "street_name": "Queen St W", - "city": "Toronto", - "country": "Canada", - "postal_code": "M5H 2N2" -} -``` -Insert "province" before "country" -```c++ -auto it = o.find("country"); -o.insert_or_assign(it,"province","Ontario"); - -std::cout << pretty_print(o) << std::endl; -``` -Output: -```json -{ - "street_number": "100", - "street_name": "Queen St W", - "city": "Toronto", - "province": "Ontario", - "country": "Canada", - "postal_code": "M5H 2N2" -} -``` - -For more information, consult the latest [examples](https://github.com/danielaparker/jsoncons/blob/master/doc/Examples.md), [documentation](https://github.com/danielaparker/jsoncons/blob/master/doc/Home.md) and [roadmap](https://github.com/danielaparker/jsoncons/blob/master/Roadmap.md). - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Tutorials/Basics.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Tutorials/Basics.md deleted file mode 100644 index d0d31ddfc5..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Tutorials/Basics.md +++ /dev/null @@ -1,448 +0,0 @@ -## Examples - -The examples below illustrate the use of the [json](../ref/json.md) class and [json_query](../ref/jsonpath/json_query.md) function. - -### json construction - -```c++ -#include - -// For convenience -using jsoncons::json; - -// Construct a book object -json book1; - -book1["category"] = "Fiction"; -book1["title"] = "A Wild Sheep Chase: A Novel"; -book1["author"] = "Haruki Murakami"; -book1["date"] = "2002-04-09"; -book1["price"] = 9.01; -book1["isbn"] = "037571894X"; - -// Construct another using the insert_or_assign function -json book2; - -book2.insert_or_assign("category", "History"); -book2.insert_or_assign("title", "Charlie Wilson's War"); -book2.insert_or_assign("author", "George Crile"); -book2.insert_or_assign("date", "2007-11-06"); -book2.insert_or_assign("price", 10.50); -book2.insert_or_assign("isbn", "0802143415"); - -// Use insert_or_assign again, but more efficiently -json book3; - -// Reserve memory, to avoid reallocations -book3.reserve(6); - -// Insert in name alphabetical order -// Give insert_or_assign a hint where to insert the next member -auto hint = book3.insert_or_assign(book3.object_range().begin(),"author", "Haruki Murakami"); -hint = book3.insert_or_assign(hint, "category", "Fiction"); -hint = book3.insert_or_assign(hint, "date", "2006-01-03"); -hint = book3.insert_or_assign(hint, "isbn", "1400079276"); -hint = book3.insert_or_assign(hint, "price", 13.45); -hint = book3.insert_or_assign(hint, "title", "Kafka on the Shore"); - -// Construct a fourth from a string -json book4 = json::parse(R"( -{ - "category" : "Fiction", - "title" : "Pulp", - "author" : "Charles Bukowski", - "date" : "2004-07-08", - "price" : 22.48, - "isbn" : "1852272007" -} -)"); - -// Construct a booklist array - -json booklist = json::array(); - -// For efficiency, reserve memory, to avoid reallocations -booklist.reserve(4); - -// For efficency, tell jsoncons to move the contents -// of the four book objects into the array -booklist.add(std::move(book1)); -booklist.add(std::move(book2)); - -// Add the third book to the front -auto pos = booklist.add(booklist.array_range().begin(),std::move(book3)); - -// and the last one immediately after -booklist.add(pos+1,std::move(book4)); - -// See what's left of book1, 2, 3 and 4 (expect nulls) -std::cout << book1 << "," << book2 << "," << book3 << "," << book4 << std::endl; - - -++ -//Loop through the booklist elements using a range-based for loop -for (const auto& book : booklist.array_range()) -{ - std::cout << book["title"].as() - << "," - << book["price"].as() << std::endl; -} - -// The second book -json& book = booklist[1]; - -//Loop through the book's name-value pairs using a range-based for loop -for (const auto& member : book.object_range()) -{ - std::cout << member.key() - << "," - << member.value() << std::endl; -} - -auto it = book.find("author"); -if (it != book.object_range().end()) -{ - // member "author" found -} - -if (book.contains("author")) -{ - // book has a member "author" -} - -book.get("author", "author unknown").as(); -// Returns author if found, otherwise "author unknown" - -try -{ - book["ratings"].as(); -} -catch (const std::out_of_range& e) -{ - // member "ratings" not found -} - -// Add ratings -book["ratings"]["*****"] = 4; -book["ratings"]["*"] = 2; - -// Delete one-star ratings -book["ratings"].erase("*"); - -``` -```c++ - // Serialize the booklist to a file - std::ofstream os("booklist.json"); - os << pretty_print(booklist); -``` - -The JSON output `booklist.json` -```json -[ - { - "author":"Haruki Murakami", - "category":"Fiction", - "date":"2006-01-03", - "isbn":"1400079276", - "price":13.45, - "title":"Kafka on the Shore" - }, - { - "author":"Charles Bukowski", - "category":"Fiction", - "date":"2004-07-08", - "isbn":"1852272007", - "price":22.48, - "ratings": - { - "*****":4 - }, - "title":"Pulp" - }, - { - "author":"Haruki Murakami", - "category":"Fiction", - "date":"2002-04-09", - "isbn":"037571894X", - "price":9.01, - "title":"A Wild Sheep Chase: A Novel" - }, - { - "author":"George Crile", - "category":"History", - "date":"2007-11-06", - "isbn":"0802143415", - "price":10.5, - "title":"Charlie Wilson's War" - } -] -``` -### json query - -```c++ -#include -#include -#include - -// For convenience -using jsoncons::json; -using jsoncons::jsonpath::json_query; - -// Deserialize the booklist -std::ifstream is("booklist.json"); -json booklist; -is >> booklist; - -// Use a JSONPath expression to find -// -// (1) The authors of books that cost less than $12 -json result = json_query(booklist, "$[*][?(@.price < 12)].author"); -std::cout << result << std::endl; - -// (2) The number of books -result = json_query(booklist, "$.length"); -std::cout << result << std::endl; - -// (3) The third book -result = json_query(booklist, "$[2]"); -std::cout << std::endl << pretty_print(result) << std::endl; - -// (4) The authors of books that were published in 2004 -result = json_query(booklist, "$[*][?(@.date =~ /2004.*?/)].author"); -std::cout << result << std::endl; - -// (5) The titles of all books that have ratings -result = json_query(booklist, "$[*][?(@.ratings)].title"); -std::cout << result << std::endl; - -// (6) All authors and titles of books -result = json_query(booklist, "$..['author','title']"); -std::cout << pretty_print(result) << std::endl; -``` -Result: -```json -(1) ["Haruki Murakami","George Crile"] -(2) [4] -(3) -[ - { - "author":"Haruki Murakami", - "category":"Fiction", - "date":"2002-04-09", - "isbn":"037571894X", - "price":9.01, - "title":"A Wild Sheep Chase: A Novel" - } -] -(4) ["Charles Bukowski"] -(5) ["Pulp"] -(6) -[ - "Nigel Rees", - "Sayings of the Century", - "Evelyn Waugh", - "Sword of Honour", - "Herman Melville", - "Moby Dick", - "J. R. R. Tolkien", - "The Lord of the Rings" -] -``` -## Once again, this time with wide characters - -### wjson construction - -```c++ -#include - -// For convenience -using jsoncons::wjson; - -// Construct a book object -wjson book1; - -book1[L"category"] = L"Fiction"; -book1[L"title"] = L"A Wild Sheep Chase: A Novel"; -book1[L"author"] = L"Haruki Murakami"; -book1[L"date"] = L"2002-04-09"; -book1[L"price"] = 9.01; -book1[L"isbn"] = L"037571894X"; - -// Construct another using the insert_or_assign function -wjson book2; - -book2.insert_or_assign(L"category", L"History"); -book2.insert_or_assign(L"title", L"Charlie Wilson's War"); -book2.insert_or_assign(L"author", L"George Crile"); -book2.insert_or_assign(L"date", L"2007-11-06"); -book2.insert_or_assign(L"price", 10.50); -book2.insert_or_assign(L"isbn", L"0802143415"); - -// Use insert_or_assign again, but more efficiently -wjson book3; - -// Reserve memory, to avoid reallocations -book3.reserve(6); - -// Insert in name alphabetical order -// Give insert_or_assign a hint where to insert the next member -auto hint = book3.insert_or_assign(book3.object_range().begin(), L"author", L"Haruki Murakami"); -hint = book3.insert_or_assign(hint, L"category", L"Fiction"); -hint = book3.insert_or_assign(hint, L"date", L"2006-01-03"); -hint = book3.insert_or_assign(hint, L"isbn", L"1400079276"); -hint = book3.insert_or_assign(hint, L"price", 13.45); -hint = book3.insert_or_assign(hint, L"title", L"Kafka on the Shore"); - -// Construct a fourth from a string -wjson book4 = wjson::parse(LR"( -{ - "category" : "Fiction", - "title" : "Pulp", - "author" : "Charles Bukowski", - "date" : "2004-07-08", - "price" : 22.48, - "isbn" : "1852272007" -} -)"); - -// Construct a booklist array - -wjson booklist = wjson::array(); - -// For efficiency, reserve memory, to avoid reallocations -booklist.reserve(4); - -// For efficency, tell jsoncons to move the contents -// of the four book objects into the array -booklist.add(std::move(book1)); -booklist.add(std::move(book2)); - -// Add the third book to the front -auto pos = booklist.add(booklist.array_range().begin(),std::move(book3)); - -// and the last one immediately after -booklist.add(pos+1,std::move(book4)); - -// See what's left of book1, 2, 3 and 4 (expect nulls) -std::wcout << book1 << L"," << book2 << L"," << book3 << L"," << book4 << std::endl; - -++ -//Loop through the booklist elements using a range-based for loop -for (const auto& book : booklist.array_range()) -{ - std::wcout << book[L"title"].as() - << L"," - << book[L"price"].as() << std::endl; -} - -// The second book -wjson& book = booklist[1]; - -//Loop through the book's name-value pairs using a range-based for loop -for (const auto& member : book.object_range()) -{ - std::wcout << member.key() - << L"," - << member.value() << std::endl; -} - -auto it = book.find(L"author"); -if (it != book.object_range().end()) -{ - // member "author" found -} - -if (book.contains(L"author")) -{ - // book has a member "author" -} - -book.get(L"author", L"author unknown").as(); -// Returns author if found, otherwise "author unknown" - -try -{ - book[L"ratings"].as(); -} -catch (const std::out_of_range& e) -{ - // member "ratings" not found -} - -// Add ratings -book[L"ratings"][L"*****"] = 4; -book[L"ratings"][L"*"] = 2; - -// Delete one-star ratings -book[L"ratings"].erase(L"*"); - -``` -```c++ -// Serialize the booklist to a file -std::wofstream os("booklist2.json"); -os << pretty_print(booklist); -``` -### wjson query - -```c++ -// Deserialize the booklist -std::wifstream is("booklist2.json"); -wjson booklist; -is >> booklist; - -// Use a JSONPath expression to find -// -// (1) The authors of books that cost less than $12 -wjson result = json_query(booklist, L"$[*][?(@.price < 12)].author"); -std::wcout << result << std::endl; - -// (2) The number of books -result = json_query(booklist, L"$.length"); -std::wcout << result << std::endl; - -// (3) The third book -result = json_query(booklist, L"$[2]"); -std::wcout << pretty_print(result) << std::endl; - -// (4) The authors of books that were published in 2004 -result = json_query(booklist, L"$[*][?(@.date =~ /2004.*?/)].author"); -std::wcout << result << std::endl; - -// (5) The titles of all books that have ratings -result = json_query(booklist, L"$[*][?(@.ratings)].title"); -std::wcout << result << std::endl; - -// (6) All authors and titles of books -result = json_query(booklist, L"$..['author','title']"); -std::wcout << pretty_print(result) << std::endl; -``` -Result: -```json -(1) ["Haruki Murakami","George Crile"] -(2) [4] -(3) -[ - { - "author":"Haruki Murakami", - "category":"Fiction", - "date":"2002-04-09", - "isbn":"037571894X", - "price":9.01, - "title":"A Wild Sheep Chase: A Novel" - } -] -(4) ["Charles Bukowski"] -(5) ["Pulp"] -(6) -[ - "Nigel Rees", - "Sayings of the Century", - "Evelyn Waugh", - "Sword of Honour", - "Herman Melville", - "Moby Dick", - "J. R. R. Tolkien", - "The Lord of the Rings" -] -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Tutorials/Custom Allocators.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Tutorials/Custom Allocators.md deleted file mode 100644 index c2c32ad6c1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Tutorials/Custom Allocators.md +++ /dev/null @@ -1,156 +0,0 @@ -## Examples - -### Using `json` with boost stateless `fast_pool_allocator` -```c++ -#include -#include "jsoncons/json.hpp" - -typedef jsoncons::basic_json> bfp_json; - -bfp_json j; - -j.insert_or_assign("FirstName","Joe"); -j.insert_or_assign("LastName","Smith"); -``` - -### Using `json` with stateful Boost.Interprocess allocators - -```c++ -#include -#include -#include -#include -#include //std::system -#include - -using namespace jsoncons; - -typedef boost::interprocess::allocator shmem_allocator; - -struct boost_sorted_policy : public sorted_policy -{ - template - using sequence_container_type = boost::interprocess::vector; - - template - using key_storage = boost::interprocess::basic_string; - - template - using string_storage = boost::interprocess::basic_string; -}; - -typedef basic_json shm_json; - -int main(int argc, char *argv[]) -{ - typedef std::pair MyType; - - if(argc == 1){ //Parent process - //Remove shared memory on construction and destruction - struct shm_remove - { - shm_remove() { boost::interprocess::shared_memory_object::remove("MySharedMemory"); } - ~shm_remove(){ boost::interprocess::shared_memory_object::remove("MySharedMemory"); } - } remover; - - //Construct managed shared memory - boost::interprocess::managed_shared_memory segment(boost::interprocess::create_only, - "MySharedMemory", 65536); - - //Initialize shared memory STL-compatible allocator - const shmem_allocator allocator(segment.get_segment_manager()); - - // Create json value with all dynamic allocations in shared memory - - shm_json* j = segment.construct("my json")(shm_json::array(allocator)); - j->push_back(10); - - shm_json o(allocator); - o.insert_or_assign("category", "reference"); - o.insert_or_assign("author", "Nigel Rees"); - o.insert_or_assign("title", "Sayings of the Century"); - o.insert_or_assign("price", 8.95); - - j->push_back(o); - - shm_json a = shm_json::array(2,shm_json::object(allocator),allocator); - a[0]["first"] = 1; - - j->push_back(a); - - std::pair res; - res = segment.find("my json"); - - std::cout << "Parent:" << std::endl; - std::cout << pretty_print(*(res.first)) << std::endl; - - //Launch child process - std::string s(argv[0]); s += " child "; - if(0 != std::system(s.c_str())) - return 1; - - - //Check child has destroyed all objects - if(segment.find("my json").first) - return 1; - } - else{ - //Open managed shared memory - boost::interprocess::managed_shared_memory segment(boost::interprocess::open_only, - "MySharedMemory"); - - std::pair res; - res = segment.find("my json"); - - if (res.first != nullptr) - { - std::cout << "Child:" << std::endl; - std::cout << pretty_print(*(res.first)) << std::endl; - } - else - { - std::cout << "Result is null" << std::endl; - } - - //We're done, delete all the objects - segment.destroy("my json"); - } - return 0; -} -``` -Output: -``` -Parent: -[ - 10, - { - "author": "Nigel Rees", - "category": "reference", - "price": 8.95, - "title": "Sayings of the Century" - }, - [ - { - "first": 1 - }, - {} - ] -] -Child: -[ - 10, - { - "author": "Nigel Rees", - "category": "reference", - "price": 8.95, - "title": "Sayings of the Century" - }, - [ - { - "first": 1 - }, - {} - ] -] -``` diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Tutorials/Unicode support.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Tutorials/Unicode support.md deleted file mode 100644 index 161716a0f1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/Tutorials/Unicode support.md +++ /dev/null @@ -1,182 +0,0 @@ -### Narrow character support for UTF8 encoding - -In the Linux and web worlds, `UTF-8` is the dominant character encoding. - -Note that (at least in MSVS) you cannot open a Windows file with a Unicode name using the standard -```c++ -std::fstream fs(const char* filename) -``` -Instead you need to use the non standard Microsoft extension -```c++ -std::fstream fs(const wchar_t* filename) -``` - -#### Unicode escaping -```c++ -string inputStr("[\"\\u0040\\u0040\\u0000\\u0011\"]"); -std::cout << "Input: " << inputStr << std::endl; - -json arr = json::parse(inputStr); -std::string str = arr[0].as(); -std::cout << "Hex dump: ["; -for (size_t i = 0; i < str.size(); ++i) -{ - unsigned int val = static_cast(str[i]); - if (i != 0) - { - std::cout << " "; - } - std::cout << "0x" << std::setfill('0') << std::setw(2) << std::hex << val; -} -std::cout << "]" << std::endl; - -std::ostringstream os; -os << arr; -std::cout << "Output: " << os.str() << std::endl; -``` - -Output: - -``` -Input: ["\u0040\u0040\u0000\u0011"] -Hex dump: [0x40 0x40 0x00 0x11] -Output: ["@@\u0000\u0011"] -``` -Note that just the two control characters are escaped on output. - -#### Reading escaped unicode into utf8 encodings and writing back escaped unicode -```c++ -string inputStr("[\"\\u007F\\u07FF\\u0800\"]"); -std::cout << "Input: " << inputStr << std::endl; - -json arr = json::parse(inputStr); -std::string s = arr[0].as(); -std::cout << "Hex dump: ["; -for (size_t i = 0; i < s.size(); ++i) -{ - if (i != 0) - std::cout << " "; - unsigned int u(s[i] >= 0 ? s[i] : 256 + s[i] ); - std::cout << "0x" << std::hex<< std::setfill('0') << std::setw(2) << u; -} -std::cout << "]" << std::endl; - -std::ostringstream os; -json_options options; -format.escape_all_non_ascii(true); -os << print(arr,options); -std::string outputStr = os.str(); -std::cout << "Output: " << os.str() << std::endl; - -json arr2 = json::parse(outputStr); -std::string s2 = arr2[0].as(); -std::cout << "Hex dump: ["; -for (size_t i = 0; i < s2.size(); ++i) -{ - if (i != 0) - std::cout << " "; - unsigned int u(s2[i] >= 0 ? s2[i] : 256 + s2[i] ); - std::cout << "0x" << std::hex<< std::setfill('0') << std::setw(2) << u; -} -std::cout << "]" << std::endl; -``` - -Output: - -``` -Input: ["\u007F\u07FF\u0800"] -Hex dump: [0x7f 0xdf 0xbf 0xe0 0xa0 0x80] -Output: ["\u007F\u07FF\u0800"] -Hex dump: [0x7f 0xdf 0xbf 0xe0 0xa0 0x80] -``` -Since the escaped unicode consists of a control character (0x7f) and non-ascii, we get back the same text as what we started with. - -#### Reading escaped unicode into utf8 encodings and writing back escaped unicode (with continuations) -```c++ -string input = "[\"\\u8A73\\u7D30\\u95B2\\u89A7\\uD800\\uDC01\\u4E00\"]"; -json value = json::parse(input); -json_options options; -format.escape_all_non_ascii(true); -string output; -value.dump(output,options); - -std::cout << "Input:" << std::endl; -std::cout << input << std::endl; -std::cout << std::endl; -std::cout << "Output:" << std::endl; -std::cout << output << std::endl; -``` -Since all of the escaped unicode is non-ascii, we get back the same text as what we started with. -``` -Input: -["\u8A73\u7D30\u95B2\u89A7\uD800\uDC01\u4E00"] - -Output: -["\u8A73\u7D30\u95B2\u89A7\uD800\uDC01\u4E00"] -``` -### Wide character support for UTF16 and UTF32 encodings - -jsoncons supports wide character strings and streams with `wjson` and `wjson_reader`. It assumes `UTF16` encoding if `wchar_t` has size 2 (Windows) and `UTF32` encoding if `wchar_t` has size 4. - -It is necessary to deal with UTF-16 character encoding in the Windows world because of lack of UTF-8 support in the Windows system API. - -Even if you choose to use wide character streams and strings to interact with the Windows API, you can still read and write to files in the more widely supported, endiness independent, UTF-8 format. To handle that you need to imbue your streams with the facet `std::codecvt_utf8_utf16`, which encapsulates the conversion between `UTF-8` and `UTF-16`. - -Note that (at least in MSVS) you cannot open a Windows file with a Unicode name using the standard - - std::wfstream fs(const char* filename) - -Instead you need to use the non standard Microsoft extension - - std::wfstream fs(const wchar_t* filename) - -#### Constructing a wjson value -```c++ -using jsoncons::wjson; - -wjson root; -root[L"field1"] = L"test"; -root[L"field2"] = 3.9; -root[L"field3"] = true; -std::wcout << root << L"\n"; -``` -Output: -``` -{"field1":"test","field2":3.9,"field3":true} -``` -#### Escaped unicode -```c++ -wstring input = L"[\"\\u007F\\u07FF\\u0800\"]"; -std::wistringstream is(input); - -wjson val = wjson::parse(is); - -wstring s = val[0].as(); -std::cout << "length=" << s.length() << std::endl; -std::cout << "Hex dump: ["; -for (size_t i = 0; i < s.size(); ++i) -{ - if (i != 0) - std::cout << " "; - uint32_t u(s[i] >= 0 ? s[i] : 256 + s[i] ); - std::cout << "0x" << std::hex<< std::setfill('0') << std::setw(2) << u; -} -std::cout << "]" << std::endl; - -std::wofstream os("output/xxx.txt"); -os.imbue(std::locale(os.getloc(), new std::codecvt_utf8_utf16)); - -wjson_options options; -format.escape_all_non_ascii(true); - -os << pretty_print(val,options) << L"\n"; -``` -Output: -``` -length=3 -Hex dump: [0x7f 0x7ff 0x800] -``` -and the file `xxx.txt` contains -``` -["\u007F\u07FF\u0800"] -``` diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/build.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/build.md deleted file mode 100644 index 41e9b7c013..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/build.md +++ /dev/null @@ -1,6 +0,0 @@ -Start Visual Studio Command prompt for x64 - -mkdir build64 & pushd build64 -cmake -G "Visual Studio 14 2015 Win64" .. -popd -cmake --build build64 --config Debug diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/big_integer_chars_format.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/big_integer_chars_format.md deleted file mode 100644 index 70797ebfd7..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/big_integer_chars_format.md +++ /dev/null @@ -1,101 +0,0 @@ -### jsoncons::bigint_chars_format - -```c++ -enum class bigint_chars_format : uint8_t {number, base10, base64, base64url}; -``` - -#### Header -```c++ -#include -``` - -Specifies `bignum` formatting. - -### Examples - -#### Initializing with bignum - -```c++ -#include - -using namespace jsoncons; - -int main() -{ - std::string s = "-18446744073709551617"; - - json j(bignum(s.c_str())); - - std::cout << "(default) "; - j.dump(std::cout); - std::cout << "\n\n"; - - std::cout << "(integer) "; - json_options options1; - options1.bigint_format(bigint_chars_format::number); - j.dump(std::cout, options1); - std::cout << "\n\n"; - - std::cout << "(base64) "; - json_options options3; - options3.bigint_format(bigint_chars_format::base64); - j.dump(std::cout, options3); - std::cout << "\n\n"; - - std::cout << "(base64url) "; - json_options options4; - options4.bigint_format(bigint_chars_format::base64url); - j.dump(std::cout, options4); - std::cout << "\n\n"; -} -``` -Output: -``` -(default) "-18446744073709551617" - -(integer) -18446744073709551617 - -(base64) "~AQAAAAAAAAAA" - -(base64url) "~AQAAAAAAAAAA" -``` - -#### Integer overflow during parsing - -```c++ -#include - -using namespace jsoncons; - -int main() -{ - std::string s = "-18446744073709551617"; - - json j = json::parse(s); - - std::cout << "(1) "; - j.dump(std::cout); - std::cout << "\n\n"; - - std::cout << "(2) "; - json_options options1; - options1.bigint_format(bigint_chars_format::number); - j.dump(std::cout, options1); - std::cout << "\n\n"; - - std::cout << "(3) "; - json_options options2; - options2.bigint_format(bigint_chars_format::base64url); - j.dump(std::cout, options2); - std::cout << "\n\n"; -} -``` -Output: -``` -(1) "-18446744073709551617" - -(2) -18446744073709551617 - -(3) "~AQAAAAAAAAAB" -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bignum.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bignum.md deleted file mode 100644 index 42b0502b2a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bignum.md +++ /dev/null @@ -1,279 +0,0 @@ -### jsoncons::bignum - -```c++ -typedef basic_bignum> bignum; -``` -The `bignum` class is an instantiation of the `basic_bignum` class template that uses `std::allocator` as the allocator type. - -An arbitrary-precision integer. - -#### Header -```c++ -#include -``` - -#### Constructor - - bignum(); - - explicit bignum(const Allocator& alloc); - - explicit bignum(const char* str); -Constructs a bignum from the decimal string representation of a bignum. - - explicit bignum(const char* data, size_t length); -Constructs a bignum from the decimal string representation of a bignum. - - explicit bignum(const char* str, const Allocator& alloc); - - bignum(int signum, std::initializer_list magnitude); -Constructs a bignum from the sign-magnitude representation. -The magnitude is an unsigned integer `n` encoded as a byte string data item in big-endian byte-order. -If the value of signum is 1, the value of the bignum is `n`. -If the value of signum is -1, the value of the bignum is `-1 - n`. -An empty list means a zero value. - - bignum(int signum, std::initializer_list magnitude, const Allocator& alloc); - - bignum(const bignum& s); - - bignum(bignum&& s); - -#### Assignment - - bignum& operator=(const bignum& s); - - bignum& operator=(bignum&& s); - -#### Accessors - - template - void dump(std::basic_string& data) const - - template - void dump(int& signum, std::vector& data) const - -#### Arithmetic operators - - explicit operator bool() const - - explicit operator int64_t() const - - explicit operator uint64_t() const - - explicit operator double() const - - explicit operator long double() const - - bignum operator-() const - - bignum& operator+=( const bignum& y ) - - bignum& operator-=( const bignum& y ) - - bignum& operator*=( int64_t y ) - - bignum& operator*=( uint64_t y ) - - bignum& operator*=( bignum y ) - - bignum& operator/=( const bignum& divisor ) - - bignum& operator%=( const bignum& divisor ) - - bignum& operator<<=( uint64_t k ) - - bignum& operator>>=(uint64_t k) - - bignum& operator++() - - bignum operator++(int) - - bignum& operator--() - - bignum operator--(int) - - bignum& operator|=( const bignum& a ) - - bignum& operator^=( const bignum& a ) - - bignum& operator&=( const bignum& a ) - -#### Non-member functions - - bool operator==(const bignum& lhs, const bignum& rhs); - - bool operator!=(const bignum& lhs, const bignum& rhs); - - template - friend std::basic_ostream& operator<<(std::basic_ostream& os, const bignum& o); - -#### Global arithmetic operators - - bool operator==( const bignum& x, const bignum& y ) - - bool operator==( const bignum& x, int y ) - - bool operator!=( const bignum& x, const bignum& y ) - - bool operator!=( const bignum& x, int y ) - - bool operator<( const bignum& x, const bignum& y ) - - bool operator<( const bignum& x, int64_t y ) - - bool operator>( const bignum& x, const bignum& y ) - - bool operator>( const bignum& x, int y ) - - bool operator<=( const bignum& x, const bignum& y ) - - bool operator<=( const bignum& x, int y ) - - bool operator>=( const bignum& x, const bignum& y ) - - bool operator>=( const bignum& x, int y ) - - bignum operator+( bignum x, const bignum& y ) - - bignum operator+( bignum x, int64_t y ) - - bignum operator-( bignum x, const bignum& y ) - - bignum operator-( bignum x, int64_t y ) - - bignum operator*( int64_t x, const bignum& y ) - - bignum operator*( bignum x, const bignum& y ) - - bignum operator*( bignum x, int64_t y ) - - bignum operator/( bignum x, const bignum& y ) - - bignum operator/( bignum x, int y ) - - bignum operator%( bignum x, const bignum& y ) - - bignum operator<<( bignum u, unsigned k ) - - bignum operator<<( bignum u, int k ) - - bignum operator>>( bignum u, unsigned k ) - - bignum operator>>( bignum u, int k ) - - bignum operator|( bignum x, const bignum& y ) - - bignum operator|( bignum x, int y ) - - bignum operator|( bignum x, unsigned y ) - - bignum operator^( bignum x, const bignum& y ) - - bignum operator^( bignum x, int y ) - - bignum operator^( bignum x, unsigned y ) - - bignum operator&( bignum x, const bignum& y ) - - bignum operator&( bignum x, int y ) - - bignum operator&( bignum x, unsigned y ) - - bignum abs( const bignum& a ) - - bignum power( bignum x, unsigned n ) - - bignum sqrt( const bignum& a ) - -### Examples - - -### Examples - -#### Initializing with bignum - -```c++ -#include - -using namespace jsoncons; - -int main() -{ - std::string s = "-18446744073709551617"; - - json j(bignum(s.c_str())); - - std::cout << "(1) " << j.as() << "\n\n"; - - std::cout << "(2) " << j.as() << "\n\n"; - - std::cout << "(3) "; - j.dump(std::cout); - std::cout << "\n\n"; - - std::cout << "(4) "; - json_options options1; - options1.bigint_format(bigint_chars_format::number); - j.dump(std::cout, options1); - std::cout << "\n\n"; - - std::cout << "(5) "; - json_options options2; - options2.bigint_format(bigint_chars_format::base64url); - j.dump(std::cout, options2); - std::cout << "\n\n"; -} -``` -Output: -``` -(1) -18446744073709551617 - -(2) -18446744073709551617 - -(3) "-18446744073709551617" - -(4) -18446744073709551617 - -(5) "~AQAAAAAAAAAB" -``` - -#### Integer overflow during parsing - -```c++ -#include - -using namespace jsoncons; - -int main() -{ - std::string s = "-18446744073709551617"; - - json j = json::parse(s); - - std::cout << "(1) "; - j.dump(std::cout); - std::cout << "\n\n"; - - std::cout << "(2) "; - json_options options1; - options1.bigint_format(bigint_chars_format::number); - j.dump(std::cout, options1); - std::cout << "\n\n"; - - std::cout << "(3) "; - json_options options2; - options2.bigint_format(bigint_chars_format::base64url); - j.dump(std::cout, options2); - std::cout << "\n\n"; -} -``` -Output: -``` -(1) "-18446744073709551617" - -(2) -18446744073709551617 - -(3) "~AQAAAAAAAAAB" -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/bson.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/bson.md deleted file mode 100644 index 3d9add7cbf..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/bson.md +++ /dev/null @@ -1,29 +0,0 @@ -### bson extension - -The bson extension implements decode from and encode to the [Binary JSON](http://bsonspec.org/) data format. -You can either parse into or serialize from a variant-like structure, [basic_json](../json.md), or your own -data structures, using [json_type_traits](../json_type_traits.md). - -[decode_bson](decode_bson.md) - -[encode_bson](encode_bson.md) - -[bson_encoder](bson_encoder.md) - -#### jsoncons-BSON mappings - -jsoncons data item|jsoncons tag|BSON data item ---------------|------------------|--------------- -null | | null -bool | | true or false -int64 | | int32 or int64 -int64 | timestamp | datetime -uint64 | | int32 or int64 -uint64 | timestamp | datetime -double | | double -string | | string -byte_string | | binary -array | | 0x04 (array ) -object | | 0x03 (document) - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/bson_encoder.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/bson_encoder.md deleted file mode 100644 index 9c33accf25..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/bson_encoder.md +++ /dev/null @@ -1,147 +0,0 @@ -### jsoncons::bson::basic_bson_encoder - -```c++ -template< - class Result> -> class basic_bson_encoder : public jsoncons::json_content_handler -``` - -`basic_bson_encoder` is noncopyable. - -#### Header - - #include - -![bson_encoder](./diagrams/bson_encoder.png) - -Two specializations for common result types are defined: - -Type |Definition ----------------------------|------------------------------ -bson_encoder |basic_bson_encoder -bson_bytes_encoder |basic_bson_encoder - -#### Member types - -Type |Definition ----------------------------|------------------------------ -char_type |char -result_type |Result -string_view_type | - -#### Constructors - - explicit basic_bson_encoder(result_type result) -Constructs a new encoder that writes to the specified result. - -#### Destructor - - virtual ~basic_bson_encoder() - -### Inherited from [basic_json_content_handler](../json_content_handler.md) - -#### Member functions - - bool begin_object(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool begin_object(size_t length, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool end_object(const ser_context& context = null_ser_context()) - - bool begin_array(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool begin_array(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool end_array(const ser_context& context=null_ser_context()); - - bool name(const string_view_type& name, - const ser_context& context=null_ser_context()); - - bool string_value(const string_view_type& value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool byte_string_value(const byte_string_view& b, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool byte_string_value(const uint8_t* p, size_t size, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool int64_value(int64_t value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool uint64_value(uint64_t value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool double_value(double value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool bool_value(bool value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool null_value(semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - void flush() - -### Examples - -#### Encode to BSON - -```c++ -#include -#include -#include - -int main() -{ - std::vector buffer; - bson::bson_bytes_encoder encoder(buffer); - encoder.begin_array(); // The total number of bytes comprising - // the bson document will be calculated - encoder.string_value("cat"); - encoder.byte_string_value(byte_string({'p','u','r','r'})); - encoder.int64_value(1431027667, semantic_tag::timestamp); - encoder.end_array(); - encoder.flush(); - - for (auto c : buffer) - { - std::cout << std::hex << std::setprecision(2) << std::setw(2) - << std::noshowbase << std::setfill('0') << static_cast(c); - } - std::cout << "\n\n"; - -/* - 22000000 -- Total number of bytes comprising the document (34 bytes) - 02 -- UTF-8 string - 3000 -- "0" - 04000000 -- number bytes in the string (including trailing byte) - 636174 -- "cat" - 00 -- trailing byte - 05 -- binary - 3100 -- "1" - 04000000 -- number of bytes - 70757272 -- 'P''u''r''r' - 09 -- datetime - 3200 -- "2" - d3bf4b55 -- 1431027667 - 00 -*/ -} -``` -Output: -``` -2200000002300004000000636174000531000400000070757272093200d3bf4b5500 -``` diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/decode_bson.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/decode_bson.md deleted file mode 100644 index 51aa31f9ea..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/decode_bson.md +++ /dev/null @@ -1,30 +0,0 @@ -### jsoncons::bson::decode_bson - -Decodes a [Binary JSON (BSON)](http://bsonspec.org/) data format into a C++ data structure. - -#### Header -```c++ -#include - -template -T decode_bson(const std::vector& v); // (1) - -template -T decode_bson(std::istream& is); // (2) -``` - -(1) Reads a BSON bytes buffer into a type T if T is an instantiation of [basic_json](../json.md) -or if T supports [json_type_traits](../json_type_traits.md). - -(2) Reads a BSON binary stream into a type T if T is an instantiation of [basic_json](../json.md) -or if T supports [json_type_traits](../json_type_traits.md). - -#### Exceptions - -Throws [ser_error](../ser_error.md) if parsing fails. - -#### See also - -- [encode_bson](encode_bson.md) encodes a json value to the [Binary JSON](http://bsonspec.org/) data format. - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/diagrams/bson_encoder.png b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/diagrams/bson_encoder.png deleted file mode 100644 index 1bb1c73475c8b30fbfc618c5d1969c1cd2d9eba1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5216 zcmeI0^;Z;3+s7A>S_G7vP)bSZ4gm=lC8QSEMQSNwkuGVFMM^?IkOl$irKDpel%sKO&nQP+up3h7qSX-46$Or@g0F>%#uk-)_0w}(2Lq>ux zu>`Kt002>kAH?v3laH#qrR#ew7i$|w06+`USB+GAfMw)ZGBG8JdT9hS{H!U)(_29s z?;8$kYmGkldgT(8sqJ1ZrtS2o#5kcWMjC-k=h3KsJ|}Wa0#F+JL)~=?_??$Z=>cGt zF@*DrII4~3D>#K8nM8{&smTACFS{xn<*x|xP&w)Ar*Q)OTTiQwJ*ttlKUC73;2_od zD#Ae%&FORQAoSuUC4kDwr2FITXti6HI7+aiQm{7Bc8lrhL)2hY5{gDK8Xl%*w+!JV z^d3xRU-yTj%teLXfU~ic18>Ax$1npl8;|8!lITYmFNXCDt6_)bX=@*Q)2~fbdfiR_ zW;|MUy8WVZlqtU^+0QH;Lu<2z&WrkH-R7NUrwBQ%L8T~W>P|(95}0A7N_5ZW{{`*u zM&P8TI!p$Al(W+&PWQg8dBfvu^PTolBZT;)8~>gUU=|iz`=hW z@HM%~Fa0L7$?$8|e6mERZKANdfxOx;Mu+-0>S``!rxP?Htu^a^5urbP>2m#B{-j)T z{@wUtVsz%p|5Zju3#m4j*ZrtIdSPJ3by@N;;9MJ#$R(zjM&_|?b*V3;yN5ogT`)@# z8oG2eyV)Z8!2j*+>VCxJ$^DdgTDh6ln9-yh7s@320Bv^`4bL{F&_5kj5kn5VDh4bT zicMvqFFqKhO1)-q&0<(pMA;r`>E{J;NXw40zRzpZ0uUY`Gj7~D9EQ>1XulKwi^XJ>H+Q%zbZ zL=Ce|YS#po)5|i;aaTu#u53cGI7B5z&5F+C7>TqmmP*XKlOAz$1)tb&QH#{I^AsO_FaBcwTuHRVE`p`bhpO8yRk|Efp}lDy&f*g94W}`FQM&E3#_VH zcMSp4)KjplbFP1T)o6&iE4JOY{o*owHGBa(upB(X*>0O)0(<`Y;zm;2u3~I8TMfEX zxjUkPli=V&?x2*F(Kc*Lc)#n;3%*ex_{>*#Qx`Iq2@5tTH?|~zAAr}B5j#ORY$e}F z|MoGmJv8Wc;sMh4i`i4Je~`iV5}XZ*$)G}^>HJE?Q!nmGCT^SlkVB<*W>YIeD$T6LFKv6Gh(UhpWOHQb`V2FK z!`^762*INE%`sHj3V~u2^a4K>PVpK}k7|3@&0!Xg)lTvn+Y#1@dZ+n>jW?JNJCnQ~ zm?^;;PmDp0JsMeE7B`yfcZUn~*cg0VV>6IU-}7%*Gy}XRmIb~!oK@(tIsCed;o@`$ zT{<*bqCYg6s~o+Panb$xcLfYJ__re>daB%Txs(KLMM+U2pPSCA1*(%R%>tj@Lxzx0 zijh*YO2j=;lzOJ~Jw{B+`W1(2Y>Yw2-h6|TN^}fR2UPn+!~zir;*j$Yhu*e2FUroF zUvmozW)%e9UW$4JoTA@Fl_$7KOGvyraaw2`D}$lxSKdsu4ra+o_v300j%7%c;RnMt z)_pWLSGmd&?DC9EOnD=huz)$%|j=0FjPx%IKjS3^9^L_JhdH z`JQ;5YFw;X;YhY(hK$EXYKFASoKK}`)3^Yvc@h_M=`>$oKD+5$7`1FW^4;mXLZCf- zC_O2ik^=Iu=Uq?itb(C!;xirXOltEjncBTxTr9JU)s5LNpRrTS^hoSH%KKo_zjX4b zmi29gY1M4~<%io#H@TvK{W(gQWu-Op(s#8pvOaE#(&L5y`Jid@hmtfYV#{1c+ChkU z+earCt9Hk~f5PAgOD*-N+xr&WPULRNmQ?@?M&$du9W?)qZ?aQ)ZEy!Z>e%{BzkrxX zr^*@N&nM4a+gEY{#ee@KU*b;`j=Q~jxSQDNUZYVJbhz@He#g%d0aP+7H4v31`YwF$*?iFL zFUMK2nt&^J3bB}Z{2eAs*q@^ZSPBvd8LowGbaXI{_Any8q^@CaF6ftZzQ3rlAnd)q zIB*HvUTX0_1%F00{LA)9l8Oh;a~d%rH27aQ`(H07 zXrkq@(Q@x4Aa^7m{V5D{A`v&+BBTDICbZ&k+vVwPK&TJf`LdeAI zv*OC{d<&99TZu!bg{pk}zwwqxqCwy!Ww|XbNt+TQ!q%jI2&@<+Ebvs|uvQc5#rhLE zT{K~jl%=1+HAv{Ue8e_C7KdvlpP7D@62unW{R@6YrJG;41GhksQB}ta%KM%2AlV4r z-6Wwfrf=!?b5l@@NU7q;*1#JnWC{1s+u}kbK^igA*(*}v5s{?X1zT>?ro(ik0?VVt3DTyP~@;5Zzi=*0F{cpF(rdo!d#T-Cc|S#V(<{&fU6J7=A%3B&h04 z+<+I%0`H2V8(PxW^26kDrSY|YsS6{$ND(&YBODY6 zNhU&mMrGnK6Y!RGmBGWQ`>fUOha5$7Zu3FbvFb%LZXdg{aL=-SU>XR=wK36Qe7oao z+wQ7*g}`|3tYcewY>zMLlpu}bTWPF{@@sLlKyJR@>qoDZZ}NTkrzTS(oWCDyV7N@?UTmKAHg)h_Cr8S9cuGzjv>Uw9k@lE8u?~!@)2oL% zm;BAl&WIaBn#4myI?ZR$p5N%42;35PGC>G6|YcvU*UX zQ5ZFtlx7bT46(@Mf`pLUd}MKRKt4~SZ>+bQ2$w|PhSv)TK7(Myc0bBwoqp$AiJG`O z-<#jOK3x{Bv(qY|dkWz%7wHi$R?1DetAwH8-LWhtu9NmIOLgDr=^27&+!=A|+tCGfKG5ATE(@gZVkG+;)r zi>IKCA>PSgN`+3GoLceq6At547y(EUW#DMUhF1g02P_?a;MwV)S(y?81bE$#Oda2x z|9)eV0VG)zoHttqd<8C0faD|tB_t)$?$?t|M>=;$KM1>cMv;A+bg-CbjvBRS_6Hw!UOrDHTD~AyBlo>yxtChOW|I7bx?+j!hR3}>@ zjhKQy^$atep%8e@seN;M1ReYTl2ri+gNnPBgeIYn~)a=tgE~ zXUW>&ZNaAAdC?yKfUwzP7PWd6nODU`ciDzd5tj0Bdk}OW-E+x49F)`iUlzC59%>?d zi@$O&+SOeh935-Ztd*5Rj2fJ?oa)`yRBF8Tj5*SF;@Ol3SG%Hz5ClYn_@}^+-Sgor zIrzW5U+YKkNJH5kO0gD*y++r!xUGfD_@9@hiP=P95AY!T!>!qC&%RJ6+quH5g;CDs ze`(khA9Jjj+Ki;<4&>zWXzjfjKy>)~bMPCvOL6Cg=O=h9Z?i`F33z`*_;klSo}xZI z7|jhas#+_v`=ErjcHUi_#r+G}`03l>s4mMaZn z5HkHNG3_xuR(*dM3X*J$=a!=-g*IKCc|Ur&ywba*o%!4{*Y9ZcwMq0hr<#Cgt_erg zQ;?crimhl$SqjgFqd9}cVeCSkTt(r|HTFM<+}RPIS84Qx%fccDAN? zN=rlSjn+^7e?Bh;YA$4$KY=v4{-!rH$44!V=RwK;Ff~=cc+QSUXN%(oBngLl!$ZlCj?i%K^c*n!l-w1q(u(n9uncAfDaa zV(&j&nPlFB&dL&LzfYmtuuHSgly~;2wnlCd6$9g=`ygi>KlXXf45ZfvfKY@D9@lkYqP+TFG8o)`zf2W^Ze3R{;X1s7VR~9n@LOA((}e+H0A%2A|9Cp&g& zI!TXcqdogzxlL6P+Jc?tij=q`LyNd5{AaT9QXt<`>X`hF5}@`_u15|aH$CL(Smzx7 z3ai3V*7#gc=UWOgs5yr+3h%+a1^AQ_>QKom!P_?!Z{O3sqe6K5b}zwq8~8c)9$ycD bt^|GizSz8yer|&QWB{nY)_zq1viS5rm*nHQ diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/diagrams/bson_encoder.xml b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/diagrams/bson_encoder.xml deleted file mode 100644 index e87493a576..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/diagrams/bson_encoder.xml +++ /dev/null @@ -1 +0,0 @@ -vVZdb9owFP01eexEYqD0sQXabajVOiZBn5CJbxNvjs0cA2G/fjaxSZwElUpV+1D5nuvrj3POdQjQOCseJN6kj4IAC6IeKQI0CaLoOhzp/wY4lAAa9kogkZSUUFgBc/oPLOimbSmB3JuohGCKbnwwFpxDrDwMSyn2/rRXwfxdNziBFjCPMWujC0pUWqKjQa/CvwJNUrdz2LOZDLvJFshTTMS+BqFpgMZSCFWOsmIMzHDneCnr7s9kTweTwNUlBdOnXwWfPc/TaPx3phaP2Uvx/cqussNsay+8xjmNV79zwVeaU6UXX6WYEwYyiIZM73S3NqPEjLqQ41XVwfEnxZYTMEfo6fQ+pQrmGxyb7F4bRmOpypiOwlN1/UrufCAVFDXIXvEBRAZKHvQUmx1Ztq3dQsf+vhIvdBZMa8L1LYatX5LTyhWlemBZfQfD12cYXhuGgce6YT6B2VfK2FgwIY+1CI5/Gs+VFH+glkFDdIPIx2hxkRj9zxRj1BKjRStwcmveDR3FDOdaKp9JKKhaWtLN+MWMvwxsNClqqcnBBVwffukWMEGtyoRV2TFydWc1yMVWxvC26xSWCai3+x+I9wq2Fa0pNugQzGESGFZ057+dXSraHX4Iqm92Mkzf90uEGj4or22L6k9dY52bhu9GjXVKWlrrHC11uvRFLkt2j2k0e1rin5BcLZ5nsJtBR8t/4ylIqjDXoulvIcWJxFnLerqtlG82vz254NDoZQthRhNuHKuNop8TdGealOqv2K1NZJQQs03nO+G/JB/Q+GFDgKjX0fiow0fR+xtfh9U3tFSw+iGCpv8B \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/encode_bson.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/encode_bson.md deleted file mode 100644 index 21a0644a41..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/bson/encode_bson.md +++ /dev/null @@ -1,25 +0,0 @@ -### jsoncons::bson::encode_bson - -Encodes a C++ data structure to the [Binary JSON (BSON)](http://bsonspec.org/) data format. - -#### Header -```c++ -#include - -template -void encode_bson(const T& jval, std::vector& v); // (1) - -template -void encode_bson(const T& jval, std::ostream& os); // (2) -``` - -(1) Writes a value of type T into a bytes buffer in the BSON data format. Type T must be an instantiation of [basic_json](../json.md) -or support [json_type_traits](../json_type_traits.md). - -(2) Writes a value of type T into a binary stream in the BSON data format. Type T must be an instantiation of [basic_json](../json.md) -or support [json_type_traits](../json_type_traits.md). - -#### See also - -- [decode_bson](decode_bson) decodes a [Binary JSON](http://bsonspec.org/) data format to a json value. - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/byte_string.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/byte_string.md deleted file mode 100644 index c95c81dd4a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/byte_string.md +++ /dev/null @@ -1,130 +0,0 @@ -### jsoncons::byte_string - -```c++ -typedef basic_byte_string> byte_string; -``` -The `byte_string` class is an instantiation of the `basic_byte_string` class template that uses `std::allocator` as the allocator type. - -#### Header -```c++ -#include -``` - -#### Member types - -Member type |Definition -------------------------------------|------------------------------ -`const_iterator`| -`iterator`|Same as `const_iterator` -`size_type`|std::size_t - -#### Constructor - - byte_string(); - - explicit byte_string(const Allocator& alloc); - - byte_string(std::initializer_list init); - - byte_string(std::initializer_list init, const Allocator& alloc); - - explicit byte_string(const byte_string_view& v); - - byte_string(const byte_string_view& v, const Allocator& alloc); - - byte_string(const char* s); - - byte_string(const byte_string& s); - - byte_string(byte_string&& s); - -#### Assignment - - byte_string& operator=(const byte_string& s); - - byte_string& operator=(byte_string&& s); - -#### Iterators - - const_iterator begin() const noexcept; - - const_iterator end() const noexcept; - -#### Element access - - const uint8_t* data() const; - - uint8_t operator[](size_type pos) const; - - operator byte_string_view() const noexcept; - -#### Capacity - - size_t size() const; - - size_t length() const; - -#### Non-member functions - - bool operator==(const byte_string& lhs, const byte_string& rhs); - - bool operator!=(const byte_string& lhs, const byte_string& rhs); - - template - friend std::ostream& operator<<(std::ostream& os, const byte_string& o); - -### Examples - -#### Byte string from initializer list - -```c++ -json j(byte_string({'H','e','l','l','o'})); -byte_string bs = j.as(); - -std::cout << "(1) "<< bs << "\n\n"; - -std::cout << "(2) "; -for (auto b : bs) -{ - std::cout << (char)b; -} -std::cout << "\n\n"; - -std::cout << "(3) " << j << std::endl; -``` - -Output: -``` -(1) 0x480x650x6c0x6c0x6f - -(2) Hello - -(3) "SGVsbG8_" -``` - -#### Byte string from char array - -```c++ -json j(byte_string("Hello")); -byte_string bs = j.as(); - -std::cout << "(1) "<< bs << "\n\n"; - -std::cout << "(2) "; -for (auto b : bs) -{ - std::cout << (char)b; -} -std::cout << "\n\n"; - -std::cout << "(3) " << j << std::endl; -``` - -Output: -``` -(1) 0x480x650x6c0x6c0x6f - -(2) Hello - -(3) "SGVsbG8_" -``` diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/byte_string_chars_format.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/byte_string_chars_format.md deleted file mode 100644 index 231988d1b3..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/byte_string_chars_format.md +++ /dev/null @@ -1,58 +0,0 @@ -### jsoncons::byte_string_chars_format - -```c++ -enum class byte_string_chars_format : uint8_t {base16, base64, base64url}; -``` - -#### Header -```c++ -#include -``` - -Specifies byte string formatting. - -### Examples - -#### Byte string formatting - -```c++ -#include - -using namespace jsoncons; - -int main() -{ - std::vector bytes = {'H','e','l','l','o'}; - - json j(byte_string(bytes.data(),bytes.size())); - - // default - std::cout << "(1) "<< j << "\n\n"; - - // base16 - json_options options2; - options2.byte_string_format(byte_string_chars_format::base16); - std::cout << "(2) "<< print(j, options2) << "\n\n"; - - // base64 - json_options options3; - options3.byte_string_format(byte_string_chars_format::base64); - std::cout << "(3) "<< print(j, options3) << "\n\n"; - - // base64url - json_options options4; - options4.byte_string_format(byte_string_chars_format::base64url); - std::cout << "(4) "<< print(j, options4) << "\n\n"; -} -``` -Output: -``` -(1) "SGVsbG8" - -(2) "48656C6C6F" - -(3) "SGVsbG8=" - -(4) "SGVsbG8" -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor.md deleted file mode 100644 index eb2da36205..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor.md +++ /dev/null @@ -1,175 +0,0 @@ -## cbor extension - -The cbor extension implements decode from and encode to the IETF standard [Concise Binary Object Representation (CBOR)](http://cbor.io/). -You can either parse into or serialize from a variant-like structure, [basic_json](../json.md), or your own -data structures, using [json_type_traits](../json_type_traits.md). - -[decode_cbor](decode_cbor.md) - -[encode_cbor](encode_cbor.md) - -[cbor_encoder](cbor_encoder.md) - -[cbor_options](cbor_options.md) - -### Tag handling and extensions - -All tags not explicitly mentioned below are ignored. - -0 (standard date/time string) -CBOR standard date/time strings are decoded into jsoncons strings tagged with `semantic_tag::datetime`. -jsoncons strings tagged with `semantic_tag::datetime` are encoded into CBOR standard date/time strings. - -1 (epoch time) -CBOR epoch times are decoded into jsoncons int64_t, uint64_t and double and tagged with `semantic_tag::timestamp`. -jsoncons int64_t, uint64_t and double tagged with `semantic_tag::timestamp` are encoded into CBOR epoch time. - -2,3 (positive and negative bignum) -CBOR positive and negative bignums are decoded into jsoncons strings and tagged with `semantic_tag::bigint`. -jsoncons strings tagged with `semantic_tag::bigint` are encoded into CBOR positive or negative bignums. - -4 (decimal fratction) -CBOR decimal fractions are decoded into jsoncons strings tagged with `semantic_tag::bigdec`. -jsoncons strings tagged with `semantic_tag::bigdec` are encoded into CBOR decimal fractions. - -5 (bigfloat) -CBOR bigfloats are decoded into a jsoncons string that consists of the following parts - -- (optional) minus sign -- 0x -- nonempty sequence of hexadecimal digits (defines mantissa) -- p followed with optional minus or plus sign and nonempty sequence of hexadecimal digits (defines base-2 exponent) - -and tagged with `semantic_tag::bigfloat`. - -jsoncons strings that consist of the following parts - -- (optional) plus or minus sign -- 0x or 0X -- nonempty sequence of hexadecimal digits optionally containing a decimal-point character -- (optional) p or P followed with optional minus or plus sign and nonempty sequence of decimal digits - -and tagged with `semantic_tag::bigfloat` are encoded into CBOR bignums. - -21, 22, 23 (byte string expected conversion is base64url, base64 or base16) -CBOR byte strings tagged with 21, 22 and 23 are decoded into jsoncons byte strings tagged with -`semantic_tag::base64url`, `semantic_tag::base64` and `semantic_tag::base16`. -jsoncons byte strings tagged with `semantic_tag::base64url`, `semantic_tag::base64` and `semantic_tag::base16` -are encoded into CBOR byte strings tagged with 21, 22 and 23. - -32 (URI) -CBOR URI strings are decoded into jsoncons strings tagged with `semantic_tag::uri`. -jsoncons strings tagged with `semantic_tag::uri` are encoded into CBOR URI strings. - -33, 34 (UTF-8 string is base64url or base64) -CBOR strings tagged with 33 and 34 are decoded into jsoncons strings tagged with `semantic_tag::base64url` and `semantic_tag::base64`. -jsoncons strings tagged with `semantic_tag::base64url` and `semantic_tag::base64` are encoded into CBOR strings tagged with 33 and 34. - -256, 25 [stringref-namespace, stringref](http://cbor.schmorp.de/stringref) -Tags 256 and 25 are automatically decoded when detected. They are encoded when CBOR option `pack_strings` is set to true. - -### jsoncons - CBOR mappings - -jsoncons data item|jsoncons tag|CBOR data item|CBOR tag ---------------|------------------|---------------|-------- -null | | null |  -null | undefined | undefined |  -bool | | true or false |  -int64 | | unsigned or negative integer |  -int64 | timestamp | unsigned or negative integer | 1 (epoch-based date/time) -uint64 | | unsigned integer |  -uint64 | timestamp | unsigned integer | 1 (epoch-based date/time) -double | | half-precision float, float, or double |  -double | timestamp | double | 1 (epoch-based date/time) -string | | string |  -string | bigint | byte string | 2 (positive bignum) or 2 (negative bignum) -string | bigdec | array | 4 (decimal fraction) -string | bigfloat | array | 5 (bigfloat) -string | datetime | string | 0 (date/time string) -string | uri | string | 32 (uri) -string | base64url | string | 33 (base64url) -string | base64 | string | 34 (base64) -byte_string | | byte string |  -byte_string | base64url | byte string | 21 (Expected conversion to base64url encoding) -byte_string | base64 | byte string | 22 (Expected conversion to base64 encoding) -byte_string | base16 | byte string | 23 (Expected conversion to base16 encoding) -array | | array |  -object | | map |  - -### Examples - -```c++ -#include -#include -#include - -using namespace jsoncons; - -int main() -{ - ojson j1 = ojson::parse(R"( - { - "application": "hiking", - "reputons": [ - { - "rater": "HikingAsylum.example.com", - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rating": 0.90 - } - ] - } - )"); - - // Encode a basic_json value to a CBOR value - std::vector data; - cbor::encode_cbor(j1, data); - - // Decode a CBOR value to a basic_json value - ojson j2 = cbor::decode_cbor(data); - std::cout << "(1)\n" << pretty_print(j2) << "\n\n"; - - // Accessing the data items - - const ojson& reputons = j2["reputons"]; - - std::cout << "(2)\n"; - for (auto element : reputons.array_range()) - { - std::cout << element.at("rated").as() << ", "; - std::cout << element.at("rating").as() << "\n"; - } - std::cout << std::endl; - - // Get a CBOR value for a nested data item with jsonpointer - std::error_code ec; - const auto& rated = jsonpointer::get(j2, "/reputons/0/rated", ec); - if (!ec) - { - std::cout << "(3) " << rated.as_string() << "\n"; - } - - std::cout << std::endl; -} -``` -Output: -``` -(1) -{ - "application": "hiking", - "reputons": [ - { - "rater": "HikingAsylum.example.com", - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rating": 0.9 - } - ] -} - -(2) -Marilyn C, 0.9 - -(3) Marilyn C -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor_decode_options.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor_decode_options.md deleted file mode 100644 index ade07e2c12..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor_decode_options.md +++ /dev/null @@ -1,19 +0,0 @@ -### jsoncons::cbor::cbor_decode_options - -An abstract class that defines accessors for CBOR decoding options. - -#### Header -```c++ -#include -``` - -#### Implementing classes - -[cbor_options](cbor_options.md) - -#### Destructor - - virtual ~cbor_decode_options(); - -#### Accessors - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor_encode_options.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor_encode_options.md deleted file mode 100644 index 5ecffd0980..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor_encode_options.md +++ /dev/null @@ -1,20 +0,0 @@ -### jsoncons::cbor::cbor_encode_options - -An abstract class that defines accessors for CBOR encoding options. - -#### Header -```c++ -#include -``` - -#### Implementing classes - -[cbor_options](cbor_options.md) - -#### Destructor - - virtual ~cbor_encode_options(); - -#### Accessors - - bool pack_strings() const; diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor_encoder.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor_encoder.md deleted file mode 100644 index 5cf48cf87f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor_encoder.md +++ /dev/null @@ -1,197 +0,0 @@ -### jsoncons::cbor::basic_cbor_encoder - -```c++ -template< - class Result> -> class basic_cbor_encoder : public jsoncons::json_content_handler -``` - -`basic_cbor_encoder` is noncopyable - -#### Header - - #include - -![cbor_encoder](./diagrams/cbor_encoder.png) - -Four specializations for common result types are defined: - -Type |Definition ----------------------------|------------------------------ -cbor_encoder |basic_cbor_encoder -cbor_bytes_encoder |basic_cbor_encoder - -#### Member types - -Type |Definition ----------------------------|------------------------------ -char_type |char -result_type |Result -string_view_type | - -#### Constructors - - explicit basic_cbor_encoder(result_type result) -Constructs a new encoder that writes to the specified result. - -#### Destructor - - virtual ~basic_cbor_encoder() - -### Inherited from [basic_json_content_handler](../json_content_handler.md) - -#### Member functions - - bool begin_object(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool begin_object(size_t length, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool end_object(const ser_context& context = null_ser_context()) - - bool begin_array(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool begin_array(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool end_array(const ser_context& context=null_ser_context()); - - bool name(const string_view_type& name, - const ser_context& context=null_ser_context()); - - bool string_value(const string_view_type& value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool byte_string_value(const byte_string_view& b, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool byte_string_value(const uint8_t* p, size_t size, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool int64_value(int64_t value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool uint64_value(uint64_t value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool double_value(double value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool bool_value(bool value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool null_value(semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - void flush() - -### Examples - -#### Encode to CBOR buffer - -```c++ -#include -#include -#include - -int main() -{ - std::vector buffer; - cbor::cbor_bytes_encoder encoder(buffer); - - encoder.begin_array(); // Indefinite length array - encoder.string_value("cat"); - encoder.byte_string_value(byte_string({'p','u','r','r'})); - encoder.byte_string_value(byte_string({'h','i','s','s'}), - semantic_tag::base64); // suggested conversion to base64 - encoder.int64_value(1431027667, semantic_tag::timestamp); - encoder.end_array(); - encoder.flush(); - - for (auto c : buffer) - { - std::cout << std::hex << std::setprecision(2) << std::setw(2) - << std::noshowbase << std::setfill('0') << static_cast(c); - } - std::cout << "\n\n"; - -/* - 9f -- Start indefinte length array - 63 -- String value of length 3 - 636174 -- "cat" - 44 -- Byte string value of length 4 - 70757272 -- 'p''u''r''r' - d6 - Expected conversion to base64 - 44 - 68697373 -- 'h''i''s''s' - c1 -- Tag value 1 (seconds relative to 1970-01-01T00:00Z in UTC time) - 1a -- 32 bit unsigned integer - 554bbfd3 -- 1431027667 - ff -- "break" -*/ -} -``` -Output: -``` -9f636361744470757272d64468697373c11a554bbfd3ff -``` - -#### Encode to CBOR stream - -```c++ - -#include -#include -#include - -int main() -{ - std::ostringstream os; - cbor::cbor_encoder encoder(os); - - encoder.begin_array(3); // array of length 3 - encoder.string_value("-18446744073709551617", semantic_tag::bigint); - encoder.string_value("184467440737095516.16", semantic_tag::bigdec); - encoder.int64_value(1431027667, semantic_tag::timestamp); - encoder.end_array(); - encoder.flush(); - - for (auto c : os.str()) - { - std::cout << std::hex << std::setprecision(2) << std::setw(2) - << std::noshowbase << std::setfill('0') << (int)unsigned char(c); - } - std::cout << "\n\n"; - -/* - 83 -- array of length 3 - c3 -- Tag 3 (negative bignum) - 49 -- Byte string value of length 9 - 010000000000000000 -- Bytes content - c4 -- Tag 4 (decimal fraction) - 82 -- Array of length 2 - 21 -- -2 (exponent) - c2 Tag 2 (positive bignum) - 49 -- Byte string value of length 9 - 010000000000000000 - c1 -- Tag 1 (seconds relative to 1970-01-01T00:00Z in UTC time) - 1a -- 32 bit unsigned integer - 554bbfd3 -- 1431027667 -*/ -} -``` -Output: -``` -83c349010000000000000000c48221c249010000000000000000c11a554bbfd3 -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor_options.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor_options.md deleted file mode 100644 index 6cae4cc9ce..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/cbor_options.md +++ /dev/null @@ -1,43 +0,0 @@ -### jsoncons::cbor::cbor_options - -Specifies options for encoding and decoding CBOR. - -#### Header -```c++ -#include -``` - -![cbor_options](./diagrams/cbor_options.png) - -#### Constructors - - cbor_options() -Constructs a `cbor_options` with default values. - -#### Modifiers - - cbor_options& pack_strings(bool value) - -If set to `true`, then encode will store text strings and -byte strings once, and use string references to represent repeated occurences -of the strings. Decoding the resulting CBOR requires a decoder -that supports the -[stringref extension to CBOR](http://cbor.schmorp.de/stringref), such as -jsoncons itself, or [Perl CBOR::XS](http://software.schmorp.de/pkg/CBOR-XS.html) - -If set to `false` (the default), then encode -will encode strings the usual CBOR way. - -This option does not affect decode - jsoncons will always decode -string references if present. - -#### Static member functions - - static const cbor_options& default_options() -Default CBOR encode and decode options. - -### See also - -[cbor_decode_options](cbor_decode_options.md) -[cbor_encode_options](cbor_encode_options.md) - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/decode_cbor.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/decode_cbor.md deleted file mode 100644 index 9fd5993c5d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/decode_cbor.md +++ /dev/null @@ -1,240 +0,0 @@ -### jsoncons::cbor::decode_cbor - -Decodes a [Concise Binary Object Representation](http://cbor.io/) data format into a C++ data structure. - -#### Header -```c++ -#include - -template -T decode_cbor(const std::vector& v); // (1) - -template -T decode_cbor(std::istream& is); // (2) -``` - -(1) Reads a CBOR bytes buffer into a type T if T is an instantiation of [basic_json](../json.md) -or if T supports [json_type_traits](../json_type_traits.md). - -(2) Reads a CBOR binary stream into a type T if T is an instantiation of [basic_json](../json.md) -or if T supports [json_type_traits](../json_type_traits.md). - -#### Exceptions - -Throws [ser_error](../ser_error.md) if parsing fails. - -### Examples - -#### Round trip (JSON to CBOR bytes back to JSON) - -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - ojson j1 = ojson::parse(R"( - { - "application": "hiking", - "reputons": [ - { - "rater": "HikingAsylum.example.com", - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rating": 0.90 - } - ] - } - )"); - - std::vector v; - cbor::encode_cbor(j1, v); - - ojson j2 = cbor::decode_cbor(v); - std::cout << pretty_print(j2) << std::endl; -} -``` -Output: -```json -{ - "application": "hiking", - "reputons": [ - { - "rater": "HikingAsylum.example.com", - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rating": 0.9 - } - ] -} -``` - -#### Round trip (JSON to CBOR file back to JSON) - -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - json j = json::parse(R"( - { - "application": "hiking", - "reputons": [ - { - "rater": "HikingAsylum.example.com", - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rating": 0.90 - } - ] - } - )"); - - std::ofstream os; - os.open("./output/store.cbor", std::ios::binary | std::ios::out); - cbor::encode_cbor(j,os); - - std::vector v; - std::ifstream is; - is.open("./output/store.cbor", std::ios::binary | std::ios::in); - - json j2 = cbor::decode_cbor(is); - - std::cout << pretty_print(j2) << std::endl; -} -``` -Output: -```json -{ - "application": "hiking", - "reputons": [ - { - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rater": "HikingAsylum.example.com", - "rating": 0.9 - } - ] -} -``` - -#### Decode CBOR byte string - -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - // byte string of length 5 - std::vector buf = {0x45,'H','e','l','l','o'}; - json j = cbor::decode_cbor(buf); - - auto bs = j.as(); - - // byte_string to ostream displays as hex - std::cout << "(1) "<< bs << "\n\n"; - - // byte string value to JSON text becomes base64url - std::cout << "(2) " << j << std::endl; -} -``` -Output: -``` -(1) 0x480x650x6c0x6c0x6f - -(2) "SGVsbG8" -``` - -#### Decode CBOR byte string with base64 encoding hint - -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - // semantic tag indicating expected conversion to base64 - // followed by byte string of length 5 - std::vector buf = {0xd6,0x45,'H','e','l','l','o'}; - json j = cbor::decode_cbor(buf); - - auto bs = j.as(); - - // byte_string to ostream displays as hex - std::cout << "(1) "<< bs << "\n\n"; - - // byte string value to JSON text becomes base64 - std::cout << "(2) " << j << std::endl; -} -``` -Output: -``` -(1) 0x480x650x6c0x6c0x6f - -(2) "SGVsbG8=" -``` - -#### Decode packed strings [stringref-namespace, stringref](http://cbor.schmorp.de/stringref) - -This example taken from [CBOR stringref extension](http://cbor.schmorp.de/stringref) shows three stringref-namespace tags, -with two nested inside another: - -```c++ -int main() -{ - std::vector v = {0xd9,0x01,0x00, // tag(256) - 0x85, // array(5) - 0x63, // text(3) - 0x61,0x61,0x61, // "aaa" - 0xd8, 0x19, // tag(25) - 0x00, // unsigned(0) - 0xd9, 0x01,0x00, // tag(256) - 0x83, // array(3) - 0x63, // text(3) - 0x62,0x62,0x62, // "bbb" - 0x63, // text(3) - 0x61,0x61,0x61, // "aaa" - 0xd8, 0x19, // tag(25) - 0x01, // unsigned(1) - 0xd9, 0x01,0x00, // tag(256) - 0x82, // array(2) - 0x63, // text(3) - 0x63,0x63,0x63, // "ccc" - 0xd8, 0x19, // tag(25) - 0x00, // unsigned(0) - 0xd8, 0x19, // tag(25) - 0x00 // unsigned(0) - }; - - ojson j = cbor::decode_cbor(v); - - std::cout << pretty_print(j) << "\n"; -} -``` -Output: -``` -[ - "aaa", - "aaa", - ["bbb", "aaa", "aaa"], - ["ccc", "ccc"], - "aaa" -] -``` - -#### See also - -- [byte_string](../byte_string.md) -- [encode_cbor](encode_cbor.md) encodes a json value to the [Concise Binary Object Representation](http://cbor.io/) data format. - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/diagrams/cbor_encoder.xml b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/diagrams/cbor_encoder.xml deleted file mode 100644 index 39a9783b45..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/diagrams/cbor_encoder.xml +++ /dev/null @@ -1 +0,0 @@ -3Vffb9owEP5reGRKYmD0sYWu21CrdUyCPiE3vibeHJs5BsL++p2JQ35WbacKTfCC77Pv7Pvu43rtkUmS3Wi6jm8VA9ELPJb1yLQXBL4X+PhlkX2OkEGQA5HmzB0qgTn/A4WnQzecQVo7aJQShq/rYKikhNDUMKq12tWPPSlRv3VNI2gB85CKNrrgzMQ5Oh56Jf4ZeBQXN/ue20locdgBaUyZ2lUgct0jE62UyVdJNgFhySt4yf0+PbN7fJgGaV7jcH33I5Oz+3kcTH7PzOI2eci+9l2ULRUbl/AjTXm4+pkquUJODQZfxVQyAboXjATedPVoV5FddSGHVM2+4E+rjWRgn+Dh9i7mBuZrGtrdHSoGsdgkAi3/6F1NqXgfaANZBXIp3oBKwOg9HnG7Y8e2k5tfsL8ri+ePHBZXCjdwGHV6iY6RS0px4Vh9A8Mfn2E4fFR6lYLmVKDoT0DuExdiooTSB18Chw/iqdHqF1R2yIhcEPY+5XhVPQanrMe4VY8WrSDZpW0daIWCplitOpOQcbN0pNv1g11/GDprmlW2pvvCkPj4ZRHAGhUva5ZuB6vwe7YGqdroEF4WnqE6AvNyCwBWa4TtilYqNuwoWIFpENTwbb19dlXR3fBNcczsKJhBXS8BaeggT9s5VbtdI85FQ3fjRpycllacg6SOSb9KZdH2Ng5md0v6HaL+4n4G2xn0h2eoss5EyVmqZ9AINDqtekZnop5q7+lM9P/qPUOv0TS8f5WP39Chf1r9tGeOLzLGUcNQiT9mHMY5jTRNWqLCP+qmLqP6cCCVhMYk4SAcYiJptYgSwGGGXNkRgeMYfek2Es6YvaZzSqnPMe8wdviN9h94HWMH6VBS8PaxA81yiM8rWP4rRK7/Ag== \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/diagrams/cbor_options.png b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/diagrams/cbor_options.png deleted file mode 100644 index e9bfa09f6314230bd821d85c1d294da91dd5088c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5744 zcmY+IcQ~8<_s1h@#Z&Sa9d>n4#B8Wdv{hS!s1RB+)ZV+HFr~Z`xz)S(B3rN zK4t2(gERs}M|W_@Oa6r#6BF|-w|>8fUth)H3E+W)r^3dWAHnlOY#qu^@|lxQE;O>Y z9$flB53YdW#&`HqC#C1rhwmSGF9eM{g(+>^|8=^^Vf?7A%RkC5dtEeji1Yfc2;N|z zn_eC!D0L~~zK|^|6Wi{f72B@O5eJpr-F)=2*{hRjRe(>P<#;x;?P>BGOtVYb@r#Kv zf4ye{VR6#=mG_M9DKY&RCCn7v9Bw`-adnutLx-2mK9BY~UAZ`;Y?M)^nZ>f*nZ6y4 zbK5DJvQ1Y#IUS!Xd6I9T_A#V^s4+x$^^x;XF|*0!hQ&*iL+NY;%g1BPi|5XF_jk+- zb-tFzCiJYv96fnG*h1ZEmKZtZe9_-OAO%~XpkJ946THfI^~Y4Tm~6t7)-5>C5Q}t< zO<%N3PP3xUU)pLlrTkyJ@4BEct*K13-r$JVi(Lo^hxq+Ntq0R$QKmOc{haFWC!RxghHVO(uFb#2YX7tg;` zjHquF;U0fE>z=t7@1tuII8bl09HvcJ;J#(Rk))k}NlqytUV&_v%649`XPC)DKIp1k zg3J>+5QeKaz17fVX5uAy8NO#N!z^_&laLkU9d+5R$WZ5e!qd{F4w$@2@r@-z#}WSc zLCXt+WhqvU*|=1r_Ya7tU8;g@(v4=S3jET<3}l_d9Meiv7>)M#pfy*ayFzacS7p_@ zc}PJY4dh#6L?R%agx8}(|<2e?C`MD^F_(aSQgj{laRaB3r+LXWCTE zr>4|oeL2h2?Y!LENQ>e-1F>rEk{H3_+pyO5);o+T0a7Y(Tb_X3`ZH(0@v-|%A=*{G z`by`1bi0fE@mzKn&c}Rl!)$BUv_QW8@CavVmBr}3$({z+Fxutt3W;kDEnV#5c{JoJ%pYjkCsKw#{0&xuJ z+|V=)v|h_Lx{cB1Z4+^pe@l#8?!M82OnEC5>vs05!P&3X*9_u@-NBLQkGArc(FNZU z=roTW^(9<#h8qkUT(##E6-$Ml3aCCU9BKS2TAm`~#<{cAHMk!>bFiMfbJDH4pa`{n z>f<;bH0~3ww$uG-?*(!>eOi-2MRarZyvKQV?+WJ*2p>HqkTd$VOJtq0|LQsS7pHoX zOsX(+h|V8>S)36hI#?i(=h%p;O@&P9vPAA1s@A;GU_9!a*vnV!2=SYr-ujl}2}O@9 z;1Y<=fv!RkNrbp+Et|tjJW5oy@;L|sMbowD%@pBLJXP+4SuC)+?fYTZ@Pxgqx}i8c zYTATv{S*tybI;ymoLh)e|A4S8&P*~bo_O+!6OAe)`bG*-bVbZ{5>jZoE{+#7&Jwww zy;&Wo2IC12O1?vZ==JPRNFbuw>R&0}f8tS(FH1yBaiCfBT~bVhDfy;pVRd(ee6a~n zpO$81@br(5uR9(!x{+-@kO^sN{VgqcF>!I$Q5!PbG*3I^~JEHApcx?1vL!rSviZc@YQ`#0c-yeOvH zfX@3mq08hMgpn4?nSKG8rriI0ze_3@+T*!<+AMqN~Gt>W=UYAW{{kGz6{nT<`Z zT1*W17+B6ie7K*V85|BTIXn#Z`>|@Vw>hP?M)9;?5nPx;Lj`SJoSi?#M^yCWs+Y^l z%RkL(hljU)5~ijm$oTF~g)>D%`jrKcmkG;L6O)mzg%c7HFqCbS3aA^pPyz z?LI=@xkJ>X!_paohS%504US=9n7!TIMgQxdr%Ovqk$QqbcfX!tR}zY%53{cIzWFd= z3!1yTVTGo9VrgkvT32V;9&qx0Za;WF%pzA@_V-{Ei>{@HIIn5t99J4~nt>@o?d_qI5v^pi;oQ))4MefPY_&1?; zW7g-Bk%VGv1u4+0-eRM&iH9c`egT0}^^+r$_q*TpmitOfmgmxJP@WMRA5I#AkM{;= zs4b>F5B6<%DB%=gO=UEB64V57zlJ=jDaPw#9g-xGTDMA&{d*+DIEbMp2Mr-&goDeUBUKlt(IN01vtpxWu(bZ}S+ zW~|mnTWY8)<8fvNkU&S1KWx4Xo({iMLZ`#WQ73V=s-M!OSj#UmvHix_fc4W&3|loN z4^DEW1@DgqB-RW8_wUAo^~T=DkczC|)&JU}m z>|mdJ?9*TrT4F70bj}Gj&HQcB!rZ$UEJHl%p6l-(g$4-y>3i^`2dxd4ZgYaSdwtY1Me-fMAWe8E~zke1qRUT~3v@e&HHz0dd_p?SMhn_+Lk&Q|~CE67^ zKdd)C7gpo_l~9*LWN7z0`ssDNu+&?OuJvd*xRtFOV5q#YKOWLkoKYVSnMDC_$yoS{ zPlg(;^ct!6>n+%XAI#mJynE&M-k^;y#=p4gQo%)vuB=-t>u_ghkuRpT(Z0WGV7F+cVAwpSV+_2l`6({gxTD>_ZED9At(K+!mF4%a3ttk>`Ocz+^z@R@Ra zPNCDxLx81$;(YuU%s&63q~zOt@JTVq1>;pmuF1aKn`1*a`-nD;K_Ve7YgA@fojW_ClN>6| z!w!jEX=1ZW(Yuc5)I!=~b+N5NvN5rSbTyLogL}Y-bnAyT&{HtKk9PI32X_(%?-pyM zkmG2T;$wIM=HT8nIxgs7H9*Y(o@OOx(wmgOiUkqkeEdUkHzAWBPA1b_@n$Q+6bW6) z@P!Kj>}$Hr1uWVT(ikA7p<&Z<6b+)M^TVzP?1_nqi2*9|K{+0XTXLDQ`kATVu|`Cu zg2ysX+5{T#*F=JSb+8Q3#54i{JWVxT+4hE_t^1xdz=+)M+xR{TGfYx9tp~q-OJ3o> zh3L#bRSESlmpx|?3JLVW;8CGLUUD4&{*KRpO(UKOtcwq7Vuw=0|EWZVuqaMgRc-N- zIVp}NhCpurr2{-;zh!cXVY`+rO01?*OUc&nORrqHVrpSw@yzj|@Au>5u>O&evgnu? zpdF?zZ)})nXJ;>WrOTmOw6PY?7#Z)!frqTjGl*uEmJjq?WWAv1fg(H|wFn$HrW!eC!t;Ro2v^n=|o8ZFP!(a~4`r`v>^n_IdJ5U#(eDKaBF`^$Bh z51Zrbe>z-Z#Qu~g%pXgq>Jz%d^C9R8XS1=s?l?Kn+gqTCqum}HzTF1D5Z}=I~no#Kze{i+6P$ndb5%ca99g&Zlk-~a54iW zkJU7iLk&@2~eX@WG^r4C)-+DTESGX z;rReJw=ccx_a%!NjQxCl7iGE*4!+1`|C8O8t7D#=k0TgFvb=B&cJ3po;R%*S-zB{u zvp(iUU|3bhn!d4`xfb^Fy8k>rnXGXkz)Xy$8=O=5CDBkk!NO=OYm6$6=p4_)(<&7d z;w9`E>TARi;`Uowt50c=JNo_(#o+K!ZlZ&efjR=U}*9Md{h*7OemQ;7hp1t?{)FazA^cee6n&E;r{m=s@{0u4KZY|~g}Af-7zha%F%4`j*OM>5%W$0N z+|cdEheta-{pBBi2j38;ijT|OxvGJ^S>%fN1t;$9E}8gEx3K(a*iq$PG|c5|TW1Ua z*o&Sof<9p(v8(;m7^@F5sVmP7Kh)XV-(R^pTH{r~^|I-lo@oS)2>&lOLPA0w>n5wN zt_+r!&CNN$Wn_q-KYzAOANl>eVR>!Mthu?lw6s)Li&#}Wtf1aMsZ>rYQ(}U3D+7;lUT<;FTW)}cHnBC?1BQ-mUUU{enZQoEv zeo9Ph7$QJw?ldY;HbGYJgU!@i5(QL0OW3`+QRBBl2saiJ6T2n~f;F;ejEs%-556uh zFU!dh7-M5Wz-k{&meMdk||P= zZC)(wf?<4A^NOqhO)2b=6tHeRbR5mX*1WPnC$TPXZ<=Y5O+-{*;oSZvNNMs0qDXKm*mD0nwXi02 zoK+y7edhum#uFBl=O$P|q`H8O()L8bvgKu$)6xQ_LWT^Xnw_Vf?ub-#IvSc^BzJ~h zrz5!c;q-N^rTA)J4OgFc^owF2V+jSHF8IeSIDf-2iyUUV# zx4|%pwL-%yCEiLoqS*tjacAN7thI<8dG8fUmmu@M&1LI_^p1BOFm?6Wj)-z?o`&@* z=8-Q{mnjFVFBYX($a$FneFf&j!i2q@txt#<$|6*~y;9MMl(IDF%ra@nAXt=r-WNj< zGtMN(1)q0u)_i~-%8OFyGe~&UNBKFDnIFhCRBXu*s--tdSnZn|R=t*HaNy5AhG3m> z9AOCgeXx*N-)^r2YlQtfX2d_vZu+&qQP!Cz^}y{5hKA77@=;uN95ng=^9E&po3?hb z$^Y$DC^P!4!Aq_gFUyX14(?V0!^w|#+wSvI^cQCI<7@(;+QG+d2H4hT{7;`Tq$xJ(V1ga2k57f-B}C`^YT-qBggAc? zn;QX-!Xhg#<^ty}YEgPC4tNw=&ZDOuPFZD%d6p6hK7uwPJCNRXN9f4ws=4s$0 O8KiU5@CF%iFY-TMA{b); diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/diagrams/cbor_options.xml b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/diagrams/cbor_options.xml deleted file mode 100644 index bbde4a1822..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/diagrams/cbor_options.xml +++ /dev/null @@ -1 +0,0 @@ -zZbLbtswEEW/RssUetSPbWOnySItghpo05XAimOJjUQqFBVJ/fqS0VBPJ04Lo/XG4FxyhuSZa0JOsMnqa0ny5JOgkDq+S2sn2Dq+v/I9/WuEphWCJQqxZLSVBsKO/QIUXVRLRqEYLVRCpIrlYzESnEOkRhqRUlTjZXuRjnfNSQwzYReRdK5+Y1QlrbpeuL1+AyxO7M6eizMZsYtRKBJCRTWQgisn2EghVDvK6g2khp3l0uZ9fGG2O5gErt6SsP/a7L/kDzeP4nP4uA63l+Vtc4FVnkha4oV/kIJF4c9C8JBCpNsZilwxwQu8hWosGilKTsFUd53gskqYgl1OIjNbaS9oLVFZqiNPD+entVuDVFAPJDz9NYgMlGz0Epx9jyDRSWsMq74t3hK1ZNASm0bQCXFXuIelB8jrD9j5r7EDfs7sOlD/DV7wGrxzoeYvJ9j8A9j8f4ltOcM2gwScfjAPn46ilBQa6pgL1EzdI0Iz/m7G7xYYbevB1LaxAdeHv7cFTDDIMmGf9hzZvBd7UIhSRnDcH4rIGNTxPyHQ0TM+7+igY4sDDbOahJQo9jR+/A91EXe4E0zfrDNMZ4ZmYiBbor03Zg0f62mh1aSQNynUgpkVejZVd+2/99nqBD47tWeOesE7Ky/MHo/po/BWL6wnVvBPZQUd9p8f7fL+Gy64+g0= \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/encode_cbor.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/encode_cbor.md deleted file mode 100644 index 4e97ec4f44..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/cbor/encode_cbor.md +++ /dev/null @@ -1,273 +0,0 @@ -### jsoncons::cbor::encode_cbor - -Encodes a C++ data structure to the [Concise Binary Object Representation](http://cbor.io/) data format. - -#### Header -```c++ -#include - -template -void encode_cbor(const T& val, std::vector& buffer, - const cbor_encode_options& options = cbor_options::default_options()); // (1) - -template -void encode_cbor(const T& val, std::ostream& os, - const cbor_encode_options& options = cbor_options::default_options()); // (2) -``` - -(1) Writes a value of type T into a bytes buffer in the CBOR data format. Type T must be an instantiation of [basic_json](../json.md) -or support [json_type_traits](../json_type_traits.md). Uses the [encode options](cbor_options.md) -supplied or defaults. - -(2) Writes a value of type T into a binary stream in the CBOR data format. Type T must be an instantiation of [basic_json](../json.md) -or support [json_type_traits](../json_type_traits.md). Uses the [encode options](cbor_options.md) -supplied or defaults. - -#### See also - -- [decode_cbor](decode_cbor) decodes a [Concise Binary Object Representation](http://cbor.io/) data format to a json value. - -### Examples - -#### cbor example - -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - ojson j1; - j1["zero"] = 0; - j1["one"] = 1; - j1["two"] = 2; - j1["null"] = null_type(); - j1["true"] = true; - j1["false"] = false; - j1["max int64_t"] = (std::numeric_limits::max)(); - j1["max uint64_t"] = (std::numeric_limits::max)(); - j1["min int64_t"] = (std::numeric_limits::lowest)(); - j1["max int32_t"] = (std::numeric_limits::max)(); - j1["max uint32_t"] = (std::numeric_limits::max)(); - j1["min int32_t"] = (std::numeric_limits::lowest)(); - j1["max int16_t"] = (std::numeric_limits::max)(); - j1["max uint16_t"] = (std::numeric_limits::max)(); - j1["min int16_t"] = (std::numeric_limits::lowest)(); - j1["max int8_t"] = (std::numeric_limits::max)(); - j1["max uint8_t"] = (std::numeric_limits::max)(); - j1["min int8_t"] = (std::numeric_limits::lowest)(); - j1["max double"] = (std::numeric_limits::max)(); - j1["min double"] = (std::numeric_limits::lowest)(); - j1["max float"] = (std::numeric_limits::max)(); - j1["zero float"] = 0.0; - j1["min float"] = (std::numeric_limits::lowest)(); - j1["Key too long for small string optimization"] = "String too long for small string optimization"; - - std::vector v; - cbor::encode_cbor(j1, v); - - ojson j2 = cbor::decode_cbor(v); - - std::cout << pretty_print(j2) << std::endl; -} -``` -Output: -```json -{ - "zero": 0, - "one": 1, - "two": 2, - "null": null, - "true": true, - "false": false, - "max int64_t": 9223372036854775807, - "max uint64_t": 18446744073709551615, - "min int64_t": -9223372036854775808, - "max int32_t": 2147483647, - "max uint32_t": 4294967295, - "min int32_t": -2147483648, - "max int16_t": 32767, - "max uint16_t": 65535, - "min int16_t": -32768, - "max int8_t": 127, - "max uint8_t": 255, - "min int8_t": -128, - "max double": 1.79769313486232e+308, - "min double": -1.79769313486232e+308, - "max float": 3.40282346638529e+038, - "zero float": 0.0, - "min float": -3.40282346638529e+038, - "Key too long for small string optimization": "String too long for small string optimization" -} -``` - -#### Encode CBOR byte string - -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - // construct byte string value - json j(byte_string("Hello")); - - std::vector buf; - cbor::encode_cbor(j, buf); - - std::cout << std::hex << std::showbase << "(1) "; - for (auto c : buf) - { - std::cout << (int)c; - } - std::cout << std::dec << "\n\n"; - - json j2 = cbor::decode_cbor(buf); - std::cout << "(2) " << j2 << std::endl; -} -``` -Output: -``` -(1) 0x450x480x650x6c0x6c0x6f - -(2) "SGVsbG8" -``` - -#### Encode byte string with encoding hint - -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - // construct byte string value - json j1(byte_string("Hello"), semantic_tag::base64); - - std::vector buf; - cbor::encode_cbor(j1, buf); - - std::cout << std::hex << std::showbase << "(1) "; - for (auto c : buf) - { - std::cout << (int)c; - } - std::cout << std::dec << "\n\n"; - - json j2 = cbor::decode_cbor(buf); - std::cout << "(2) " << j2 << std::endl; -} -``` -Output: -``` -(1) 0xd60x450x480x650x6c0x6c0x6f - -(2) "SGVsbG8=" -``` - -#### Encode packed strings [stringref-namespace, stringref](http://cbor.schmorp.de/stringref) - -This example taken from [CBOR stringref extension](http://cbor.schmorp.de/stringref) shows how to encode a -data structure that contains many repeated strings more efficiently. - -```c++ -#include -#include -#include -#include - -using namespace jsoncons; - -int main() -{ - ojson j = ojson::parse(R"( -[ - { - "name" : "Cocktail", - "count" : 417, - "rank" : 4 - }, - { - "rank" : 4, - "count" : 312, - "name" : "Bath" - }, - { - "count" : 691, - "name" : "Food", - "rank" : 4 - } - ] -)"); - - cbor::cbor_options options; - options.pack_strings(true); - std::vector buf; - - cbor::encode_cbor(j, buf, options); - - for (auto c : buf) - { - std::cout << std::hex << std::setprecision(2) << std::setw(2) - << std::noshowbase << std::setfill('0') << static_cast(c); - } - std::cout << "\n"; - -/* - d90100 -- tag (256) - 83 -- array(3) - a3 -- map(3) - 64 -- text string (4) - 6e616d65 -- "name" - 68 -- text string (8) - 436f636b7461696c -- "Cocktail" - 65 -- text string (5) - 636f756e74 -- "count" - 1901a1 -- unsigned(417) - 64 -- text string (4) - 72616e6b -- "rank" - 04 -- unsigned(4) - a3 -- map(3) - d819 -- tag(25) - 03 -- unsigned(3) - 04 -- unsigned(4) - d819 -- tag(25) - 02 -- unsigned(2) - 190138 -- unsigned(312) - d819 -- tag(25) - 00 -- unsigned(0) - 64 -- text string(4) - 42617468 -- "Bath" - a3 -- map(3) - d819 -- tag(25) - 02 -- unsigned(2) - 1902b3 -- unsigned(691) - d819 -- tag(25) - 00 -- unsigned(0) - 64 - text string(4) - 466f6f64 -- "Food" - d819 -- tag(25) - 03 -- unsigned(3) - 04 -- unsigned(4) -*/ - - ojson j2 = cbor::decode_cbor(buf); - assert(j2 == j); -} -``` -Output: -``` -d9010083a3646e616d6568436f636b7461696c65636f756e741901a16472616e6b04a3d8190304d81902190138d819006442617468a3d819021902b3d8190064466f6f64d8190304 -``` - -#### See also - -- [byte_string](../byte_string.md) -- [decode_cbor](decode_cbor.md) decodes a [Concise Binary Object Representation](http://cbor.io/) data format to a json value. - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/chars_format.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/chars_format.md deleted file mode 100644 index 3db7e79c19..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/chars_format.md +++ /dev/null @@ -1,17 +0,0 @@ -### jsoncons::chars_format - -```c++ -#if !defined(JSONCONS_NO_TO_CHARS) -using chars_format = std::chars_format; -#else -enum class chars_format : uint8_t {fixed=1,scientific=2,hex=4,general=fixed|scientific}; -#endif -``` - -#### Header -```c++ -#include -``` - -A bitmask type used to specify floating-point formatting, typedefed to [std::chars_format](http://en.cppreference.com/w/cpp/utility/chars_format) if `JSONCONS_NO_TO_CHARS` is undefined. - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv.md deleted file mode 100644 index 8e98626e22..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv.md +++ /dev/null @@ -1,175 +0,0 @@ -### csv extension - -The csv extension implements decode from and encode to the [CSV format](https://www.rfc-editor.org/rfc/rfc4180.txt) -You can either parse into or serialize from a variant-like structure, [basic_json](../json.md), or your own -data structures, using [json_type_traits](../json_type_traits.md). - -[decode_csv](decode_csv.md) - -[encode_csv](encode_csv.md) - -[csv_options](csv_options.md) - -[csv_reader](csv_reader.md) - -[csv_encoder](csv_encoder.md) - -### Examples - -#### Decode a CSV source to a basic_json value - -Example file (sales.csv) -```csv -customer_name,has_coupon,phone_number,zip_code,sales_tax_rate,total_amount -"John Roe",true,0272561313,01001,0.05,431.65 -"Jane Doe",false,416-272-2561,55416,0.15,480.70 -"Joe Bloggs",false,"4162722561","55416",0.15,300.70 -"John Smith",FALSE,NULL,22313-1450,0.15,300.70 -``` - -```c++ -#include -#include -#include - -using namespace jsoncons; - -int main() -{ - csv::csv_options options; - options.assume_header(true) - .mapping(csv::mapping_type::n_objects); - - std::ifstream is1("input/sales.csv"); - ojson j1 = csv::decode_csv(is1,options); - std::cout << "\n(1)\n"<< pretty_print(j1) << "\n"; - - options.mapping(csv::mapping_type::n_rows); - std::ifstream is2("input/sales.csv"); - ojson j2 = csv::decode_csv(is2,options); - std::cout << "\n(2)\n"<< pretty_print(j2) << "\n"; - - options.mapping(csv::mapping_type::m_columns); - std::ifstream is3("input/sales.csv"); - ojson j3 = csv::decode_csv(is3,options); - std::cout << "\n(3)\n"<< pretty_print(j3) << "\n"; -} -``` -Output: -```json -(1) -[ - { - "customer_name": "John Roe", - "has_coupon": true, - "phone_number": "0272561313", - "zip_code": "01001", - "sales_tax_rate": 0.05, - "total_amount": 431.65 - }, - { - "customer_name": "Jane Doe", - "has_coupon": false, - "phone_number": "416-272-2561", - "zip_code": 55416, - "sales_tax_rate": 0.15, - "total_amount": 480.7 - }, - { - "customer_name": "Joe Bloggs", - "has_coupon": false, - "phone_number": "4162722561", - "zip_code": "55416", - "sales_tax_rate": 0.15, - "total_amount": 300.7 - }, - { - "customer_name": "John Smith", - "has_coupon": false, - "phone_number": null, - "zip_code": "22313-1450", - "sales_tax_rate": 0.15, - "total_amount": 300.7 - } -] - -(2) -[ - ["customer_name","has_coupon","phone_number","zip_code","sales_tax_rate","total_amount"], - ["John Roe",true,"0272561313","01001",0.05,431.65], - ["Jane Doe",false,"416-272-2561",55416,0.15,480.7], - ["Joe Bloggs",false,"4162722561","55416",0.15,300.7], - ["John Smith",false,null,"22313-1450",0.15,300.7] -] - -(3) -{ - "customer_name": ["John Roe","Jane Doe","Joe Bloggs","John Smith"], - "has_coupon": [true,false,false,false], - "phone_number": ["0272561313","416-272-2561",4162722561,null], - "zip_code": ["01001",55416,55416,"22313-1450"], - "sales_tax_rate": [0.05,0.15,0.15,0.15], - "total_amount": [431.65,480.7,300.7,300.7] -} -``` - -#### Convert a CSV source to a C++ data structure that satisfies [json_type_traits](../json_type_traits.md) requirements, and back - -```c++ -#include -#include -#include - -using namespace jsoncons; - -int main() -{ - const std::string input = R"(Date,1Y,2Y,3Y,5Y -2017-01-09,0.0062,0.0075,0.0083,0.011 -2017-01-08,0.0063,0.0076,0.0084,0.0112 -2017-01-08,0.0063,0.0076,0.0084,0.0112 -)"; - - csv::csv_options ioptions; - ioptions.header_lines(1) - .mapping(csv::mapping_type::n_rows); - - typedef std::vector> table_type; - - table_type table = csv::decode_csv(input,ioptions); - - std::cout << "(1)\n"; - for (const auto& row : table) - { - std::cout << std::get<0>(row) << "," - << std::get<1>(row) << "," - << std::get<2>(row) << "," - << std::get<3>(row) << "," - << std::get<4>(row) << "\n"; - } - std::cout << "\n"; - - std::string output; - - csv::csv_options ooptions; - ooptions.column_names("Date,1Y,2Y,3Y,5Y"); - csv::encode_csv(table, output, ooptions); - - std::cout << "(2)\n"; - std::cout << output << "\n"; -} -``` -Output: -``` -(1) -2017-01-09,0.0062,0.0075,0.0083,0.011 -2017-01-08,0.0063,0.0076,0.0084,0.011 -2017-01-08,0.0063,0.0076,0.0084,0.011 - -(2) -Date,1Y,2Y,3Y,5Y -2017-01-09,0.0062,0.0075,0.0083,0.011 -2017-01-08,0.0063,0.0076,0.0084,0.0112 -2017-01-08,0.0063,0.0076,0.0084,0.0112 -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_decode_options.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_decode_options.md deleted file mode 100644 index 2125a1797d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_decode_options.md +++ /dev/null @@ -1,67 +0,0 @@ -### jsoncons::csv::csv_decode_options - -```c++ -typedef basic_csv_decode_options csv_decode_options -``` - -An abstract class that defines accessors for CSV decode options. - -#### Header -```c++ -#include -``` - -#### Implementing classes - -[csv_options](csv_options.md) - -#### Accessors - - virtual size_t header_lines() const = 0; - - virtual bool assume_header() const = 0; - - virtual bool ignore_empty_values() const = 0; - - virtual bool ignore_empty_lines() const = 0; - - virtual bool trim_leading() const = 0; - - virtual bool trim_trailing() const = 0; - - virtual bool trim_leading_inside_quotes() const = 0; - - virtual bool trim_trailing_inside_quotes() const = 0; - - virtual bool trim() const = 0; - - virtual bool trim_inside_quotes() const = 0; - - virtual bool unquoted_empty_value_is_null() const = 0; - - virtual std::vector column_names() const = 0; - - virtual std::vector column_types() const = 0; - - virtual std::vector column_defaults() const = 0; - - virtual CharT field_delimiter() const = 0; - - virtual std::pair subfield_delimiter() const = 0; - - virtual string_type line_delimiter() const = 0; - - virtual CharT quote_char() const = 0; - - virtual bool infer_types() const = 0; - - virtual bool lossless_number() const = 0; - - virtual CharT quote_escape_char() const = 0; - - virtual CharT comment_starter() const = 0; - - virtual mapping_type mapping() const = 0; - - virtual unsigned long max_lines() const = 0; - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_encode_options.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_encode_options.md deleted file mode 100644 index 078156975d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_encode_options.md +++ /dev/null @@ -1,33 +0,0 @@ -### jsoncons::csv::csv_encode_options - -An abstract class that defines accessors for CSV encode options. - -#### Header -```c++ -#include -``` - -#### Implementing classes - -[csv_options](csv_options.md) - -#### Accessors - - virtual chars_format floating_point_format() const = 0; - - virtual int precision() const = 0; - - virtual std::vector column_names() const = 0; - - virtual CharT field_delimiter() const = 0; - - virtual std::pair subfield_delimiter() const = 0; - - virtual string_type line_delimiter() const = 0; - - virtual CharT quote_char() const = 0; - - virtual CharT quote_escape_char() const = 0; - - virtual quote_style_type quote_style() const = 0; - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_encoder.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_encoder.md deleted file mode 100644 index 58c41e777c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_encoder.md +++ /dev/null @@ -1,260 +0,0 @@ -### jsoncons::csv::basic_csv_encoder - -```c++ -template< - class CharT, - class Result - class Allocator=std::allocator=std::allocator> -> class basic_csv_encoder : public jsoncons::basic_json_content_handler -``` - -`basic_csv_encoder` and `basic_json_compressed_encoder` are noncopyable and nonmoveable. - -#### Header - - #include - -![csv_encoder](./diagrams/csv_encoder.png) - -Four specializations for common character types and result types are defined: - -Type |Definition ----------------------------|------------------------------ -csv_encoder |basic_csv_encoder> -json_string_encoder |basic_csv_encoder> -wcsv_encoder |basic_csv_encoder> -wjson_string_encoder |basic_csv_encoder> - -#### Member types - -Type |Definition ----------------------------|------------------------------ -char_type |CharT -result_type |Result -string_view_type | - -#### Constructors - - explicit basic_csv_encoder(result_type result) -Constructs a new csv encoder that is associated with the output adaptor `result`. - - basic_csv_encoder(result_type result, - const basic_csv_options& options) -Constructs a new csv encoder that is associated with the output adaptor `result` -and uses the specified [csv options](csv_options.md). - -#### Destructor - - virtual ~basic_csv_encoder() - -### Inherited from [basic_json_content_handler](../json_content_handler.md) - -#### Member functions - - bool begin_object(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool begin_object(size_t length, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool end_object(const ser_context& context = null_ser_context()) - - bool begin_array(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool begin_array(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool end_array(const ser_context& context=null_ser_context()); - - bool name(const string_view_type& name, - const ser_context& context=null_ser_context()); - - bool string_value(const string_view_type& value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool byte_string_value(const byte_string_view& b, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool byte_string_value(const uint8_t* p, size_t size, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool int64_value(int64_t value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool uint64_value(uint64_t value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool double_value(double value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool bool_value(bool value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool null_value(semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - void flush() - -### Examples - -### Serializing an array of json values to a comma delimted file - -#### JSON input file -```json -[ - ["country_code","name"], - ["ABW","ARUBA"], - ["ATF","FRENCH SOUTHERN TERRITORIES, D.R. OF"], - ["VUT","VANUATU"], - ["WLF","WALLIS & FUTUNA ISLANDS"] -] -``` -Note - -- The third array element has a value that contains a comma, in the CSV file this value will be quoted. - -#### Serializing the comma delimited file with csv_encoder -```c++ -std::string in_file = "input/countries.json"; -std::ifstream is(in_file); - -json_decoder decoder; -json_reader reader(is,decoder); -reader.read(); -json countries = decoder.get_result(); - -csv_encoder encoder(std::cout); - -countries.dump(encoder); -``` -#### Output -``` -country_code,name -ABW,ARUBA -ATF,"FRENCH SOUTHERN TERRITORIES, D.R. OF" -VUT,VANUATU -WLF,WALLIS & FUTUNA ISLANDS -``` -### Serializing an array of json objects to a tab delimted file - -#### JSON input file -```json -[ - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - } -] -``` -Note - -- The names in the first object in the array will be used for the header row of the CSV file -- The fourth object has a `note` member whose value contains escaped quotes and an escaped new line character, in the CSV file, this value will be quoted, and it will contain a new line character and escaped quotes. - -#### Dump json value to a tab delimited file -```c++ -std::string in_file = "input/employees.json"; -std::ifstream is(in_file); - -json_decoder decoder; -csv_options options; -options.field_delimiter = '\t'; - -json_reader reader(is,decoder); -reader.read(); -json employees = decoder.get_result(); - -csv_encoder encoder(std::cout,options); - -employees.dump(encoder); -``` -#### Tab delimited output file -``` -dept employee-name employee-no note salary -sales Smith, Matthew 00000001 150,000.00 -sales Brown, Sarah 00000002 89,000.00 -finance Oberc, Scott 00000003 110,000.00 -sales Scott, Colette 00000004 ""Exemplary"" employee -Dependable, trustworthy 75,000.00 -``` - -#### Dump json value to csv file - -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - const json books = json::parse(R"( - [ - { - "title" : "Kafka on the Shore", - "author" : "Haruki Murakami", - "price" : 25.17 - }, - { - "title" : "Women: A Novel", - "author" : "Charles Bukowski", - "price" : 12.00 - }, - { - "title" : "Cutter's Way", - "author" : "Ivan Passer" - } - ] - )"); - - csv_options options; - options.column_names("author,title,price"); - - csv_encoder encoder(std::cout, options); - - books.dump(encoder); -} -``` - -Output: - -```csv -author,title,price -Haruki Murakami,Kafka on the Shore,25.17 -Charles Bukowski,Women: A Novel,12.0 -Ivan Passer,Cutter's Way, -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_options.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_options.md deleted file mode 100644 index 082d702b23..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_options.md +++ /dev/null @@ -1,111 +0,0 @@ -### jsoncons::csv::csv_options - -```c++ -typedef basic_csv_options csv_options -``` -Specifies options for encoding and decoding csv data. The `csv_options` class is an instantiation of the `basic_csv_options` class template that uses `char` as the character type. - -#### Header -```c++ -#include -``` - -![csv_options](./diagrams/csv_options.png) - -#### Constructors - - csv_options() -Constructs an `csv_options` with default values. - -#### Modifiers - basic_csv_options& floating_point_format(chars_format value); -Overrides [floating point format](../chars_format.md) when serializing csv from json. For a floating point value that was previously decoded from json text, preserves the original format when serializing.For a floating point value that was directly inserted into a json value, serializes with [chars_format::general](chars_format.md). - - basic_csv_options& precision(int value); -Overrides floating point precision when serializing csv from json. The default, For a floating point value that was previously decoded from json text, preserves the original precision. The fefault, For a floating point value that was directly inserted into a json value, serializes with shortest representation. - - basic_csv_options& header_lines(size_t value); -Number of header lines in the CSV text. Defaults to 1 if assume_header is true, otherwise 0 - - basic_csv_options& assume_header(bool value); -Assume first row in file is header, use field names to construct objects. Default is `false`. - - basic_csv_options& ignore_empty_values(bool value); -Do not read CSV fields that have empty values. Default is `false`. - - basic_csv_options& ignore_empty_lines(bool value); -If set to true, all lines in the file that are empty (apart from record delimiter characters) are ignored. To ignore lines with only spaces or tabs, set trim to true. Default is `true`. - - basic_csv_options& trim_leading(bool value); -Trim leading whitespace. Default is `false`. - - basic_csv_options& trim_trailing(bool value); -Trim trailing whitespace. Default is `false`. - - basic_csv_options& trim(bool value); -Trim both leading and trailing whitespace. Default is `false`. - - basic_csv_options& trim_leading_inside_quotes(bool value); -Trim leading whitespace inside quote characters. Default is `false`. - - basic_csv_options& trim_trailing_inside_quotes(bool value); -Trim trailing whitespace inside quote characters. Default is `false`. - - basic_csv_options& trim_inside_quotes(bool value); -Trim both leading and trailing whitespace inside quote characters. Default is `false`. - - basic_csv_options& unquoted_empty_value_is_null(bool value); -Replace empty field with json null value. Default is `false`. - - basic_csv_options& column_names(const string_type& names); -A comma separated list of names corresponding to the fields in the file. Example: "bool-field,float-field,string-field" - - basic_csv_options& column_types(const string_type& types); -A comma separated list of data types corresponding to the columns in the file. The following data types are supported: string, integer, float and boolean. Example: "bool,float,string"} - - basic_csv_options& column_defaults(const string_type& defaults); -A comma separated list of strings containing default json values corresponding to the columns in the file. Example: "false,0.0,"\"\"" - - basic_csv_options& field_delimiter(CharT value); -A delimiter character that indicates the end of a field. Default is `,` - - basic_csv_options& subfield_delimiter(CharT value); -A delimiter character that indicates the end of a single value in a multi-valued field. Default is no multi-valued fields. - - basic_csv_options& line_delimiter(string_type value); -String to write between records. Default is \n. - - basic_csv_options& quote_char(CharT value); -Quote character. Default is quote character " - - basic_csv_options& infer_types(bool value); -Infer null, true, false, integers and floating point values in the CSV source. Default is `true`. - - basic_csv_options& lossless_number(bool value); -If set to `true`, parse numbers with exponents and fractional parts as strings with semantic tagging `semantic_tag::bigdec`. Default is `false`. - - basic_csv_options& quote_escape_char(CharT value); -Character to escape quote character (by default the quote character is doubled). Default is quote character `"`. - - basic_csv_options& comment_starter(CharT value); -Character to comment out a line, must be at column 1. Default is no comments. - - basic_csv_options& quote_style(quote_style_type value); -quote_style_type::all, quote_style_type::minimal, quote_style_type::none, or quote_style_type::nonnumeric. Default is quote_style_type::minimal - - basic_csv_options& mapping(mapping_type value); -mapping_type::n_rows, mapping_type::n_objects, mapping_type::m_columns. If assume_header is true or column_names is not empty, defaults to mapping_type::n_rows, otherwise mapping_type::n_columns. - - basic_csv_options& max_lines(unsigned long value); -Maximum number of lines to read. Default is unlimited. - -#### Static member functions - - static const basic_csv_options& default_options() -Default CSV encode and decode options. - -### See also - -[csv_decode_options](csv_decode_options.md) -[csv_encode_options](csv_encode_options.md) - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_reader.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_reader.md deleted file mode 100644 index a5cd2fa706..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/csv_reader.md +++ /dev/null @@ -1,432 +0,0 @@ -### jsoncons::csv::csv_reader - -```c++ -typedef basic_csv_reader> csv_reader -``` - -The `csv_reader` class is an instantiation of the `basic_csv_reader` class template that uses `char` as the character type -and `stream_source` as the input source. It reads a [CSV file](http://tools.ietf.org/html/rfc4180) and produces JSON parse events. - -`csv_reader` is noncopyable and nonmoveable. - -#### Member types - -Type |Definition ----------------------------|------------------------------ -char_type |CharT -source_type |Src - -#### Header -```c++ -#include -``` -#### Constructors - - template - basic_csv_reader(Source&& source, - basic_json_content_handler& handler); // (1) - - - template - basic_csv_reader(Source&& source, - basic_json_content_handler& handler, - const basic_csv_options& options); // (2) - - template - basic_csv_reader(Source&& source, - basic_json_content_handler& handler, - parse_error_handler& err_handler); // (3) - - template - basic_csv_reader(Source&& source, - basic_json_content_handler& handler, - const basic_csv_decode_options& options, - parse_error_handler& err_handler); // (4) - -(1) Constructs a `csv_reader` that reads from a character sequence or stream `source` -and a [json_content_handler](../json_content_handler.md) that receives -JSON events. Uses default [csv_options](csv_options.md). - -(2) Constructs a `csv_reader` that that reads from a character sequence or stream `source`, a [json_content_handler](../json_content_handler.md) that receives -JSON events, and [csv_options](csv_options.md). - -(3) Constructs a `csv_reader` that reads from a character sequence or stream `source`, a [json_content_handler](../json_content_handler.md) that receives -JSON events and the specified [parse_error_handler](../parse_error_handler.md). -Uses default [csv_options](csv_options.md). - -(4) Constructs a `csv_reader` that reads from a character sequence or stream `source`, a [json_content_handler](../json_content_handler.md) that receives -JSON events, [csv_options](csv_options.md), -and the specified [parse_error_handler](../parse_error_handler.md). - -Note: It is the programmer's responsibility to ensure that `basic_csv_reader` does not outlive any source, -content handler, and error handler passed in the constuctor, as `basic_csv_reader` holds pointers to but does not own these resources. - -#### Parameters - -`source` - a value from which a `jsoncons::basic_string_view` is constructible, -or a value from which a `source_type` is constructible. In the case that a `jsoncons::basic_string_view` is constructible -from `source`, `source` is dispatched immediately to the parser. Otherwise, the `csv_reader` reads from a `source_type` in chunks. - -#### Member functions - - bool eof() const -Returns `true` when there is no more data to be read from the stream, `false` otherwise - - void read() -Reports JSON related events for JSON objects, arrays, object members and array elements to a [json_content_handler](../json_content_handler.md), such as a [json_decoder](json_decoder.md). -Throws [ser_error](../ser_error.md) if parsing fails. - - size_t buffer_length() const - - void buffer_length(size_t length) - -### Examples - -#### Reading a comma delimted file into an array of json values - -#### Comma delimited input file -``` -country_code,name -ABW,ARUBA -ATF,"FRENCH SOUTHERN TERRITORIES, D.R. OF" -VUT,VANUATU -WLF,WALLIS & FUTUNA ISLANDS -``` -Note - -- The first record contains a header line, but we're going to ignore that and read the entire file as an array of arrays. -- The third record has a field value that contains an embedded comma, so it must be quoted. - -#### Code -```c++ -std::string in_file = "countries.csv"; -std::ifstream is(in_file); - -json_decoder decoder; - -csv_reader reader(is,decoder); -reader.read(); -json countries = decoder.get_result(); - -std::cout << pretty_print(countries) << std::endl; -``` -#### Output -```json -[ - ["country_code","name"], - ["ABW","ARUBA"], - ["ATF","FRENCH SOUTHERN TERRITORIES, D.R. OF"], - ["VUT","VANUATU"], - ["WLF","WALLIS & FUTUNA ISLANDS"] -] -``` -#### Reading a tab delimted file into an array of json objects - -#### Tab delimited input file -``` -employee-no employee-name dept salary note -00000001 Smith, Matthew sales 150,000.00 -00000002 Brown, Sarah sales 89,000.00 -00000003 Oberc, Scott finance 110,000.00 -00000004 Scott, Colette sales 75,000.00 """Exemplary"" employee -Dependable, trustworthy" -``` -Note - -- The first record is a header line, which will be used to associate data values with names -- The fifth record has a field value that contains embedded quotes and a new line character, so it must be quoted and the embedded quotes escaped. - -#### Code -```c++ -std::string in_file = "employees.txt"; -std::ifstream is(in_file); - -json_decoder decoder; -csv_options options; -options.field_delimiter = '\t' - .assume_header = true; - -csv_reader reader(is,decoder,options); -reader.read(); -json employees = decoder.get_result(); - -std::cout << pretty_print(employees) << std::endl; -``` - -#### Output - -```json -[ - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "note":"", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "salary":"75,000.00" - } -] -``` - -#### Reading the comma delimited file as an array of objects with user supplied columns names - -Note - -- The first record contains a header line, but we're going to ignore that and use our own names for the fields. - -#### Code -```c++ -std::string in_file = "countries.csv"; -std::ifstream is(in_file); - -json_decoder decoder; - -csv_options options; -options.column_names("Country Code,Name") - .header_lines(1); - -csv_reader reader(is,decoder,options); -reader.read(); -json countries = decoder.get_result(); - -std::cout << pretty_print(countries) << std::endl; -``` - -#### Output -```json -[ - { - "Country Code":"ABW", - "Name":"ARUBA" - }, - { - "Country Code":"ATF", - "Name":"FRENCH SOUTHERN TERRITORIES, D.R. OF" - }, - { - "Country Code":"VUT", - "Name":"VANUATU" - }, - { - "Country Code":"WLF", - "Name":"WALLIS & FUTUNA ISLANDS" - } -] -``` - - -#### Reading a comma delimited file with different mapping options - -#### Input - -```csv -Date,1Y,2Y,3Y,5Y -2017-01-09,0.0062,0.0075,0.0083,0.011 -2017-01-08,0.0063,0.0076,0.0084,0.0112 -2017-01-08,0.0063,0.0076,0.0084,0.0112 -``` - -#### Code - -```c++ -json_decoder decoder; -csv_options options; -options.assume_header(true) - .column_types("string,float,float,float,float"); - -options.mapping(mapping_type::n_rows); -std::istringstream is1("bond_yields.csv"); -csv_reader reader1(is1,decoder,options); -reader1.read(); -ojson val1 = decoder.get_result(); -std::cout << "\n(1)\n"<< pretty_print(val1) << "\n"; - -options.mapping(mapping_type::n_objects); -std::istringstream is2("bond_yields.csv"); -csv_reader reader2(is2,decoder,options); -reader2.read(); -ojson val2 = decoder.get_result(); -std::cout << "\n(2)\n"<< pretty_print(val2) << "\n"; - -options.mapping(mapping_type::m_columns); -std::istringstream is3("bond_yields.csv"); -csv_reader reader3(is3, decoder, options); -reader3.read(); -ojson val3 = decoder.get_result(); -std::cout << "\n(3)\n" << pretty_print(val3) << "\n"; -``` - -#### Output - -```json -(1) -[ - ["Date","1Y","2Y","3Y","5Y"], - ["2017-01-09",0.0062,0.0075,0.0083,0.011], - ["2017-01-08",0.0063,0.0076,0.0084,0.0112], - ["2017-01-08",0.0063,0.0076,0.0084,0.0112] -] - -(2) -[ - { - "Date": "2017-01-09", - "1Y": 0.0062, - "2Y": 0.0075, - "3Y": 0.0083, - "5Y": 0.011 - }, - { - "Date": "2017-01-08", - "1Y": 0.0063, - "2Y": 0.0076, - "3Y": 0.0084, - "5Y": 0.0112 - }, - { - "Date": "2017-01-08", - "1Y": 0.0063, - "2Y": 0.0076, - "3Y": 0.0084, - "5Y": 0.0112 - } -] - -(3) -{ - "Date": ["2017-01-09","2017-01-08","2017-01-08"], - "1Y": [0.0062,0.0063,0.0063], - "2Y": [0.0075,0.0076,0.0076], - "3Y": [0.0083,0.0084,0.0084], - "5Y": [0.011,0.0112,0.0112] -} -``` - -#### Convert CSV to json when last column repeats - -```c++ -int main() -{ - const std::string bond_yields = R"(Date,Yield -2017-01-09,0.0062,0.0075,0.0083,0.011,0.012 -2017-01-08,0.0063,0.0076,0.0084,0.0112,0.013 -2017-01-08,0.0063,0.0076,0.0084,0.0112,0.014 -)"; - - // array of arrays - json_decoder decoder1; - csv_options options1; - options1.header_lines(1); - options1.assume_header(false); - options1.column_types("string,float*"); - std::istringstream is1(bond_yields); - csv_reader reader1(is1, decoder1, options1); - reader1.read(); - ojson val1 = decoder1.get_result(); - std::cout << "\n(1)\n" << pretty_print(val1) << "\n"; - - // array of objects - json_decoder decoder2; - csv_options options2; - options2.assume_header(true); - options2.column_types("string,[float*]"); - std::istringstream is2(bond_yields); - csv_reader reader2(is2, decoder2, options2); - reader2.read(); - ojson val2 = decoder2.get_result(); - std::cout << "\n(2)\n" << pretty_print(val2) << "\n"; -} -``` - -Output: -``` -(1) -[ - ["2017-01-09",0.0062,0.0075,0.0083,0.011,0.012], - ["2017-01-08",0.0063,0.0076,0.0084,0.0112,0.013], - ["2017-01-08",0.0063,0.0076,0.0084,0.0112,0.014] -] - -(2) -[ - { - "Date": "2017-01-09", - "Yield": [0.0062,0.0075,0.0083,0.011,0.012] - }, - { - "Date": "2017-01-08", - "Yield": [0.0063,0.0076,0.0084,0.0112,0.013] - }, - { - "Date": "2017-01-08", - "Yield": [0.0063,0.0076,0.0084,0.0112,0.014] - } -] -``` - -#### Convert CSV to json when last two columns repeat - -```c++ -const std::string holidays = R"(1,CAD,2,UK,3,EUR,4,US + UK,5,US -38719,2-Jan-2006,40179,1-Jan-2010,38719,2-Jan-2006,38719,2-Jan-2006,39448,1-Jan-2008 -38733,16-Jan-2006,40270,2-Apr-2010,38733,16-Jan-2006,38733,16-Jan-2006,39468,21-Jan-2008 -)"; - - json_decoder decoder; - csv_options options; - options.column_types("[integer,string]*"); - - // Default - std::istringstream is1(holidays); - csv_reader reader1(is1, decoder, options); - reader1.read(); - ojson val1 = decoder.get_result(); - std::cout << pretty_print(val1) << "\n"; -``` - -Output: -``` -[ - [ - [1,"CAD"], - [2,"UK"], - [3,"EUR"], - [4,"US + UK"], - [5,"US"] - ], - [ - [38719,"2-Jan-2006"], - [40179,"1-Jan-2010"], - [38719,"2-Jan-2006"], - [38719,"2-Jan-2006"], - [39448,"1-Jan-2008"] - ], - [ - [38733,"16-Jan-2006"], - [40270,"2-Apr-2010"], - [38733,"16-Jan-2006"], - [38733,"16-Jan-2006"], - [39468,"21-Jan-2008"] - ] -] -``` diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/decode_csv.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/decode_csv.md deleted file mode 100644 index caaf623fdc..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/decode_csv.md +++ /dev/null @@ -1,338 +0,0 @@ -### jsoncons::csv::decode_csv - -Decodes a [comma-separated variables (CSV)](https://en.wikipedia.org/wiki/Comma-separated_values) data format into a C++ data structure. - -#### Header -```c++ -#include - -template -T decode_csv(const std::basic_string& s, - const basic_csv_options& options = basic_csv_options::default_options())); // (1) - -template -T decode_csv(std::basic_istream& is, - const basic_csv_options& options = basic_csv_options::default_options())); // (2) -``` - -(1) Reads a CSV string value into a type T if T is an instantiation of [basic_json](../json.md) -or if T supports [json_type_traits](../json_type_traits.md). - -(2) Reads a CSV input stream into a type T if T is an instantiation of [basic_json](../json.md) -or if T supports [json_type_traits](../json_type_traits.md). - -#### Return value - -Returns a value of type `T`. - -#### Exceptions - -Throws [ser_error](ser_error.md) if parsing fails. - -### Examples - -#### Decode a CSV file with type inference (default) - -Example file (sales.csv) -```csv -customer_name,has_coupon,phone_number,zip_code,sales_tax_rate,total_amount -"John Roe",true,0272561313,01001,0.05,431.65 -"Jane Doe",false,416-272-2561,55416,0.15,480.70 -"Joe Bloggs",false,"4162722561","55416",0.15,300.70 -"John Smith",FALSE,NULL,22313-1450,0.15,300.70 -``` - -```c++ -#include -#include -#include - -using namespace jsoncons; - -int main() -{ - csv::csv_options options; - options.assume_header(true); - - options.mapping(csv::mapping_type::n_objects); - std::ifstream is1("input/sales.csv"); - ojson j1 = csv::decode_csv(is1,options); - std::cout << "\n(1)\n"<< pretty_print(j1) << "\n"; - - options.mapping(csv::mapping_type::n_rows); - std::ifstream is2("input/sales.csv"); - ojson j2 = csv::decode_csv(is2,options); - std::cout << "\n(2)\n"<< pretty_print(j2) << "\n"; - - options.mapping(csv::mapping_type::m_columns); - std::ifstream is3("input/sales.csv"); - ojson j3 = csv::decode_csv(is3,options); - std::cout << "\n(3)\n"<< pretty_print(j3) << "\n"; -} -``` -Output: -```json -(1) -[ - { - "customer_name": "John Roe", - "has_coupon": true, - "phone_number": "0272561313", - "zip_code": "01001", - "sales_tax_rate": 0.05, - "total_amount": 431.65 - }, - { - "customer_name": "Jane Doe", - "has_coupon": false, - "phone_number": "416-272-2561", - "zip_code": 55416, - "sales_tax_rate": 0.15, - "total_amount": 480.7 - }, - { - "customer_name": "Joe Bloggs", - "has_coupon": false, - "phone_number": "4162722561", - "zip_code": "55416", - "sales_tax_rate": 0.15, - "total_amount": 300.7 - }, - { - "customer_name": "John Smith", - "has_coupon": false, - "phone_number": null, - "zip_code": "22313-1450", - "sales_tax_rate": 0.15, - "total_amount": 300.7 - } -] - -(2) -[ - ["customer_name","has_coupon","phone_number","zip_code","sales_tax_rate","total_amount"], - ["John Roe",true,"0272561313","01001",0.05,431.65], - ["Jane Doe",false,"416-272-2561",55416,0.15,480.7], - ["Joe Bloggs",false,"4162722561","55416",0.15,300.7], - ["John Smith",false,null,"22313-1450",0.15,300.7] -] - -(3) -{ - "customer_name": ["John Roe","Jane Doe","Joe Bloggs","John Smith"], - "has_coupon": [true,false,false,false], - "phone_number": ["0272561313","416-272-2561",4162722561,null], - "zip_code": ["01001",55416,55416,"22313-1450"], - "sales_tax_rate": [0.05,0.15,0.15,0.15], - "total_amount": [431.65,480.7,300.7,300.7] -} -``` - -#### Decode a CSV string without type inference - -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - std::string s = R"(employee-no,employee-name,dept,salary -00000001,"Smith,Matthew",sales,150000.00 -00000002,"Brown,Sarah",sales,89000.00 -)"; - - csv::csv_options options; - options.assume_header(true) - .infer_types(false); - ojson j = csv::decode_csv(s,options); - - std::cout << pretty_print(j) << std::endl; -} -``` -Output: -```json -[ - { - "employee-no": "00000001", - "employee-name": "Smith,Matthew", - "dept": "sales", - "salary": "150000.00" - }, - { - "employee-no": "00000002", - "employee-name": "Brown,Sarah", - "dept": "sales", - "salary": "89000.00" - } -] -``` - -#### Decode a CSV string with specified types - -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - const std::string s = R"(Date,1Y,2Y,3Y,5Y -2017-01-09,0.0062,0.0075,0.0083,0.011 -2017-01-08,0.0063,0.0076,0.0084,0.0112 -2017-01-08,0.0063,0.0076,0.0084,0.0112 -)"; - - csv::csv_options options; - options.assume_header(true) - .column_types("string,float,float,float,float"); - - // mapping_type::n_objects - options.mapping(csv::mapping_type::n_objects); - ojson j1 = csv::decode_csv(s,options); - std::cout << "\n(1)\n"<< pretty_print(j1) << "\n"; - - // mapping_type::n_rows - options.mapping(csv::mapping_type::n_rows); - ojson j2 = csv::decode_csv(s,options); - std::cout << "\n(2)\n"<< pretty_print(j2) << "\n"; - - // mapping_type::m_columns - options.mapping(csv::mapping_type::m_columns); - ojson j3 = csv::decode_csv(s,options); - std::cout << "\n(3)\n" << pretty_print(j3) << "\n"; -} -``` -Output: -```json -(1) -[ - { - "Date": "2017-01-09", - "1Y": 0.0062, - "2Y": 0.0075, - "3Y": 0.0083, - "5Y": 0.011 - }, - { - "Date": "2017-01-08", - "1Y": 0.0063, - "2Y": 0.0076, - "3Y": 0.0084, - "5Y": 0.0112 - }, - { - "Date": "2017-01-08", - "1Y": 0.0063, - "2Y": 0.0076, - "3Y": 0.0084, - "5Y": 0.0112 - } -] - -(2) -[ - ["Date","1Y","2Y","3Y","5Y"], - ["2017-01-09",0.0062,0.0075,0.0083,0.011], - ["2017-01-08",0.0063,0.0076,0.0084,0.0112], - ["2017-01-08",0.0063,0.0076,0.0084,0.0112] -] - -(3) -{ - "Date": ["2017-01-09","2017-01-08","2017-01-08"], - "1Y": [0.0062,0.0063,0.0063], - "2Y": [0.0075,0.0076,0.0076], - "3Y": [0.0083,0.0084,0.0084], - "5Y": [0.011,0.0112,0.0112] -} -``` -#### Decode a CSV string with multi-valued fields separated by subfield delimiters - -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - const std::string s = R"(calculationPeriodCenters,paymentCenters,resetCenters -NY;LON,TOR,LON -NY,LON,TOR;LON -"NY";"LON","TOR","LON" -"NY","LON","TOR";"LON" -)"; - json_options print_options; - print_options.array_array_line_splits(line_split_kind::same_line); - - csv::csv_options options1; - options1.assume_header(true) - .subfield_delimiter(';'); - - json j1 = csv::decode_csv(s,options1); - std::cout << "(1)\n" << pretty_print(j1,print_options) << "\n\n"; - - csv::csv_options options2; - options2.mapping(csv::mapping_type::n_rows) - .subfield_delimiter(';'); - - json j2 = csv::decode_csv(s,options2); - std::cout << "(2)\n" << pretty_print(j2,print_options) << "\n\n"; - - csv::csv_options options3; - options3.assume_header(true) - .mapping(csv::mapping_type::m_columns) - .subfield_delimiter(';'); - - json j3 = csv::decode_csv(s,options3); - std::cout << "(3)\n" << pretty_print(j3,print_options) << "\n\n"; -} -``` -Output: -```json -(1) -[ - - { - "calculationPeriodCenters": ["NY","LON"], - "paymentCenters": "TOR", - "resetCenters": "LON" - }, - { - "calculationPeriodCenters": "NY", - "paymentCenters": "LON", - "resetCenters": ["TOR","LON"] - }, - { - "calculationPeriodCenters": ["NY","LON"], - "paymentCenters": "TOR", - "resetCenters": "LON" - }, - { - "calculationPeriodCenters": "NY", - "paymentCenters": "LON", - "resetCenters": ["TOR","LON"] - } -] - -(2) -[ - - ["calculationPeriodCenters","paymentCenters","resetCenters"], - [["NY","LON"],"TOR","LON"], - ["NY","LON",["TOR","LON"]], - [["NY","LON"],"TOR","LON"], - ["NY","LON",["TOR","LON"]] -] - -(3) -{ - "calculationPeriodCenters": [["NY","LON"],"NY",["NY","LON"],"NY"], - "paymentCenters": ["TOR","LON","TOR","LON"], - "resetCenters": ["LON",["TOR","LON"],"LON",["TOR","LON"]] -} -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/diagrams/csv_encoder.png b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/diagrams/csv_encoder.png deleted file mode 100644 index 9115c418ab5e29506f100cdea83cca286bec4284..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5268 zcmeI0=T}o*v%qQ3gMxHIPAi=9B7#AxAWb@m5ITfTDAKEl zv;+)Q5C|Ab2>s2w*8O__f_u-0z4kgY`moBLPkao zChcF{_?NV93oKuekx_PrnmzaT2-fy>^nUZy%juOn8QD{_;re2E#)~`J{ycfnTo9@c z^SA7Ky6K|l)ib$ZmvQ)|KM$YQWj8Q*vD{3wZx4L}=95189~HXtNL zrY!iKxhj09{vPFZ|DosvHf4upzO}njP4_U6fBPqBgx%W0AWJ${Ew1~AZk_?m@80~e zf-A};_<10V$ovi6dkrsBja^g~UHng< zU&C!{omJN@oqVH-0csCh7zN)V8n%9gTTwjnULLM1Pn93YWT#{?&bJ28Uj0?%VECHh z72G}eGt$uV2oT>%K|>8R4$-Thh36v*f&Don`UYYe+v8;Ayo}==o+()s zyZ%q}nw(5agq@R!_q5?_cQq=2t%1{eT@{AJb8=+fkMiH`T2jceTj5G%5uL?`QjKY< z71QqXY&@`tCF_2*PY1qqAI{cGn{;f=5#wVPF-azLZMzR6Id0c@+p%LDCiaE7a*AFW zWxT!>H5#+R_4=pxps9eM%-^qNy2s7I+Pa4`$R$Jh@dvlKCh(FQq77ZYL*4O5(I=y| z<}5;^EqE36^VBlUwVko_oN9sWc10)4e9O$aDiF1sO}??mM~bP0_hN17Iy=8gx^srI zFV^LcAOHI;C-E56liNO)U$pzGypu_FdPByCp(>RAH(#oyLD^?C)7YX;5>Q8QgZC)G z_qM_DZ2kv)UZnl#UtGpS0q1di_@if@(sGE9?YTa4Db4(=J^Y`=X%U6)BcB(r^Lc1usrxAd(t48Zf<(`7+V zMLO@S*n6#XuL`3dm39M*5IdAL9o3kY6i*jtAGO5{YVZGe(b2a3C(v65M3zsMsL_`y zN=I9-ws)U*_~nk@ueWFLL;Hoc-qBnzx29sG3Go?Qh9|SLa<+G_zTA)^f2-Waf_J9p z{RdK~wzR||D_Qu@sNIx@#mJKQ9`pA)k^{ZS{f`LzC$RR0O<6B+NzlZkh!VXQ^ZcK{ zx>iTFJ55Y)I^aRNKj?#WT0T+1>563dkg_)*akD!fpjb;QNio#Z@)97@TCUKoEHLRJiP^UZUUls?>>=i95aD5 zn=l$sj`B*Xht=)H>Rak{eRX0ivPc$Wj*3f8XNI4xv|a>Ny5!=Un5Cc|88;Cr;zn$Jml9R>9RL)KObYo zb~8T?QhXe-ieVb#6igyX4+XR+vR1<`&jPI4{L9DkRl#2ja~0pn?#$I>_^b^KFSh#W zV-)>q!U^HK{9&Pnc=rl;@d zIA(u54*}wq+AR|~75H;xJyO#}EiI%~g;D%i=y}yHwhJulxhTKYad~k#c`$3^HH`7< z5)ZGk)=uIM07#28D){|wITSMdyk+5M76#pj$6D6dAS}HQ5OGnZd)58&84v7IV|k$RNd7Y7l{iX3>qBzT zuMlnS@S~}c$LF|a?0P>Coc1l2pga*ce@!9K!z|&*1=_h=MP!dTAww7*sEK zCsnT@h1+QK<-w@LtC6(()rzo#VWFkx+dS?Vc;}lLkMR1s7avZrFl>U)ayNPF_0{=C zGy^F4e6M}KDR@R=ajmB7strP1iQVJ8#ve}RDs6%!4&mi9e z9pjKoDD_ao~1xQKjk$Oa(fc*Zg}qGxZu?qzOO<=RIh5YysGW~|5;#V_a{1$pkU z-MW9z323qNw#;sjCKt%ABX~-jrpzcmkdwE7LHTb){N=B({Qy+SY0NB5--3OTAO1EC ztC)3W*N9m^@T7U*(d)5HC(ezkm!E!4o0Q1`z@1i(G6*lkjDYhi#V3B_+{c@aU|utcmBVQ;B@sW(|R5dglpg*Pp%?+k!A17(yfZsTKAt+_BU z69@I27yUmIY4MFJ>y5TMi{GViQD@UestTZx8O8n+2|V(0sFH@cxAcW;{~X~2hlIXD z8*nwk-#VzXAJ?wA0^s{c5HWfqmE$xPscUQ4SjY2~nvH@eM9~(8aEZK9CN4+Qb$X3% zd@s~EAURLah#c()TT75z46ls0duShhiYh0ffu0u2y8v31c zeEtbc&h5?fq-9cG2Ss@TX#iXRa6<7{qN~h)SIdFwq&54lyvu4lTa?Q8d+O#MV6i3< zl!aIo@vMs9gSIEB(g(j43v?gh(}#G_(hz>i=4EDgl9dpD{W?7U$BgHsXBGt4`0TyE z^1VXA$SHft3u^8X0h3LPc*heWR{c>)b(b%YR69_o;PwBfSlQocrzUDU@Id^NH1cIMIylNmw zMa^@vq%#r4agk?c@8(wuJYp+Ez+i(P)?_WsunYWP2~X}_wGbtxi+!}!sjFN6S(f=J zkZHPp^AP~d{xh!_#9W+hjGw7s*(`J#6r2(_w^9LP4jVfMD?Ofz=mq;kXZX7ia*iRR z7AO>W`*3%m!7mIo-e(C0MP(2=kYzL}o|YRs@e;v>B5IS8H3u_7w~4Ojn;|OT9NI>% ze{D{om&bDl#I>Z1Tugc{_C3tR1fd>oRiXm!T8(C3iQx>*7Bb2+2yI|YXw0j@dALpj z9gbq`?g(<(grT+)9@w@#Bj7L@?%?S>iO_OFOh))ZdOBHG+dJ_wjgA|KKw8lQ7ii(h zC8@xb5Fp45ggBLL{_LUogDnm>Orh0I34D0Q41)8dUP>=8uSNBM+XJigZ;6;Lx}V)} z#fD-&#dLD3r!C-;U-1MU1S+Kg!1MZ;nSz<>fvx>O9S-dvIZjpR;!|l@bM)TW5CURO zwPKmFM325#>p^`{DdCi?WQP1JXLNKK?6XAbTd(@z0J?KXLtyKzF9$K*FFhS2uP;xq zh|GQQg{?H=3(qZ2>UPndCbneDUBM{%E!z9xqxrLeYJkrK3soI9bP2X7u`c*tpqd2G4@j}&H^l{9HYxF1`k{I7U>n}$Em|*7M8n=_%s}3S zBb(|srB77aYx#C89WeFv+c(2FFD@USvCx}Td?@H$_ml#?z8q-#U<-{?lgS}&L8a6D zgL!F!=dn@}dEW`CAym^96@{%0&;<>voGh*pZp?)oWfQ-2{xv}bGWa0IF7!_8X6|p)RVyqPT8s>zbu>=pZ7%3a){^fqPk&#!pyiMy%Q}>14a)k_aatu%kYVggotX zK`Lts{msrM!)khr1;kGg{iYZVk{R69jdx_p;a>Bn?|YTtUaT&!lex*^5@nh=(uX0- zBItjrkse3sPrsl;vg_Q6@%DoxR=AtLM1LfH9YRdYx-1elJxX{llG2e_(f3N zA=#lI;40KT(oA6e9(m^miNqo%_*yZMB%v_sGbNJ6VZ$bjW=oBZiWy(!=uWRoaWj&1 zIk+t6r>(0~;CLmSc-OPAiE%C;sdOT!C`iOd$9vO1l1hgcL~=LLcOp*r|8M`_@kwE; z_vyhatB&Bh|DyLm5>Ni0Us(^G7ZsEOcax2awUDd>$2)Vc{`@GFhFw9jBpn|Y`q}C^nWmRL35A`YKD_ zr-(%Hs^y=OxRmZqe|hRRC-%Hnjn_fmVLwUr%6pSr(O(C2MSPj164rEnU2WIStW>O> z^r_WvbEGGl+H{ah!M8+-XK<=m`y-qcI-d1Vn>2W&f1q74I23yN>{z+);md(rt)a); z9_cm6jU>dw!rQo*K<+ydxSBT_|iaq5JrXA>)Q=J(ZlX~-*jRKf`I zcopyEh@JH9wKixO{}%$B0EthG?^h46HLt>gYEkIJi^=e==};7OadRTqkCvAHLp;_T zBI5~T+vqQob$Hk}Yp{=H!o5%aZ z$@55=ziRYQ;td4!SLW3Kuq)qUg~A=X=5l8b$CSXzH$oV9w2$W zvhsNy|H38$x;^oM6xtC~_<8E(1bVfbow!e$Zwx|FO*jV|z7V`Ir2R%S?+M; zD+O->t^TZj6PaxZtqgMkarQ>O#tAq1_1!xs>je8m1Uzol2B3@PBWLD~WF{5+yAF6W zW?pjUuW(Y65=m{;pb%7}xJ&8?`rWJLq^;mHK;$(g*QRZo^iBi=>2D*M?h`}J>cvVZdb9owFP01eeyUxIXSxxZot6FW65gEfUJufJt4c2zmGAj79bOJTeIkqFSq2ofK91xff5xzrkOAxnl5L/E6exAEWBCHpAzQJIjjq2ik/xtgXwFoGFZAKimpoKgG5vQfWNBN21AChTdRCcEUXftgIjiHRHkYllLs/Gmvgvm7rnEKHWCeYNZFF5SorEJHg7DGvwJNM7dzFNpMjt1kCxQZJmLXgNA0QGMphKpGeTkGZrhzvFR1dyeyx4NJ4Oqcgunjr5LPnuZZPP47U4uH/Ln8fmFX2WK2sRd+wQVNVr8LwVeaU6UXX2WYEwYyiIdM73T7YkapGfUhh6uqveNPig0nYI4Q6vQuowrma5yY7E4bRmOZypmOomN180rufCAVlA3IXvEeRA5K7vUUmx1Ztq3dIsf+rhYvchbMGsJdWgxbv6THlWtK9cCy+g6Gr04wnBTbFfBE98snEPtKGRsLJuShFsHhT+OFkuIPNDJoiK4R+RgpztJi8JlajDpadGgFTm7Ms6GjhOFCK+UzCSVVS0u6GT+b8ZeBjSZlIzXZu4Drwy/dAiZoVJmwLjtEru6kBoXYyATeNp3CMgX1dvsD8R7BrqINxfoEc5gEhhXd+k9nn4p2hx+C6psdDXPp+yVGLR9U17ZFzZeutc51y3ej1joVLZ11DpY6Xvosl6XbhyyePS7xT0gvFk8z2M6gp+O/8QwkVZhr0fSnkOJU4rxjPd1Wyjeb355ccGj1soUwoyk3jtVG0c8JujVNSvVH7MYmckqI2ab3nfBfko9o/LAlZNjT+KjHR/H7G1+H9Se0UrD+HYKm/wE= \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/diagrams/csv_options.png b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/diagrams/csv_options.png deleted file mode 100644 index 19d138a26a037f08a9f78fe60e4fc59ce0516cbc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6333 zcmY*eXE+>T*Iq=zvLT{G3!+3PdP#&ube7#EI*As&Cx|W*gh;d~8#XL!bwUs<5xqpo z>NQ%_=w-EU^1a`W_xm$5*IaY%bLP38x$kqIiGk^BK)@_u5C{a((o{17fk=?RbpXXx z;26VDISK-inFS&q`?|f*@V51I((|yhcLjm;5aX3N1=@3#qW*v}<6BpP?HF&+tI^JU z|HvQN+c`SxIsJ7_DbY8uWUE+e)%8|kmZU`!Xh}`(rT!#FX+Flk2c@HyB z`;oib%ScQ7G`=I88F&?e+Sa4)>F~ z1A3u>(3#cI%slRdVL}f->mzOPUh~Iv;ENL8FIM^5UW(Z8H;xY_+A+W z;3dNM3@lqmKpQ=Uj`$;`)gwC>?} z`>oe<^4<9ZxORYD^!as$Lk86SPkrp)k}`yh5vm4@Y9Z70l|2U){xgmRr4OY=n90WU zX8I%xSj`*1SN>J!pocX1oE9j)7Xkn1a4qGl5gxx*nW|JD=p7x2ZL`A&-2-7$GQ(Zn z_M5k=Ul%pJhZc7&#I1?sIqE;@bYtvFq2GI%!|oZy9D~|P=)0Nmrtv;~Jumim*+S>^ zA8o=NMe*K^0M??6Huo8`82Q5Tn5*Q1c|E_{f^X-yY-otRKEL&6WJzwfVy(NerWSOY(yBRD+!9?G2lkf}U6%m-}+$ioe{oOvsp@c^+9W3DPTpAjXX#jX z7K%6EA@?77@y|ZEYpXCp4F#e&f27p3s4MeJMa$G=)WitPVvf9Ke3$zgj+TF-(mgn( zf_+w|eDFRiNBs|uSdIiRR{T~7P8D*ihBonxm1FfESYqDkd10Dv>Rn+j0H zE{iUvbBLoePG~0|C_@Qb#{=JX88{I=qi+qOupl(c=Z{kMuK;ic+s?9^QO+SxvkC8S z;!NXRwcFLP`8+UR$#T*tXPd$?aWFXrfggeC;{n2x5ZS>((3B)j7a*f)oKttZDmGu( z#&^^tTp7lG`?m4H!NJ<^wz#IN=PbfvVnylc%zg)JNWbkl75s9{EI#DC-ps<{?bK{7 z(F-j2l+__C@@fJMf}@|rsE3cCGZ<~(W_z`WmWF7nNXLxP;riGI8f^KeY;dIhO3kwScg8xobyC@9%IpD&yTwGiX zS@9&?xh%JC4fN@T4!x6f?B^e|G&dLOjvXBxCCts`uyAmgNJ~q9n!aV;@s7i!z#!Y$ z;M08pJ6xrQB;V7{KCy!9?bd~sb>}bSMWvlQJj8a#MMCs9Wo2Z*LUdyI!E5`iehcjo z@>xemN2IE1WJJ@-&uo*Uzq4Y*2E=_zB!0aA^cfs@2anmN66;SB6J`(%WrondTU*9Y zpFSmGK8%l`6ra;u8GV_Ym@oEiDy=JQ&-pA9!8i!b#al zR=I5YrG+pwWMl_dTDPAKvwiXe3e>+UcGZI4#6~xtetUaf6+-)^HSJBNb=c3v#k`5B zsb4>zHlR=_GYyO76K|qG$6s!Z1gCS7ps&X*9%V}pm7edN4(4hQb?B|Hw|ndl4-e;7 zJG;0H2ZEZYsHk3vS&QK*317G(FNgxPw{YTim2fe+E@1=|slYX9QRj^*HY@4+)Enor zK|0~CEVBu$Ut#~J2j=@jY2Al*eWS;U3JSQupdbd<^odH64{2@%_@V6j`ufkp3Yrfw z&J+C%pbL%*E@RL)le4LuHMV!d9#5IZo6@QDu|&Q5J{+*~oeqwJ?yDJdG)fEVNrQnB4gv>APOpK?c^Q6AxyGbns_Y+*Lp~rj9N0?b!9#|&S+rQk^ zX-?b>~9y;ARX~NM=kim3g&{y$>H0u!4^)(CTZM}Se;($HkeKfWa zsAFTxbiBLN6R0t^aeV>PY($$jg-H{$FP!$3-D&X0Ea}%+T(n-@LK~coddSBfdNZ+$ z9YkKE*&j>v(h6M?pb*1HY+U62%sD@{FIwh_uA;7LY%Ez`Svlx$xLnKIpLVr09@)_2H5G*-_g)d4?|0NFAgU?2z5RyzW(v`a$va9tZTt? zKMO++#~thGcYDO}^nOti&DNpkZN<_SC(lGk0cxdC!*tDolYNe}OCZI$?}I8zsS=L)wefN#Bo`Y&SX`R~G0Xe3;g)GCoAP{K&(;*T*csD%4;7 z`!S=KU}6+N_H?+apAn?CP=Z~F+-Pie`|mnq{VzO#A|=64!H=#Y9ZIq+v;g_yV-L?d z;_%NGTv97xFkLsX4#r%ncmfN39VlH6;lS|;tVW3(7U%|Dp6$K6cqTkzCeNG;C|bOk z?SxpNG8-(V+86*!Gz7pNm56VelOWTpm4Xg+vt8^*y1STPk+R?nZ{_O=(jvmWLv^LG zmSz6Oyl>=2Gt@@9XYok4{f*F&*|=KAFXtHpty%nGW-e2vE*47(Sy2P^ws)GMaDKLE zRWCu>=OfzKcIN5h(>sWT(X0n%xUeoWFDPd>uL9b6^$JCo2vnMf>lq&_nA4@LZlnx< zp-|~l66Gwj96QQSL2&Iz%PoS1^H&)|tH0;uE#8Al^M-%U+!3;=G(PjU$)cxNzk_Hs z8q!yc!lAs-CX51M93E|AT+~HCS~L173JWd-H*E@ERe|zHQz3Ll;|Z6B8rDIviOsST z$T&SYIs7mlxk^a_m0rww?2rJf7UuNr2qAZ$uE)3(r;M0UB2S-M#J`SmmR{Q#xA>R# zmsw981;P85wzWA_I{0302avOvVZ{uP9Nh zQN?bZAo_@0cO+I%*sjcq3C;2pM)xNcrl;)dpYBFp9`h>nxbI)p4vH61&NT}?gS>gl zg~int*y}VjMMPqt-_GwgoJLhcL$QN{!!JUS385wOT2YN-97&2ed)uZYiv?3AUN7#IK;=ueDfw^8Yq42f&*4KoosjLf0DXBc->1j2!xw*MQ zWZOz@Z{zf|rQhlhFQ8a`DNNnZrg9ATlIXP@Q;BEBu;vHp&jDv!LxQZU(-4*0I$}1`iJ2y}Y znxXU8M6+a-Le6THS_TIOa^~*J-2OX@x8ASI7F||CvmgY3Pai30b79B`EcJ;!YZzsS zE~8kmn_EdVch1DGUwJJ7yNg=q%r*nl(j_Bti{6QYp*4Zq~%mSp--QC(G zfr^TXxS}E*RT`U}dXV_jQDr*LYj%BAPLV|z45l;xjWOL{^)Zg9|$Fga=7=^aVkk<0>;wtKz%LC2xaPPXaEhDz#%6TY_g z@#Ec|4@4 zVW0{@sEQ`gECN>Y+kQS5(QH7Hgd)W*(_65(valIC($}6T#r&_DMzV<xdCZK1}u*vFJkkpJ-yWpoFP^_GaZ>Tw2=(!+ekj1ZKfcc{m9@h zfrdL*!T*^L_#r4yPsXGo-|Xvlob~R+>V~Yht7Mcp>&$ z;pIS>v`$s-{^S23L2{H4Kt(LdXK$sr5R!D7*K-5OKlkcW64n-y7^$M1&F-i_I?6%Ri4CSjPJz@< zRAYEUhU?Y83Gv!exQp;%^8R*}jTe!QnqXLOcma4T-l5b4NkOcHVztzqyk{J&PbJaq z_LDJ~y}z^1m%VvbbjCY5bH#3Pboa)u3GtO~Ksik2Ydp$rl~cwGgT4ByG@_h8<$AP? z6}=Lyf>T9kGQ|QiE6+uT%rOE`^`u-k8URHWEYSE2Mvx*M3Sy$)qE7jhvEH#fIAwVt zPlgkAEI=6`fiP%=sT5ca4fLf&O~=2f4%W-?jDDRPQY`1sq+bFLY)fO>BHJe*{@cmM9WoX_&xi0SI;>d(cu#s~h}{%M+TZf?HX zxN+|8;ZbyQ65w}v5zPCy^Dd=3i;Odv+%sV z{gJKu?Cc#U0rqY2#LvhpOu8~I)o%D=u`6k9W4uaTELgy)XCi;t zIyZ8^t+{PAZP6KwRa0UQZ$*ViH)UATpbBYKMa3XNOg0qq!)6jP^N0y9b5E)B zLtLhQe@3yepC*2%STe*xw_VomSRd8*U?na5i+y_>-TT2}L^^pB(DToFglsD@l&gd{ z!&1_(TZvl~=%WPfR9^%%_#fKy#VqBHpVI(t5&Zw$VyfVc&SKFuT|)W+Q7_e70(tWx ze8>|@5QC=_$VL>KX`^;I5$AV${xfyZ6S+YqWlDYFo*MD`I=di_O4}xofUw`Bt(Gbi zM#Kt++pJ|{m-PNu_H^|a4i}^2rg(P2$;E}AYlK|lymk9AjhWV1>4nFM=KtKF4gnD_ z8s=Ka?ZyY!h&LHgKvml?O5u$| ztEQKxepR!zxETi!vN+CBE)d`xa14cxt0|H}5N%>6Fqc9u#0n*T)0 zI(^jS)=ZD1Xe^LM)0d7|@ z4kpLHhUu9$|0yj4rgvp(uCy2cGzZbJbtswEIafRscWWurl2thZDm1R1ECbngRGHEtEKVGhKEvq05eMhlqdOA2M1heD85MzJL/5TcgJNml9K0mefBYUuOO7tHaCreP7q0Wgf43QtEKw9Fohloy20kDYsd+AootqySgUo4VKCK5YPhYjkWUQqZFGpBTVeNle8PGuOYlhJuwiwufqD0ZV0qrrhdvrd8DixO7suTiTErsYhSIhVFQDKbh2go0UQrWjtN4AN+wslzbv5pnZ7mASMvWahP33Zv8t/3X3KL6Ej+twe1V+at5hlQPhJV74gRQsCqPiEFKIdDdDkSsmsgIvoRpLRooyo2CKu05wVSVMwS4nkZmttBW0lqiU68jTw/lh7c4gFdQDCQ9/CyIFJRu9BGc/IEc00hrDqu+Kt0QtGXTEphE0QtwV7lnpAeL6C3T+C+ggu2R0Haf/xi54gd2lQPOXE2r+EWr+v6S2nFGbQYKMfjSvno4iTgrNdMwFaqbuEaEZ/zTj9wuMtvVgatvYINOHv7cFTDDIMmGf9hTZvGd7UIhSRnDaHorIGNTpvyDQ0Rs+7+igY4sjDbOaBE4UO4xf/mNdxB2+CqZv1hmmM0MzMZAt0d4bs4Yv9bTQalLImxRqwcwKPZmqu/bbfbY6g8/O7ZmTXvAuyguzx2P6KLzWC+uJFfxzWUGH/bdHu7z/gAuu/wA= \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/encode_csv.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/encode_csv.md deleted file mode 100644 index c1590232d8..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/csv/encode_csv.md +++ /dev/null @@ -1,71 +0,0 @@ -### jsoncons::csv::encode_csv - -Encodes a C++ data structure into the CSV data format. - -#### Header -```c++ -#include - -template -void encode_csv(const T& val, - std::basic_string& s, - const basic_csv_options& options = basic_csv_options::default_options())); // (1) - -template -void encode_csv(const T& val, - std::basic_ostream& os, - const basic_csv_options& options = basic_csv_options::default_options())); // (2) -``` - -(1) Writes a value of type T into a string in the CSV data format. Type T must be an instantiation of [basic_json](../json.md) -or support [json_type_traits](../json_type_traits.md). Uses the [encode options](csv_options.md) -supplied or defaults. - -(2) Writes a value of type T into an output stream in the CSV data format. Type T must be an instantiation of [basic_json](../json.md) -or support [json_type_traits](../json_type_traits.md). Uses the [encode options](csv_options.md) -supplied or defaults. - -### Examples - -#### Write a json value to a CSV output stream - -```c++ -#include -#include -#include - -using namespace jsoncons; - -int main() -{ - const json books = json::parse(R"( - [ - { - "title" : "Kafka on the Shore", - "author" : "Haruki Murakami", - "price" : 25.17 - }, - { - "title" : "Women: A Novel", - "author" : "Charles Bukowski", - "price" : 12.00 - }, - { - "title" : "Cutter's Way", - "author" : "Ivan Passer" - } - ] - )"); - - csv::encode_csv(books, std::cout); -} -``` -Output: -```json -author,price,title -Haruki Murakami,00,Kafka on the Shore -Charles Bukowski,00,Women: A Novel -Ivan Passer,,Cutter's Way -``` - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/data-model.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/data-model.md deleted file mode 100644 index f5c824f1da..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/data-model.md +++ /dev/null @@ -1,209 +0,0 @@ -## jsoncons data model - -The jsoncons data model supports the familiar JSON types - nulls, -booleans, numbers, strings, arrays, objects - plus byte strings. It has - -- null -- bool -- int64 -- uint64 -- double -- string -- byte string -- array -- object - -In addition, jsoncons supports semantic tagging of date-time values, timestamp values, big integers, -big decimals, bigfloats and binary encodings. This allows it to preserve these type semantics when parsing -JSON-like data formats such as CBOR that have them, for example, the mappings between the jsoncons -and CBOR data items are shown below: - -jsoncons data item|jsoncons tag|CBOR data item|CBOR tag ---------------|------------------|---------------|-------- -null | | null |  -null | undefined | undefined |  -bool | | true or false |  -int64 | | unsigned or negative integer |  -int64 | timestamp | unsigned or negative integer | 1 (epoch-based date/time) -uint64 | | unsigned integer |  -uint64 | timestamp | unsigned integer | 1 (epoch-based date/time) -double | | half-precision float, float, or double |  -double | timestamp | double | 1 (epoch-based date/time) -string | | string |  -string | bigint | byte string | 2 (positive bignum) or 2 (negative bignum) -string | bigdec | array | 4 (decimal fraction) -string | bigfloat | array | 5 (bigfloat) -string | datetime | string | 0 (date/time string) -string | uri | string | 32 (uri) -string | base64url | string | 33 (base64url) -string | base64 | string | 34 (base64) -byte_string | | byte string |  -byte_string | base64url | byte string | 21 (Expected conversion to base64url encoding) -byte_string | base64 | byte string | 22 (Expected conversion to base64 encoding) -byte_string | base16 | byte string | 23 (Expected conversion to base16 encoding) -array | | array |  -object | | map |  - -### Examples - -#### json value to CBOR item - -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - json j = json::array(); - - j.emplace_back("foo"); - j.emplace_back(byte_string{ 'b','a','r' }); - j.emplace_back("-18446744073709551617", semantic_tag::bigint); - j.emplace_back("273.15", semantic_tag::bigdec); - j.emplace_back("2018-10-19 12:41:07-07:00", semantic_tag::datetime); - j.emplace_back(1431027667, semantic_tag::timestamp); - j.emplace_back(-1431027667, semantic_tag::timestamp); - j.emplace_back(1431027667.5, semantic_tag::timestamp); - - std::cout << "(1)\n" << pretty_print(j) << "\n\n"; - - std::vector bytes; - cbor::encode_cbor(j, bytes); - std::cout << "(2)\n"; - for (auto c : bytes) - { - std::cout << std::hex << std::noshowbase << std::setprecision(2) << std::setw(2) - << std::setfill('0') << static_cast(c); - } - std::cout << "\n\n"; -/* -88 -- Array of length 8 - 63 -- String value of length 3 - 666f6f -- "foo" - 43 -- Byte string value of length 3 - 626172 -- 'b''a''r' - c3 -- Tag 3 (negative bignum) - 49 Byte string value of length 9 - 010000000000000000 -- Bytes content - c4 - Tag 4 (decimal fraction) - 82 -- Array of length 2 - 21 -- -2 - 19 6ab3 -- 27315 - c0 -- Tag 0 (date-time) - 78 19 -- Length (25) - 323031382d31302d31392031323a34313a30372d30373a3030 -- "2018-10-19 12:41:07-07:00" - c1 -- Tag 1 (epoch time) - 1a -- uint32_t - 554bbfd3 -- 1431027667 - c1 - 3a - 554bbfd2 - c1 - fb - 41d552eff4e00000 -*/ -} -``` -Output -``` -(1) -[ - "foo", - "YmFy", - "-18446744073709551617", - "273.15", - "2018-10-19 12:41:07-07:00", - 1431027667, - -1431027667, - 1431027667.5 -] - -(2) -8863666f6f43626172c349010000000000000000c48221196ab3c07819323031382d31302d31392031323a34313a30372d30373a3030c11a554bbfd3c13a554bbfd2c1fb41d552eff4e00000 -``` - -#### CBOR item to json value - -```c++ -#include -#include - -using namespace jsoncons; - -void main() -{ - std::vector bytes; - cbor::cbor_bytes_encoder encoder(bytes); - encoder.begin_array(); // indefinite length outer array - encoder.string_value("foo"); - encoder.byte_string_value(byte_string({'b','a','r'})); - encoder.string_value("-18446744073709551617", semantic_tag::bigint); - encoder.decimal_value("273.15"); - encoder.string_value("2018-10-19 12:41:07-07:00", semantic_tag::datetime) ; - encoder.epoch_time_value(1431027667); - encoder.int64_value(-1431027667, semantic_tag::timestamp); - encoder.double_value(1431027667.5, semantic_tag::timestamp); - encoder.end_array(); - encoder.flush(); - - std::cout << "(1)\n"; - for (auto c : bytes) - { - std::cout << std::hex << std::noshowbase << std::setprecision(2) << std::setw(2) - << std::setfill('0') << static_cast(c); - } - std::cout << "\n\n"; - -/* -9f -- Start indefinite length array - 63 -- String value of length 3 - 666f6f -- "foo" - 43 -- Byte string value of length 3 - 626172 -- 'b''a''r' - c3 -- Tag 3 (negative bignum) - 49 Byte string value of length 9 - 010000000000000000 -- Bytes content - c4 - Tag 4 (decimal fraction) - 82 -- Array of length 2 - 21 -- -2 - 19 6ab3 -- 27315 - c0 -- Tag 0 (date-time) - 78 19 -- Length (25) - 323031382d31302d31392031323a34313a30372d30373a3030 -- "2018-10-19 12:41:07-07:00" - c1 -- Tag 1 (epoch time) - 1a -- uint32_t - 554bbfd3 -- 1431027667 - c1 - 3a - 554bbfd2 - c1 - fb - 41d552eff4e00000 - ff -- "break" -*/ - - json j = cbor::decode_cbor(bytes); - - std::cout << "(2)\n" << pretty_print(j) << "\n\n"; -} -``` -Output: -``` -(1) -9f63666f6f43626172c349010000000000000000c48221196ab3c07819323031382d31302d31392031323a34313a30372d30373a3030c11a554bbfd3c13a554bbfd2c1fb41d552eff4e00000ff - -(2) -[ - "foo", - "YmFy", - "-18446744073709551617", - "273.15", - "2018-10-19 12:41:07-07:00", - 1431027667, - -1431027667, - 1431027667.5 -] -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/decode_json.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/decode_json.md deleted file mode 100644 index d07dbe3576..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/decode_json.md +++ /dev/null @@ -1,78 +0,0 @@ -### jsoncons::decode_json - -Decodes a JSON data format to a C++ data structure. `decode_json` will -work for all C++ classes that have [json_type_traits](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/json_type_traits.md) defined. - -#### Header - -```c++ -#include - -template -T decode_json(std::basic_istream& is, - const basic_json_decode_options& options = basic_json_options::default_options()); // (1) - -template -T decode_json(const std::basic_string& s, - const basic_json_decode_options& options = basic_json_options::default_options()); // (2) - - -template -T decode_json(const basic_json& j, - std::basic_istream& is, - const basic_json_decode_options& options = basic_json_options::default_options()); // (3) - -template -T decode_json(const basic_json& j, - const std::basic_string& s, - const basic_json_decode_options& options = basic_json_options::default_options()); // (4) -``` - -(1) Reads a JSON string value into a type T if T is an instantiation of [basic_json](../json.md) -or if T supports [json_type_traits](../json_type_traits.md). - -(2) Reads a JSON input stream into a type T if T is an instantiation of [basic_json](../json.md) -or if T supports [json_type_traits](../json_type_traits.md). - -Functions (1)-(2) perform encodings using the default json type `basic_json`. -Functions (3)-(4) are the same but perform encodings using the supplied `basic_json`. - -### Examples - -#### Map with string-tuple pairs - -```c++ -#include -#include -#include -#include - -int main() -{ - typedef std::map> employee_collection; - - std::string s = R"( - { - "Jane Doe": ["Commission","Sales",20000.0], - "John Smith": ["Hourly","Software Engineer",10000.0] - } - )"; - - employee_collection employees = jsoncons::decode_json(s); - - for (const auto& pair : employees) - { - std::cout << pair.first << ": " << std::get<1>(pair.second) << std::endl; - } -} -``` -Output: -``` -Jane Doe: Sales -John Smith: Software Engineer -``` - -#### See also - -- [encode_json](encode_json.md) - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/default_parse_error_handler.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/default_parse_error_handler.md deleted file mode 100644 index 8f3790d5af..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/default_parse_error_handler.md +++ /dev/null @@ -1,21 +0,0 @@ -### jsoncons::default_parse_error_handler - -```c++ -class default_parse_error_handler; -``` - -#### Header - - #include - -#### Base class - -[parse_error_handler](parse_error_handler.md) - -##### Private virtual implementation methods - - bool do_error(std::error_code ec, const ser_context& context) override; - -Returns `true` if `ec` indicates a comment, otherwise `false` - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/deprecated.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/deprecated.md deleted file mode 100644 index 547e2786c1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/deprecated.md +++ /dev/null @@ -1,110 +0,0 @@ -## Deprecated Features - -As the `jsoncons` library has evolved, names have sometimes changed. To ease transition, jsoncons deprecates the old names but continues to support many of them. The deprecated names can be suppressed by defining macro JSONCONS_NO_DEPRECATED, which is recommended for new code. - -In the table, indicates that the old name is still supported. - -Component or location|Old name, now deprecated||New name ---------|-----------|--------------|------------------------ -class parse_error|`parse_error`||`serialization_error` -class basic_json|`object_iterator`||`object_iterator_type` -class basic_json|`const_object_iterator`||`const_object_iterator_type` -class basic_json|`array_iterator`||`array_iterator_type` -class basic_json|`const_array_iterator`||`const_array_iterator_type` -class basic_json|add(size_t index, const json& val)||`insert(array_iterator pos, const json& val)` -class basic_json|add(size_t index, json&& val)||`insert(array_iterator pos, json&& val)` -class basic_json|dump_body||`dump` -class basic_json|remove_range(size_t from_index, size_t to_index)||`erase(array_iterator first, array_iterator last)` -class basic_json|remove(const std::string& name)||`erase(const string_view_type& name)` -class basic_json|parse_stream(std::istream& is)||`parse(std::istream& is)` -class basic_json|parse_stream(std::istream& is, parse_error_handler& err_handler)||`parse(std::istream& is, parse_error_handler& err_handler)` -class basic_json|as_int() const||`as` -class basic_json|as_uint() const||`as` -class basic_json|as_long() const||`as` -class basic_json|as_ulong() const||`as` -class basic_json|as_longlong() const||`as` -class basic_json|as_ulonglong() const||`as` -class basic_json|is_longlong() const||is() -class basic_json|is_ulonglong() const||is() -class basic_json|is_numeric() const||`is_number()` -class basic_json|remove_member(const std::string& name)||erase(const string_view_type& name) -class basic_json|const json& get(const std::string& name) const||Use const json get(const std::string& name, T default_val) const with default `json::null_type()` -class basic_json|has_member(const std::string& name) const||Use `contains(const string_view_type& name)` -class basic_json|has_key(const std::string& name) const||Use `contains(const string_view_type& name)` -class basic_json|add||`push_back` -class basic_json|set||`insert_or_assign` -class basic_json|members()||object_range() -class basic_json|elements()||array_range() -class basic_json|begin_members()||Use object_range().begin() -class basic_json|end_members()||Use object_range().end() -class basic_json|begin_elements()||Use array_range().begin() -class basic_json|end_elements()||Use array_range().end() -class basic_json|is_empty() const||`empty()` -class basic_json|parse_string(const std::string& s)||parse(const std::string& s) -class basic_json|parse_string(const std::string& s,parse_error_handler& err_handler)||Use parse(const std::string& s,parse_error_handler& err_handler) -class basic_json|resize_array(size_t n)||resize(size_t n) -class basic_json|resize_array(size_t n, const json& val)||resize(size_t n, const json& val) -class basic_json|to_stream||Use dump -class basic_json|write||Use dump -class basic_json|`json` initializer-list constructor||Construct from `json::array` with initializer-list -class basic_json_deserializer|json_deserializer||Use json_decoder` -class basic_json_deserializer|wjson_deserializer||Use `json_decoder` -class basic_json_deserializer|ojson_deserializer||Use `json_decoder` -class basic_json_deserializer|wojson_deserializer||Use `json_decoder` -class basic_json|owjson||wojson` -class basic_json|member_type name()||key() -class basic_json|rename_name_filter||rename_object_member_filter` -class basic_json|any||removed -class basic_json|member_type||key_value_pair_type -class basic_json|kvp_type||key_value_pair_type -class basic_json|null||Constant removed. Use static member function `json::null()` -class basic_json|an_object||Constant removed. Use the default constructor `json()` instead. -class basic_json|an_array||Constant removed. Use assignment to `json::array()` or `json::make_array()` instead. -class json_decoder|root()||get_result() -class json_content_handler|begin_json||Removed -class json_content_handler|end_json||`flush` -class json_content_handler|begin_document||Removed -class json_content_handler|end_document||`flush` -class json_content_handler|do_begin_json||Remove -class json_content_handler|do_end_json||Remove -class json_content_handler|do_begin_document||Remove -class json_content_handler|do_end_document||Remove -class output_format|`output_format`||`json_serializing_options` -class serialization_options|`serialization_options`||Use `json_serializing_options` -class json_reader|max_depth(),max_depth(value)||Use `json_serializing_options::max_nesting_depth` -class json_reader|max_nesting_depth(),max_nesting_depth(value)||Use `json_serializing_options::max_nesting_depth` -class json_reader|json_input_handler& parent()||Use json_input_handler& input_handler() -json_input_handler class|do_longlong_value(long long value, const parsing_context& context)||Override do_integer_value(int64_t value, const parsing_context& context) -class json_reader|do_ulonglong_value(unsigned long long value, const parsing_context& context)||Removed, override do_uinteger_value(uint64_t value, const parsing_context& context) -class json_reader|do_double_value(double value, const basic_parsing_context& context)||Removed, override do_double_value(double value, uint8_t precision, const basic_parsing_context& context) -class json_reader|`value(value,context)`| |Use `string_value(value,context)`, `integer_value(value,context)`, `uinteger_value(value,context)`, `double_value(value,precision,context)`, `bool_value(value,context)`, `null_value(context)` -class json_output_handler class|do_longlong_value(long long value)||Removed, override do_integer_value(int64_t value) - |do_ulonglong_value(unsigned long long value)||Removed, override do_uinteger_value(uint64_t value) - |do_double_value(double value)||Removed, override do_double_value(double value, uint8_t precision) - |`value(value)`||Use `string_value(value)`, `integer_value(value)`, `uinteger_value(value)`, `double_value(value,precision=0)`, `bool_value(value)`, `null_value(context)` -basic_parsing_context|last_char()||Use current_char() -json_filter|parent()||Use downstream_handler() - |input_handler()||Use downstream_handler() -file `csv_parameters.hpp`| ||Use `csv_options.hpp` -file `csv_serializing_options.hpp`| ||Use `csv_serializing_options.hpp` -class `csv_parameters`|`csv_parameters`| |Use `csv_serializing_options` -class `csv_serializing_options`|`csv_parameters`| |Use `csv_options` -class `csv_options`|`header(std::string value)`| |Use `column_names(const std::string& value)` -class `csv_options`|`column_names(std::vector> value)`||Use `column_names(const std::string& value)` -class `csv_options`|`data_types(std::string value)`||Use `column_types(const std::string& value)` -class `csv_options`|`column_types(std::vector> value)`||Use `column_types(const std::string& value)` -class `csv_options`|`column_defaults(std::vector> value)`||Use `column_defaults(const std::string& value)` -file `output_format.hpp`| | |Use `json_serializing_options.hpp` -file `json_serializing_options`.hpp| | |Use `json_options.hpp` -class json_options|`array_array_block_option accessor and modifier` accessor and modifier| |Use `array_array_line_splits` accessor and modifier -class json_options|`array_object_block_option accessor and modifier`| |Use `array_object_line_splits` accessor and modifier -class json_options|`object_array_block_option accessor and modifier`| |Use `object_array_line_splits` accessor and modifier -class json_options|`object_object_line_splits accessor and modifier`| |Use `object_object_line_splits` accessor and modifier -class json_options|`array_array_line_splits accessor and modifier` accessor and modifier||Use `array_array_line_splits` accessor and modifier -class json_options|`array_object_line_splits accessor and modifier`||Use `array_object_line_splits` accessor and modifier -class json_options|`object_array_line_splits accessor and modifier`||Use `object_array_line_splits` accessor and modifier -class json_options|`object_object_line_splits accessor and modifier`||Use `object_object_line_splits` accessor and modifier -msgpack|`jsoncons_ext/msgpack/message_pack.hpp` header file||Use `jsoncons_ext/msgpack/msgpack.hpp` - |`encode_message_pack`||Use `encode_msgpack` - |`decode_message_pack`||Use `decode_msgpack` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_decoder.png b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_decoder.png deleted file mode 100644 index d013c26a94afd69fd33b7b193f344ebf09068cd8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4080 zcmeH~_fu2r*2fbd0^v}kN=FnBK_b0`5*0*1P!zCG!jU2f0s-lfB2A*Z1=z-n261hKfN! zAP~3NHRD?#5Eu#EyK$TVu1ja8jzJ*aIy2)-HX%-{82b`Sid1L9u@fIu@$u@XwwUx; zf4>(FaNc4;ese|Wsw~5P&0kSY&>A}DM5E4j%|Sk#mEbjgb*=k?amZNldI?33pxLWC z>fnXdq}}yGKRkkmV-0BQn@1@}IfT8u?}n?%s^#(QLS5F;BX*H*rk)dy3{2e?8ea-| zurv3Ynd0}*^X}K1f>tX*-!fU~NBxA~bC1z~d@&Tt-b{-WYppZ%8og*Ofakt#E`KYY zCDvE;!N#wNa|q_h)jRih?7^+0>-%gcC8^@ZIXRySK-3<&mra1znw&F87x0>w(Q_15 zLNNVJVoN|&@dv{OJvsE9FEW292tYU}*IkP;&O$hl6={4z&*Rq9q>Xw93oVGGh5+Ki z&w|M>#F27)OA7ueab>)wr*>z1@pg^pkbJ;Y3wY>TIHMZdfAEY~+Av$rB5O=ypRta# z*74M=4_waDMNP+!*ZJlz4t%&`f`6r=7r6Ag0W}jpUgeT-uPNxU?Fc*7en;)6v*fvS zQS>q*DL3vKQ{KXYIpV$ev$7NVI(8j+llb9ELhttkUgD1o1ul$|-WOGu4#8VZp<6|( z=|4XZ-|7y%GYX+Q2h8`1POdg!V0cPBs^FVbMaqy_@Y8h)A?<6L7tS)#R*zX!RP>B+ z!M-+|TU_h4a#EigwKXoz=N3&R|NE(Tf(}MKZ(w!#(uU~z_oe{>0aSGiVRjtnbtfEu z=-m# zmEi_er|&aE8EY6(U4K_sQ!8ec!wdZnV4-uIls7Qs1TqMLEi7zbp%5tDGDy}hsI-sF z_NIuVMhGSz))%=k9e1Qpoca2eHQRf#wwzv4w)$OOs!q<11mcm>_wd^S9r{gKA08ha zxNl6i$K_piuq6A~PD^WfXU7q0xu>A61&7-}4?R{u_B!cC{^=ofQE%+ss@BW=v~QtX z9UVtY!(}!FB49Zmk5^5pzAIx@BU&26JJ&bG3k#*-T}1F6XiRTSBMjJi*My5~ZbiSM zXCmzul?&!;2OrB3!4RjRB1@(4ED3fb#?j_sN?RO(7)H{V+CF3WBlt$0LN z_zJQ}Mq_7R3gZWfO|S}5|BTNc6E_6Ws+R}YJI*eA-brdVr~GkxgP z9P$(IEFd{ORc3m#QlWOE&1MNAL(QA7i=t`dvdB@D5E(q&6vpCEaY>%pklY%%X>={f z!-BTF7Bmpij$bKCH$5vOR0LYKx}NLtF# z{7$4PImg#Wx6~>AaHO_MVrWOQXremh7CrM#n%Y*Th;8<1)$Em&iQTN|zd)Lt_Y`TkZk-+n!ClA_VVE zj?a{^<(F}IDYMr#lc)Rey^?P1aZQsS9GI<6zbZ~F2FaNQ*8%+>C;QR*B)Qt(?mJ7vCI{%5poXu7Tac-$ zTECdzz~Ipw2{+L~C-|C}+3tP(@;*B=|9p>tXNAgfoPDn3$$QUDq;b|Ql5kLCNcUeG zqsYi{@i>ykBY{iRwiId^97$3hEw_J|DsALc|HCTZkR0)pvF4~AxMXv%(;T|x{pJNL zjj=S`vqEYZiXgZX#SYl)m-xb2*?BZ(I#Se;goq2(imDEcPktnyQfIPEMAeNdS`1>r9gtd0H`Jwai|d9wRYbU}JdiH;Iech^Z6?JIY>_3QI$2 z8V_P}y>6^8md9fsO_Yq*XJ~-Xkw^P(oiZ%se51$-i?H1lPX-4q^p^k{>)7O8X&^{v zTc~7ZpF#98!P*}B9s}=L{5Fok^mgzmqpZ#|I0Sx@&5TEQ9f4cs+gu-x*lWkLhp(&%y3S zb=a>GYhg5Kw>wjb_}R9MpDOOgySwpBd_5lI5XGC42@;X?`Q<%{PS*MM=lF8&%ml$bo|^=wH*Q@J!JA=VPGP2jeJGUO#9r2NaZyq|%S@~< z|ElJW+xujdVrLbqlGZrrWf_N)I=~ z!l?1eZrWh{2%P&>5d1Y4a%^O`5_d8Z3$f=zwXN1apBzqiuVY;)+L-x{r)iX>&@Bu2 z`%6n%3a9LE(-f0O1<+gjroxJ@vx=!Kv9LGBYfwIX<{MDj+L&0P$o=a4I)AUNM zD-6cZI3yX3AXCPc+fPvKVVRGnbT}y|R8(#!*+$Xlrn?-$t=bI!i%}ASGUxz4j7pM_ z47x$~vP~47yK3p(+;wJ{9bbG#BES~%YC~qPfM}zp&w*&C?Z`E6Ig$O)RzEfCC_2&D zUGGm?k0t{0;$I8+--*Kf7mfJWv5tcNX*%&gOx0U+V(c?PxrS=4QMUVtVQ9BHK91hOeb&6lf_%=!D|lc&Mi?k5la3xRFh&WjGDF&=_0@u z7rjO+aHv!%gT~8V&B@T$SP<17?;aR#l0_euE!@c~tYo1)J=o3Rk<@ct%*B^P0Y4PC z$7v4^Ep6?|zOSXFKfFoxKR;Kx^$6Ge`TfKh%sQR=omlPG(>wK73^7OjA(Xrc2yt(= zs>_Ndmxzi6=oHV`!Qozw0b>H^H4l)cJ0SeKk-(QYR1OuG@~VgLdI#)n&RruoHU;5m zGxdNne>h{}p#HE=18`DiLUanI6InjLO+d^4he#hc9D`SM1Lnu{{hreLFvQKdo}|sI z%?sNZn?sGj^c?UeHx55Kz0dyjY>ZVVox9}Fnr^ZmMLRlR5U1OdF6f{@{z`UVD*3aD^^oQ>(WqNvgsva=bwZ*1bPPa&a(!}9inl7EGxLUAH)tgxcU@|G-YWr2^i&Xa> ztLzHli!o#)MUerxzi<(?P{ry4<>%^9YlM%4_7GWJUeq@a%?AT){%n3C7}29PX2VH| zGM9!RqhpUn&;FD_XopO^lfUWjwmm^6MXHSra8)sN0%MAOE zNGb3~^UL!`)9qa0E`NauOs6TO(54eeoNcSN$oLjY3|Kyui}R?J z7{@2|ZAUe9(=}jw!Mf)0WNm9w*;!^|+OFD26!9!UY=0Vg73`zWbRZ}v$9z3#<(9-H z|LJ_Y<2CMC z^e81E4!T26@bxF?LVj3VdNc9owEP01PtKxLaDkmABJW4ZMUzoDOTGKtbHVyhKVBZj++kq2jD8zIW2G6cAF7Vvtyvv0vCwOGsfpncSbaC4IMMd3SeqgieP7nut7+ssghxxBfT8HQkmJ3VQCC/obikiLbimBpLZRCcEU3dTBQHAOgaphWEqxr297Fqx+6gaH0AIWAWZtdEmJinJ0NHBL/BPQMCpO9lzriXGx2QJJhInYVyA0ddBYCqHyVZyOgRnyCl7yuNsXvMcHk8DVKQHT++8pnz0sIn/8a6aW8/gx/dKzWXaYbW3BTzihwfpHIvhac6p08nWEOWEgHX/I9Ek3T2YVmlUXkpWqDgV/Umw5AfMIrnbvI6pgscGB8e61YjQWqZhpyztGV0sqng+kgrQC2RLvQMSg5EFvsd6RZdvKzSvY35eX5w0tFlUurm8xbPUSHjOXlOqFZfUNDH9sMZxxSyDQ70onp//C4DNlbCyYkFksguyj8URJ8RMqHjREV4i8D+cnkT44J+mjFuktWoGTa9MftBUwnGjR15mElKqVJd2sH836w8Bak7TimhwKg+uHXxUJjFGJMmYZlllF3It3kIitDOB1dSksQ1Cvv+dAat2ufaOVG+u6sAKTwLCiu3qP7LpFe8JXQXVlR8H063rxUUMHedk2qNrSGnmuGrobNfLktLTyZJI6Fn2SysLdPPJn9yv8DcLe8mEGuxn0Bheoss5C0UWqp99INDyveoYXop5q7+ks9P/qPQO30TTcv5WP19Chd179tAeLzzwCSRXm+mXWEzfFocRxS1T6R13VZVQfDrjg0JgkLIQZDbnRopaAHl3QjRkRqJ6Vr60jpoSYYzqnlPoc8w5jh9do/77bMXagDiX5bx87tFlO6vkNlv930PQP \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_encoder.png b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_encoder.png deleted file mode 100644 index 49dc6e5763bd572f4ede3f1b16e93df519f58018..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5225 zcmeI0=Qmt$)b|HLj37lxh>RqXsF4u8w`jrOj6OyUM(^E75Q2m#5xoUt7zCq+5z$+; z(YuJ~B}Omz{QiLF-Sg^R_nWiM+Uq*|+Sgv+v-f9T@1a^MbYNC62n3>2Q&rLhfyiK_ zcbltJq~|%At0)LW(dG{rdOG>4xLdf|X}Vb3ID$Z$z{JO5vFqolszMC!o9V-($ot}7 z+kd`JF7PmdU_mE&eB$s}dPueN=~$7(-6`0t{5NA=)5nr>FwGdf^j66u90drv@2CAk zS`5Km8!9Ch|0b7#PdVbs<6XaDhOBYFOa`}zKNQX~*Sq6SScFnVpK4T52zq2+-A|5P zuT2TCOf?W3V02S_V|c|p7UE$Mi%NL!G9|d15b-X8ACgHQH^cRUi2N(bakHVdtpKi; zn>(S~`rUzN@sswB+$UH1d$s%>%G_%^mxdxKT_F{*>bns2!E9AqdiI}*6Kwb9WuEH| z5o?QNEtL2?#84p?P6@H1T95BON@G!%;ZVW0c)}=#q3f;{P7XN-mu1Ttr+tcgwwY{V( z+lIcxX89ktJT8kVPhoBc*zHuHwOsszPrHJ^P#o9in)*#uL}!U2S586Ppjm`C zMlNanO`5G@NnjRU7^@Uql6K`@;%w^J56>bM(b2c=CPSY4rpiL;&r&B6-lph8N^ZZ6 zSdW#Hoyk44bBmg5Jm*-wkIg{P4YWzjPpp?jA5mw{c=-9Osnz;18>(4w>vfq=o-y}d zoMngSZc=$m`^vyY`~gp@Ntpl?I3;O&rr*i>pbmf-gw1)a*cddOYL6rA+_IrTK1OnSmWconC5Q$p`mgrM06(O`koE0OxVEFFY7}wd_`t zAbB|7!1Plo4_$W!t;z=}Vb%@D&XNICbMAQU``sZwo%*P|6dzoU;V!G68C}XmLr*AD z*TdLBpz8=VC3(QtY%|>itv$!wQRpkr^|cC@aFanHzUPNu+^L^fejHR?$Bw0r!Cr~hZyryg1bHL-h(`Q1 zA_jVG$uXD5hxk-KfZhNLes$C#OP6DeouF-XQ4o$(61-Ln%IRjyaizRUXwj3Sb_YVB zeLAmFF|WuJaDp?wNm=0QnBYJ8%6`h|YP2UcgfE9RFzTEk4oZ|&*?-nR9)|SB`o6x! zEr%BQRQ|#wl4Q7F;Q_^xyq@0Yq`aAP@^3}E(r)m9ki#gSkY#J+%Z~5%!;goH4FrU& z+G8r5q)`{sB?$cN#vFdmb9E?xV?DZQeyJn&^+HRy$UB*!6X3U&KHb)Iot0L)n8m@y zgw-3hk5Dd|{YP-bY0sx8e)5QxKM>3_~u| zaItslyorahzp#rF_ua|%J3IO&?zd+R*ylcA6?eD(-Mu+g%G=Di0)3hruFKlI}3^>L8iq{QDwSrVaQ^ppZ)&* zQ6^-4gv&%c2M)HxG(Piv%YsOFat-BGSBspy>JV`AUK{Dk6qqA!PK~Vv6W3ru!Dqgq z$md0t$?7<(j*qX$O5t?<)YEPBj?wgLOWoI$7nNl?KCsy~1)UxwtHT`fgIaO7SI$ol zUyqktd}Df%W4x#Tn+En(P7U1@d_F1D8e#;8XG}Esl+kGiQ$0&42=mQ$p8al5R*)0* z57FpvBl}Abo8$p{-Lkt~bXC@ILGCP4{_KZsd86m5F0E|(;lZZK`eB-=iy%(! zN{FwH+QuYaC-0@v)Bvy|MFD@H|8|xK#_bJV|rw@rw|3X6(LV)W4Y`*9PxRJ~WY5&iny1NvuKv zhH>%v(Qz(CbL5{chPLD>W2oYVQm#|hspnwsqjT1(9luIb$i}xf`tIh!zuaVx&&HU@ zmp`d~{*xQ$`V#%x5A3UjNz(Z1ilA zr&t<*34G5jyk@Y#()^}^u>F&eDR9><%)gaiE<1uAyaZ14Yb=n^oMIzNo@2~ko1JCE z$VCHNyd^8Ymq&U#-f-^ivjG ze!v1{2)uK9%$LOoJ_i$A^dr)GP)l zop*S9*6^>!i2D%PU+Ln*v%H<`80b+Nm4~RA>EYh8Mx|w26xAt^;2rd$K$o)C&1IqO z{P^FB`u6Ti#L7c}jUAwUKFpDJRAN8!ptZAPQtCWbCT9f2^h-_cZG*(1_D)mP*dHs-`B8H#y(w0i zGC|1KzcMq7=^B!d=_EpNLL$g84^=)Iq6R#+$I7zAy*IwBk`#fNPnsOQfY-UV>5tv~ zxx3V{`DL$eUDPuSuUoXZ^8VJ7Ne87TXEEL9>tkhqhVnJ&9AWO%e!I39=Vc6qJS<`~ zboQZ%TK1N3eT6%II6(K`ZC-6Q!kSMiRmj@*9G+k5N&!Cjydv3ieY9`OjeoE67REPw zH~x#DP1FY#YX1#ibOZy$S(Z8wL=nPRp7fclf2JuYVE^r>RaqHCwPT7=Q(zq>+lJ)~ zMfD;*9V`%H3pvdu^Q*erI_9%IYh)y-#G8bnKY8HD1?${4o`}1a44c=09^DyrQ)fc-uY8G)UVVy5f|@6ziJikdLuz>dS6R5E0&ax8 zb34tHtfnxEyPz4TnIG`~gp!z}z{dTT@;ws9D2iAC!Wi4B(OAv@C zfoft3X5cq0%eZ-9_K*IiNo>zt4?`wA45dia<*?7@nIf3ZGj65F7tFT%6 z`y@Vjm)OViAsmVwY8~7({LgwjoL3IboJlSbmq((OkCsD#4GDIVR@;luNEAdA65#LV zRlq6dOCFii5a#tZP$6NMOYz)@2__Dp(xFW#O_jt*@5AyZ6c{-G+D{pw$p5@D{D0mn zGCTGBW~LhDouC|h(wpma}?W5WJ#`Ty$7D0tNE>@KNPx-KYG2A}&6 zlWJ(IMV2he`PT#X0*Iz@50~j$i~dY$hXSn(p9le#U{uVB68dZ zL+$6J{D(4=2d)u{lkZ6Rx|Yc9$V14jEQo=7gM^ZgCT3te7S6T| z;U-#^r%I{o-%JLa?wF#(kgnXC^j!3qy`A-Kb9vc$^z45VP>%oZqViO9qJ3rH6W5yo zMgbV34$A#DYIu14uZl%Ib+|R-9m$DCa@l@}g%VHwibhCXPInozQ~y01 z!z=PhUXgKlkrUlLa>upB&&VQEzo`9qso;r5ev|_UM&>S_gb5bU!G3xqqkscLa<4tT zg}I{yUQU$C#sozFl~l(QutyaNOy(0f2ZIrVR~TTaUD>SsdVvX3{bh0CWhC4@c|?x@ zbs{Nb#K$Y3%T1cZ$OUaG=y!Y8!ZD#xI-#;}I?H4*9~`?Ls#%_c({&Ka6iEWDUoHQB zgi?Dmu~X|G>x3^#H1p74?2FI*>-jHQH_Hes7Sp5-n99VfuXGBuX-^cDTl}Q*IN4vd z!vVbBctowEP0aH9OxLSBwTAxNW4ZMUzoDOTGKtbHVyhKVBZh+fSUsY8t2GjLNNIeM9q12Jb19u8ZDUVbcSbxNF4IA80KfFB6aemF4HYz1fwMcS2DgT0ogkZSUUFADS/obLOhbdEcJ5M5GJQRTdOuCseAcYuVgWEpxcLc9C+aeusUJdIBljFkXXVGi0hIdD/0a/wQ0SauTA996MlxttkCeYiIODQjNPBRJIVS5yooImOGu4qWM+/iC93wxCVxdEjC7/17w+cMyDaNfc7VaZI/FlyubZY/Zzj74Cec03vzIBd9oTpVOvkkxJwykF46YPun2yawSs+pDTk9Vx4o/KXacgLmCr92HlCpYbnFsvActGI2lKmPaCs7RzSdV9wOpoGhA9ol3IDJQ8qi3WO/Ysm3lFlTsH+riBSOLpY3CDSyGrV6Sc+aaUr2wrL6B4eu/MQw81g3Ty+y/8PhMGYsEE/IUi+D0p/FcSfETGh40QhNE3of5i6gf/E/qxx3qO7QCJzdmSmgrZjjXhXGZhIKqtSXdrB/N+sPQWtOi4ZoeK4Pry6+rBMZoRBmzDjtZVdyLNcjFTsbwusYUlgmo17sdiDPzuhVtVGzYU7AKk8Cwont3UvZV0Z7wVVD9srNgBq5eQtTSQflsG9QcbK08k5buxq08JS2dPCdJnR99kcqS/SIN5/dr/A2Sq9XDHPZz6GnwzzwFSRXmumj6y0dxInHWkZ5uK+WKzW1PLji0etlCmNGEG8VqoejhgW5Nk1L9zbqxjowSYo7pnRPuJHmHxg9aBQj9nsZHPToK39742qy/mGUF658daPYH \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_filter.png b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_filter.png deleted file mode 100644 index 8132c736a20d9dfbc3927ec73c69fb21c449ed6b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4163 zcmeI0=T}o}(}$@dC`Fn|3r#@~B6<*zBE1Mw1QG!uZ~*CqattjLQ4dW;QSg8?JyJs` z5s0+VdzTUjAQA`xA%su^p}gC*o;Un7w0v{`AnR@?^AXNcIMWddim!_gXH; zhu^1BR7*T0x%u+xx0zCkY7>|`vD%L`e5cstac*QeE~-4v=1ZntB=DHcpDvJ3hvc%{ zv1VpYhO>|OU^19GKirZwW)Ymf^i|^FfZ%)!zkb*~DP5Wp`>j_?7xL*LF&(CHL{axi z)|5=5=tKP#%WJ?+1+Ik`Ah>K+S!?q+V*h0u4Q2=~y(o^yPB)I|#Qay*QXt0=Ge2O> zV`mwc4kum9XB7ehgfQh?bvOv&c>{P2%@1KILn>M^ixTQwoY{YOEHZH!b*;^a#p$T_vd3_w*zNah~bB;MV`|bPYm)PWIt3+tRM+4 z*W&#lRd61OaUh<13d08G>`>=n8?$v|Kv}f)CDcXBlo7g%t*;xe#DL>&cwy;)CR80LL#|_>apX+HXEEtzQYwr8Y4^9 zZ+15_05TLXu|ETEE_j{lNBQR$3bN_!?3~oJyN2)oHGx{QR1xiOjMz@|{oYlc=YFG1 z5UFXNaXmPXI{j5{VRUXT2l;prug<|EwlajL8o#`>@C~FLX;)~Oqfo5#6~g3rphN2r zz&u%-%-W{$jRfDQ4e7z46eV6=JjlN3nLOJNMt$3d$6$1bj#CG_c$nwxx8M??IxvpU z`O&k*s&0wU>&pX~wrViS9VuB_=W${(uh37qJYNKpoL6E)2*!p=1t#5^txT+w@n2!9D zpSNB%{k8f)E=`@<^-6Sz>W42E@@*=%CO$2!ayq?wKWnM$jP1u^tp=a#0W*3ao2OyjQWJk+{skwmM#78@S5L%%&;C zfAo@-JX7!uk_v7<3X#SQYi(P+;#(A(-L8E5q?rEa(7$ot3vg)5h4#UZY67P>BqW4? z;Wep^i=_{vTwFd592V?t6L|WV5WzS&XkKkB#D}C+rihFD+U5?3^?j)3vXVEMhV$tP7!zpNciN@BL_sntLWS_*w{dbe?AW3Dfih=>_tggvb!j z#a7>`<-OD2AF>H5oK)@@+j}z>f(|tZAaMK|zI(b(U$S50n|Jv7xc4FqDHVIX3soeC znRopXAt>E-(V9=I$`WT2l2NGT)uFf8Hxxs@MrJG6;pn9S4UbGF@ZMkzbL3c8LQ6># zZLz!m>#&ZKfXV1dunaN9UP>Y2YNFR+^-R%{HH!djHh!GGi!?nK3A$J{H8)J?SWfl$m z(e#p6MY)lENfkm2L^2pMx2MbK!@^{LAaZP>Nc)j${dQ`cH!4%wo>&_&GaHzYYlk}t z20G}u95QX?4Wpb0t>wovR0J2^T~<<)N98swk+wJjKJCUtjQ)qoZKHGTkP^gf znM4@`1;-r}0)1U(`UFxub><-ps{@dWITT2Hr%1ov0#h#nEp;t~bnZHH8GA(Wp%BUFuJu?AVTEF=BRvY){f(G%w$SfTl^JpfUjvwAyjgkZPK($fwHTkMl-MR-^ z+@08c7Tt%&XDIoMbf*c#$i)P&;oK+Sy(7r^WhIx*Cx#@aFyoAH{kP&5uGAaYDqXdv zf*q=-9<{f>ki>Co&UKwj=6WpZydq@4LBppU-5ZfMx+KS_RQQH1PyPj2E0H+ zUXPZOwIP4+Ag?phhCq}$a_&1&3|(p(I}_@6D&D$bve!NSApwmM>QmQCgv;Mp zF;NAeVIrt#!k5z;i6U98N$b-{eQw70;}E{psaE-XBd`VZu!^KH}re@g*b1KJ|lp_v_x}!H=G#zVxAC zfO`MX5CO|P?E-2<)N8T|`u-YD_<>D>XS_Xa4j0|9vxcv*7^^m7c@3Po&q=OEl?@Fu z_Jsi0VIB{Pn7ADu?qVW$eMkHD56F2R!ser)#;-)K0NyQ1Ndj_~Jsn8`bE@0Rg92wS z@w^P%-*67w06vn-i#@3u_(!GvIBYmqyI1`g2XDsoSFirje@$#DJYwo2IjM_u+M<~e zSM%ec>}wa5ADOP}tB)O-3D{}#(kwD>8g0xq?3mE?iUMPkb?>zVU!u9#Nb8fIoG8To+3AXpLvLR1el&%20Ox zWynShTI}Ku`%&YE_Zazv#BJ+dDV#TWSPCYT301ie8(ks4iFJvt^9NB z_nZLaP;%135mg*yux^|xL17a}2+;ibDIb@wcw%A!qY@EYs;*i3+ zhSDBBo!!?&lh?YVNqs{5uC2@sbmva!;$D%h3-ZpHuJcu@0;6afvtq&ddzDc4Kkg&# z|3bd41kWNbw0)nEur0DoZ;IOQ1Gw+qMQ8+Z4NR#)KULXYcN~Q>2wa=Tt>isVw+vGT zc-Gp+PxOqtuaoCYZNQLW2HlaSECTApW-%lMQG)ZJL868F;$Bt;h!X?OH?hfdS1^~x(Iz}w7@dMV+%|Ml$$?%q5pBg_jA8dwU2(x3f z#;`lsA*FPkuz~jva~;y*S}tB!qU5W4FM$qw2uMcgrC-|Da}rUJAJf3rTZ4&*Cm4io?kyF#jEK1)7m|spC zHiM`{ziFh)5$2Hw3IiJL4Dv1Q9+|5u!Ub8s1NGV|c1zRW>NCUk(4=AT2u~7q{g}a#kyxR?# WLS2vR_Xd6rFqs)y8kXz3#{VBVC>2@& diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_filter.xml b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_filter.xml deleted file mode 100644 index edd7f9edaf..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_filter.xml +++ /dev/null @@ -1 +0,0 @@ -3VdRb9owEP41PDIlMTD62ALrNkS1jknQJ+TG18SbYzPHQNivn00cEidpSydUTfCC77PvnPvuy3F00CjJbiVexzNBgHUCj2QdNO4Ege8Fvv4yyD5HUC/IgUhSYg+VwJz+gcLTohtKIHUOKiGYomsXDAXnECoHw1KKnXvsSTD31jWOoAHMQ8ya6IISFefosO+V+GegUVzc7Ht2J8HFYQukMSZiV4HQpINGUgiVr5JsBMyQV/CS+316Zvf4YBK4OsVhcvcj49P7eRyMfk/VYpY8ZF+7NsoWs41N+BGnNFz9TAVfaU6VDr6KMScMZCcYMH3TzaNZRWZlE1P7gi0pNpyAudDT27uYKpivcWh2d1ofGotVwrTlH72rCRRPA1JBVoFsQrcgElByr4/Y3aHl1orLL7jelaXyBxaLK2XqWQxbdUTHyCWBemE5fAOfH1/i84kydX4adVQ2EkzIgy+Cw0fjqZLiF1R20ABdIXIe4k9ivv+ezA8bzDdoBU6uTUvQVshwquviMgkZVUtLulk/mPWHvrXGWWVrvC8Mrh9+WQQwRsXLmKXbwSr8nq1BKjYyhNclprCMQL3+agNxGlyzopWKtRWswCQwrOjWbYttVbQ3fBNUZ3YUTM/VS4BqOsjTtk7VLlaLc1XT3bAWJ6elEecgqWPSJ6ks2s7iYHq3xN8h6i7up7CdQrd/gSprTRRdpHp6tUCD91XP4ELUU+09rYn+X72n79Wahvev8vFrOvTfVz/N6eILj0FShbl+mfWQTXEkcdIQlf5RV66M3OGACw61ScJCmNGIGy1qCejRBd2YEYHq8fjabiSUEHNN65TizjFnGDv8WvsPvJaxA7UoKXj72KHNcjjPK1j+xUGTvw== \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_options.png b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_options.png deleted file mode 100644 index cb6f824ad3c000ba8f2f85a0305bc1854f31fb7e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6279 zcmZ{pXH-*Nx3&Wkgdjzv2_lF}uR^36LYEc-NdTz|f|7tVDN>{mxjL7+4D1JIU_-Td{u?s+^kcE9iB3IZ9UNBchBW((#`NQosY#0n-0 z#tOzthI`0sxZvOLbNpN)?C%Sw#6A+o5D*9-Oj$01lAgo9b{h$0G&pbMn^<#IpPg-j z760^euHGd-4inTK?RKV`*?AVnps91Sks0|eS=$>Gdc9mPRwyE~qVj_@rjO#*_EYVu)m+ky7VqzM0RrujW1;5vMi{1G5T> zrZ>Ny-93X{W{&K*ygR|g(=0X7z<7igv1v@K$VD)|D2J8E(j7NBxi+0kxywSc6GZ6j z>hC4gy&3l0G4$$i-_>%xX1LuiFe|HX7RssG<`Iz_7))n=>_d_38K=Lrrbd!1t{Kgc z+~bYR%U}xnPNK2uvVFY{ks?nYt?J747qt-Ud4{JWoSkIK#@~b+vOxE{4(?XGh97vrb;kh?eTasVfE1tF{NBRD3NnGF19@Uc-4iG zfAL!KnR#Z$`d`4-sdY4X2HIZA6UeCf;YVH-Pla3Ml(*G zPZo!`@ZTs9xae2oz%LhTE#2G>wp%(dXnqnYE?24!a~UnQe`2I~0zpDko_Y~-h10FP zqmjzJ3CFN`4S7*wgMUPjAq)CuL!GPyqoLGSyUnSO_helCDVw=9&Lp{HL zm6bPeqA^qT?yz_~aaUI5pUVYDn%cta<}>%Q);5oeTg{ZX8b5PJWhS*hzI(Q?J?P_N zdBPys@Jw5LC#T-!tVC(O6_?keKVd`R%_9mC1}(2rZVoZJ=%ew{N|g?rzASBLj~Jd^ zCI``y1$M;(bBMf2Z@u=}gP%QXET~->{;_5BhvX2jwG&ts_F%@!G6=~v7$FKPQlkEf zl(_V(Z9+}&w%={TyZV6x3PLI*&1-6@myxFQW|2i+LcVTW*3%z%=h}oeFQmsTn9tYL zUC>VDKl89f!$HRWmqOplOy$aTIu&`!iC&HFuGIH2$L{=^;PA~32?XS#WmIM>ljLc& zREYTlDgQ%$)4^t^nKR8G5XY3hwg%eYZZ*UHHhgqvgPw=}bUrfedICR@YM&1kx3n$U_eN`>*) zc)z{Ip7i~-%*k61PQR|fm}tEF(o_60{dQ;N*Ixw~Fl?C3gwx82|K7a*&)t`sj*Fq3 zN;=4n92QH=WM1fgT8Y`5O!f$&&!*bQ!Gd}?hX;c06qb{XZmOGkq#--=i~0v0`>p37oD(!il` zxq+nfKrOoOuheURNTa>MRu5Q+`OtF46mER#YwasWWMDKiPqR&y1B7lvat3!6fbhJ8 z10HJ-$&$_m1wtBI>t47i0EgaV)T$|g{CvLr`}YT8Q4xomn_CHgE!5K9p1Zuf+#x8) zCNVLw21S!y}cd%>J^Krz5V)1xdt>dD~r3Zun^p_E)q%X9B?%A=D$BRInP3@ z6MOb2As>DpX`Ct>&|umkP4e8-fOgx^kW5K&WjV-CoTR+GJO>vS{S!W3UiRbT;~Uam znas@0<}YzL+@Q|K!t|7sH|xExlvA?3j9Msj>7uppsV7*FI=Z#n(&!nWMa|{g)Ik>xtNHl(IU5@r%?lzQ+uylU)^u^j^9;#!YK=z% z0=CIE(D5+i_KQoQ2s>9Ton6E%G@AMrfRUnTKZ6*WN0%r8cb zwG>G?9GYF&Jky29d$U3Q)5}Zfas!^ABe ztvKbcH9R~V9j|_3*|628`lyF)U5n-_hsSf$GMbU@v29b(s8`qla~^C|09 zgdCZUP_xvYy!A$&h~)>Eek*SeIh+ShKY;MFx%W`vSGf%(XqDQ!kO)R21^+u?cJn z{8QrQW>$Gh=l4wMQ#|~!0Y4yzn&#RlxRkEaB`#u{D z?bbTBTOh!WPq&9mz{&w*yF)JLy&A}ww=PTv|H<&5OR{M2jqybanuA}4k(Q4h>ySt! zZ4?THSMPnVs^?==U7qCWT=cHSI7(c3WQbtW69OED*Ed2=kN^BwrW9^AZ0QLEa}c(9 z_#pmfpLmD?FSz2&2%%3AJFzlNuYHE6pB_p*C_&TjEo6nvSfQ$E2Nkctp~1?=!!M66 zokZb`(O7vtNqp+)aGIQ_I&v=hyH)RJ{+tgMAY?~(G$u?F|H2AaB-S<~4M?)3P?Ao$KpG`<*SmjcH8hiKB{Fb%S)Hx5=8_dkfZWd_`5hzaF*W9rB{jh8 z**E7d)uZi<6=>HezGOOPPZ{uNTWp%7`g+Zu1AjzI+ISdm z%4{l#r9z0jZ}(oeg3wy*T$92kG`_`;H3h#)so+x^D!+?3z$x)o57Pp`pDPDzN8Fo_ zZ&r90hhpoJ&*M|qjvRc!;%xaiFq+B8Kodq!6uxH4ARhq!Ryk-k`v*!(O8UdDfPk!@{+rd=ghsuD6KVYC0E8aOu>UHp`L}K@ zNv~1_H=g&NnF9|58K?31T*@Qx;uWsrABqGkpvJYMGo+grUMtaeV8$`y3Av?6}x zmzNKDb(eNp1@|9$m^IyF7T%<#kXe{kH+|8RAJuNVsuM#m@$y zxrrCr!XkEc)onkmThw{Sr`qT2za2(491i&{-razclhZui z$;rvwUIpFw^Ph1LaNabnMUl^FpDST@G?E~!J~*Q*1>-2%9-t3;iJqJvj35mRm=27O zOMIIPNO{GgS$`r{MWJADJtl~R2^qIyM$%vS?u=z|6RRpJETyw1<=T?*B-keu)Z9=K zXvo-W=Vv7Cx^gH zepKCYUqBygQsPlgwl%|_9OaZI(@7W5Tgn?jhO|hj&B&InY&Z=3ErT6NSCSG0jMy!h*3S zvwvB)9cNe!&CP#tcnQ*86uO3>cpL%yA*QJQWkE_ty^$cYvHkmb-Z7;{|Wh zGMC*d6>$@2gN}z5Hg6UGVZv0O3=h%y*7f7;vQAW$F+Q+wCtX2W_x4;R?k*sh!boD` z0mTuzoTZEpOv{er#m9L>h9Ez^3He$WpOOIf!v8QxLp^S;LU7wV7pqz=pB;+Lcb7MU zNkD39QA63ONoqQBCTheNnocj@Sb-brP;h9a;Y^4yJ<(Z&`p)GOWY3cT=*ZfVoOj)lounC^yb&2TP+k(6Y(IACy{>6Jil3y>2M$glgg(O#$BwC8 zWHYsa_#fx>YIJZ%sWOZEZYP`r;CBC~Sf!~hn4&c4f=!5u85IZkcn{zs2lBaU9JC{a z?-H`trn@)Q;~fxyN>J~_&?9>d@|iOpSpE$te80%RgzSaCHO_SC{~J&qAB|>4_UdNF zWjMY58@Sn<0y&55Mc4JFxw8EY$dulB#enSf+GTzhSoSv{7{hQyj|R@kBxDYTJQtsE zDROC>c>?gYOw9p^DmWx4=V7bOW~?JSA8H8d2+t;3`_cQ6oH6LxpE-o#-X`N^cRII1HHy?u%ltD*DMd>&>5-{xj*<0mx}_ zHe(UslrGqZ9Q!1Gzbh$E;4224T}rT`K9HKxz~)HbfZs>bs4#FUqqX+3IjhHo3*Es;WA|qC*zx zGVb#N1U+5Vudn<0!DX5^75U0il9JKO%h>S`j3o7=-x2uY(*zL{uQ6os4j}P^wS~A| zw-bK>NP7X$jIU2O%ecST-1K793l|Tb{0~Kb_U6e_z|x z&F#+kX=ztAh2QI~T&g{$k3yX+!Y;RY58y`>j@P6OV$4B3GX>c~5O5}I>^8c;i z#E(8gge|NvLN)A+SpSG%g|Pu$lyCYU#?DCEOimAgN$qQBaHLB3vL=S8QhAY& zZwpiC6HY>%qRaE%E$R~9F+K$7uyZGvucYtIkLs29dnMF=tzcYSTux4o5Mku&tZX`w znOe~}tg^!&mcYW%oNDqPVhw+!)X3P@C2qxG;>U5vY50SgmNOf_M&%2@G+cly{P@3I zp*|=$u*f}RtEJT#&Trs5j^hunGyM5vFHAgHM5zgIbxec4$CTk=OTZN-{MQvOTL;VL zt+YF_ywZ?r`L;BVIV*#7b8+!}nT9v0XY%?Ea8h}W-z!bQUi@O2^Kj?druW{oTgFVV z=-u$YxiiGWLQ8s*Wvr!<&Dp~uhjbUF#)hiWvQD^X(LR==e+kAEChk7Fqg_zaS2D%- z9<{T)r=+WnNZR^xoqZ&FVPXD%{9Y)Dkq3TGLwp8;DbX3k|EgNVO<85#3LH@}x$Ohz ze20KyD$PHaL_TsQSY<<`N#u>*(%p4))D9D-GT)%muCfX^@qId5ib9pFDwpqDq@i&X z8yKFv0>zBP2&Sz^^TjRYRTa&b_otHh(e9IV*NJ`IP%jxM<_J11=9BTf4B?fSM!7cwo*PwNhCoo( zwj%(mrXcM0=dY=QUhg6-|E$`vm=sr+BKu)AZW(_P0WCt+O-@&V>juGBU0u8CYu1-$ z^@`y_Pjk9){0i8Ow>LAB&)S-j8OKkFoL&C+B<`L=WAyS(>PRF51t=7JfYLqKH~iO~ z-4nTPf7;Q`Ozcciewo1u*f^{?U6-0SXDF)U#qBgeEyr%8e%J?*7!7$^eJ~)TPZ9%j zD;k7kM%ixjM*-uPNRCE^fKvl|9nK7I0R!*EJ(#f$ugncW@V=*}PH9JQqw(%jnGt0lHxyM8#a! mq&a%Kto@!*BLF>f3Q9K17qGXjxd6Oi0O{)>v`aMahW|fNBq8qr diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_options.xml b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_options.xml deleted file mode 100644 index cf5f77254d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/json_options.xml +++ /dev/null @@ -1 +0,0 @@ -zVbJbtswEP0aHVtIVLxcGztNDm1R1ECbngRGHEtsKVKhqEjq15eMSK1OnAZG6osx82Yh+eZ5IC/cZPW1xHn6WRBgHvJJ7YVbD6EVCvSvAZoWuAj8FkgkJS0U9MCO/gELurSSEihGiUoIpmg+BmPBOcRqhGEpRTVO2ws2PjXHCcyAXYzZHP1BiUpbdL3we/wGaJK6kwPfRjLski1QpJiIagCFV164kUKo1srqDTDDneOlrfv4RLS7mASuXlKw/97sv+W/b+7Fl+h+HW0vy0/NO9vlAbPSPvgOFzSOfhWCRwRiPc5I5IoKXthXqMZRI0XJCZjuvhdeVilVsMtxbKKV1oLGUpUx7QXanN/WHQ1SQT2A7O2vQWSgZKNTbPTCEmmVtLZu1Y8lWFosHYzElWGrhKRr3JOlDcvXP3CHnuMO+Dlz1xH138gLnyPvXFhDywlt6ABt6C1pW85om5EEnHwwi097McOFJnXMC9RU3VoKjf3T2O8X1tvWg9C2cQ7Xl791DYwzqDJuX/boubonZ1CIUsZwXB8KywTU8T8hkNEan090MLHFgYE5TALDij6Ml/+hKdoTvgqqX9YJphNDMxGQa9G+21YNl/W00WrSKJg0aomZNXoUVffs1+tsdQKdnVozR7UQnJUWZstjuhReqoX1RAroVFLQbv/50ab333Dh1V8= \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/rename_object_member_filter.png b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/rename_object_member_filter.png deleted file mode 100644 index 054a06ff4fd2f45d29ab432577405b467313c9fc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5760 zcmeI0c{G&o+rS4ChDb&TWyyDJ5h-LD!X!1eq_Q(*DTeI(GExj7WzCkQg&JEJYb1m0 zN@UC+yRl{)%PjBke&6%{`TqI-^E>B0bLKgp`?;QZ?&q27x;{7B)YyQFLx=+a0B{*z z*S`e-FoPN2o!D6!pJW5aK>&c?z)=6{?LfPgY-FbeY^1}06n_mQb5)R6cOj;ery_Xh zK<|PT0mlNnQOtA9@L^PeQ`(gEPJvh~>$9s4QC5~Gwg=8R@yMQzbu|%vsjvG|x>5l2 zcKoa1(0oXLPh;_%)yOKiXIW3XahaHu$h zwiZE`VtRQ=$%UCr+iX6A)9*X&&=Rops?P7=c(md}@TZP+gv8=jS9@R_+FchpVs?a< zuc>f#AE5-zF5-#z#aD+Y!k6t?e`4C;& zkPeWOfq3@=SZLOHJ(w{Uv1r3A2dyuBV8i!!Roczr55SKa60MGqYURUw&WGsXjM?g) zTkx4&CRPd-nrV&g*I|F9@-;YA2J+Il%>DWMj84dq|C#C|2xSq6m(vMMz|sj`9FajZF;EbS)kljBq8XjZutQEAO3J|oJveE59qwH}9(H07mg8f@hx|tt0M}B*OMyn^1 zmoD>xeOJjCo=q%t2i|oJk^A zrsM=O6!T@hv~qQJR>tP@+P`%HY0b{PLK8Olw71W5qa$d0^R2YOJ7k%ft_PPr2(`n3 zOCx0gAMciVWaZ=2pK=P+;yJ4)>w=vsI?GIQbj7}+qQ+9{@tLRhn3xkE! z?F|ihWk%$QvziOe{kbcOGU*nsGw99NZlzweL0j9`MHbn@IAbck-b%d9lXmHSJ*!~X zxGS0p>b;SKt`VH?5RLG~PK1Xz;v{W8DWMCwpm%M&p1Pof5w%dGc=W@c@9Lbo)41xj zCxxeE!$~0q^I7FjTYjYKof-4L5>?^rG}VR%e?dehdB~9y`(Sk)W=@=&RV7iyVS4Us zjhn{yS6^|AWMA`NvF^k7+;Oxf&jI3;(diZVEYsyoV zD`2b(_>xUkj^B&WGFF4!WJ-_FlTJzfqU8A1^gO`|KDqzdrGGzfB*^NO+hk9Bg0S^l zKce=2+l*ulGd}T*h6wXhLjlngnv$1-Rn>=oyt`g%eS*YN0jGDkn8XY6U9&6hcW}#2 z6kG_M)DGndSy$dl(EnDDwA7cUy|&WcYwGgRfd4G}rN6dgsy=h@3Dpcz;GG2Pwp*S) zvAslRDAB~=t$zA{-*r5@ zxYFWJ2Cc3^Y8){0_Nf`QwEd+p%KBI8lM}S0@1ha+y;|8g+m!n?rv=Px$lp$A?;iW} zvFytRUBFQ+OpN>9(BI^x^7Z@TU+A<0>e_1UbmE-|*fi<=AH@?DldX{FQWET6I%_l5 z9Kdsq=Yx3^cNaDqDK^nb<&kQw^=}2--4h~V4M=Xw==&6fDe9jX(HcZPmFWrF*}#@D zj)vylDbD_bgSBSQJq#5LRbO=?*N$K0ZMT9)qD$d#@}%-Ye~dAsc<0bF=9V^NaUjc3 zMbpu`m;$tTDo#8mN!io3>hi4wOimzD8vD2 z2{;>mJKH(jTGjQR_x&UwTLrgk%3Zl9Z>uP@MM9sL@wt8HvShE;q0D7@YUY)^H-OJ2 zX>P=ZFy3I|L1Pf_~|O1g{MH1g33ro@yG!kTAw+>vJp+Ayq)>@Gzs1= z0clIBN9bx_CV?(ef-(mNr}@wzcguwnHBO3#OR#)vrF;SU`a{#MQ9ud_MIxd+R160x%v%yCJVz+aT=cl~Ah`MnMO1+agknNWkZEe49>#6sB*s&J#tdSdM zlM^D_gDH=wOM-26399MaroLySKy|X=nTU_2jBI6*3bMQegTLYZr2Rp10Vc?qW7p}0 z85c@EX51FNHQ`|_ID7~L zPXRqs_vDHmV-G~|nt9Ep-rf0?38Lzk$(`y*5*3xicgfy>Nu1vMg=B={7lqtOP z&a<=cNyr!PaQe36esbX0SGQ$NGZ%)MA#M7Z4j)RKfANEBO?Q@^B-(zyiv<;POx8c9 zS1`d3U8K|=_gW;zJ`)`jM<;}*{73ft+4!b z1amyAhk~Sug_Ev!K50|R!UeLmcgo#os2I+NJm+w$V@%|(t)In2V{d-{y9L?wQUJ)u z{esxUR;#VAvxMg7TX!selGA2}%}umsO+w>M%R-zBMi2$B4vHQsiBm1VH>)F& z`|zmnQ;MJ9u1ctZ4ApDL&S%u3+!W5BsP2=fvi9A9ZX$F>GQk#s- z=c*EKn$crJ8n9RNBZtrxGZ>LZ1-t^d0>2pc!_R?EEI;boNO94CZsz}d>c&2;9){F&$fJI+?2J*O5bQ_tzjsObB-z$|Bph1XSzVAsvL>VV9$%ZB&La*He#io5ikM_nRq)!>}&k< zTJYm{=&kC8IMnucLVXy_{o@9ZP1@^PA(7nR`x;ySQlJKNKvc*{9Btbkp3W-Bc!4|~ zYJYV%c`apOMmTus_9@a_RQS}1vCaUUp%pi^GETX3cz#<8L0Ms=SaiO3+VpQphod&W z_rtsUsoT5nq#-}hP7@t2 zurPuOMRqDovEMs19|40r42EHCQCD6wGsU6#3WnxYj+AI*esR!;d}wV2^D-?=LX4n; z0Db?H1}%p=@U)PGL@&(i(Bzc$*5Eetm#OyS~*_-Z#IL%(in zXoT%oH9P++@U~pRdmBzz-^E*4hp)`44}TO{P5WOqO;6Za9-ZbEg)XZg@iTWYR?^0I-0G%niz%WAD22;SDlaAaa<^0^pq*F+ezq+$)(EBSgchh|7*7$RjtBVg5iTiJj4o~q}#X2AO7*H$mac`Icd z^NgQ>iDO!Rtr1YF;YYgdX2Xt>2#Etm*|0@8|F6;#@2Ww8os_-JCQ;Xj?O%5rX1mi* zYzXfZ@S4x&&DAp`3G4WcwO@DleU4LAE^rzACZk;RoRw>aPI5+M9S@6?arWbQe)$bu zfarO=+?`MB@e=$EoAdn#OMN*S^J(WTr)rR6b5&x=uv1gOs3P|)J$z60m7ioTlo-__ zA!PB#+pE_9ls{~Z26>Q%il6SZuu@NoN8v)cQe^A^lZDtTV%}~1?ktO>r#sedJT$Xe3~qO?OL``V=h;7ch+Mh<0PYuZz)eNnUk-8^xBdcl$!2Sdspqj7tk^!{a1+6Q1wX6e zLHMoSdG8-cD+V}v?P{LI^rMsKrn`srTu+3To=8eVC$0k^vS%w-oH8#z?AyT*rHFJkc>{H1GNG^H!Q~DE1J@gLpvv#nO**h!R;aY$y6; zv3aJBa4hJgM9ytUmGrkAqce;UKWaT)o#-duWGpmi{3sNc9Ei>saAA8~s?%p4ZL;f! z1M&~`W~pNPxKb=2aqY*{Mb?L`FYbrNjL$``Rd%N5WDXz@Lm$jAKBxXzCKo7!oe%|G z+Hq)ccsMtVnNfm)KT$vl_DkZBR;*Mx4h1vgx3|1qo$&&9*i_kF644GPG@nA97{}}U zVp=J`o{mPBU{$tiqJEoiQ3dzapNIV0QVo;gu{j%a-|PLB2;)s5))QE9Ygs{-2-@y+ z(r_lIsW%^MS3)Vwh`sKt@(+rgvF<0=f2RPizd6k;k?2?_D zC3TonfvtL}6k~SCaZu4sM(rd19`ydVGAUk-M6~J)x&lEj0EgrM=9Bj5YcYIsxC>s4 Qon?R_%virv*CFb^02y7{%>V!Z diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/rename_object_member_filter.xml b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/rename_object_member_filter.xml deleted file mode 100644 index 7c39b3d9d1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/rename_object_member_filter.xml +++ /dev/null @@ -1 +0,0 @@ -5VddU9swEPw1eaTjWEkIjxAopRloaZgB+pJR7MMWlaVUVhKHX88plvxNoQwtnSkvSCudZO3trZQemSTZqaLL+FyGwHu+F2Y9ctzz/b7nj/CfQbY5QgaDHIgUC+2kEpixB3CRFl2xENLaRC0l12xZBwMpBAS6hlGl5KY+7U7y+q5LGkELmAWUt9FrFuo4R8dDr8Q/AYtit3PfsyMJdZMtkMY0lJsKRE56ZKKk1HkrySbADXmOlzzu4xOjxYcpEPolAScXV5mYXs5if/Jzqq/Pk9vs855dZU35yh54QVMWzO9TKeZ3jGtQPX/EcYOjhWlFpmXPo7eOJCVXIgSzj4fDm5hpmC1pYEY3KAvEYp1w7PWL6Op3u48ApSGrQPYcpyAT0GqLU+yoP7acWlH1HcebMkUFFlfSM7AYtaqIiqVL4rBhufsNHvef4BEPSROYy8U9inOeQLIA9Yd4xVX5RHKpdrEEdn+Ip1rJH1AZISNyQMK3ycRg8HwmfO9vZmLcykSLVhDhobEG7AWcppinOpOQMX1jSTftW9P+MLS946wydLx1HYEff+MWMJ1KlOmWYbuei3syB6lcqQCel5ymKgL9fIlDWDO6dkYrGRt2JMxhCjjVbF23x64s2h2+SoYnK0vXqwvGJw0h5Oe2UVU7ay40bChv3FgoJ6a10E5UxbFfpLNofR7704sb+g2ivevLKayn0FHxZyIGxTQVmDa88BiNFE1a4sPC0nW51QtUSAGNarYQ5SwSRrMoFbQPcmTKlOFVdWgHEhaGZptOp6h7yVuY8P6wnkmvw4RJh5L8Nyj90e2Xq71+P/2+PfQeSHa23h/FHZeZ89h0SYVz2coFh48GjQTMYyqQuMKRcfNqwLtfd+/psZ1E+/+Lx7oXZ8Vjf6W8f8RjG68j/+CVFltUr1uoKahXWyx2y4dvPr38+UBOHgE= \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/ser_error.png b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/ser_error.png deleted file mode 100644 index e0ee8fee082c1e9a6dc91ed4f1f5005df6d4412b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9729 zcmeHtS6Ea{v+jV1g8>G~N%EkG;D8cDm;r_)LzE1H3>gWMbCjGxCFdYW6i{-KoFqyR z82C_f&S@Cf3%>u`|9<{+b*|3EK3wojuhm^$U0vPvzEvwkNkRI~P1>6v5a^Dq3{nLI z!Y2cMH-QO(|H$8Z2_O)`LpN1T2P+q8JA=1o@-{}smLQP4>aX&GXAm51VZZAHCDFTW zFh1`?PNw=yJh4UB<5Rj4tU8AC=CcO%XY#e~8(oI5PV;*kOvW;IV)l(klq)#Fit>)?K`u9Dg7kEt6-y|)~5%=t;H4ifK zJW_t@IPrD!_&RtUdfn2%C$n*}Psw{HsC4c5ZdP8};gr|z7mmb7{w34$hBu-n`bR8# zapVo<7E^Yp$%5v4C!iyOPk+LJxvB$fCq7-=s#u^on8PNUs7SRAP6kv*5e6={BK-HQKmc(xaHz=)5dKRXNJ-{a-K z(Um{TWUP5cSo4=07H^j6hJcu2Nz(i+Jnk16*mmi?JYS!`CRj|Zos>Z1J_lV6o@w~r zq{7YR?UP9HbhA5ifhwPTHor$0YdQV-(cuwBZJNrt*ZyT>1x8<*O`G09SE>L?=T5(g zx3d(L{nBr1o|dsm_Wk!<2von%eX74=Y~UdKyh!R^%FKe6EuB;I^%;)Q zD{Bm6id7G7YEHE(7wyoL$~nIpHO<%9t&-szS&n5=_IpAK`6s9fry7|oas7aG*W2mp z27(12Et{wz5$Auo%o6L59#|^&dlu^!$|$Uh>`c67$rLB-ecxiHOv;T7a|(VP)_e8)B-Zn0I;R9^nS`wUT@Gg7 z0zaCUJC>g=bU5d(hN#fM9&F=&QVYjd!R6uKX_|0 zUgLSs%5TdEX^~k=_M%Ge!WiKmdKVVF z4{+C@+&ift5EDoiDW>Y8w~=bqMKS)Zvr4wnZ1|=!z8EW-=mi~T@aSQ|(YvpscJQ1h z@|H>?eoFBBep#L&qsK|H_n!t6S{|r8t{m#{7oX%b$CqU1A^4ojh<;i0@uB>b8JVo% z%zjPLk5@~Zrgn?Z_bQi)g!bs7pAWd2Y&9I(6ek{}e>+MErv?7(=pjBJ0u~7MQwvpp zy_NFYc-AMz%R~GT&G*{~zT@G6+gO;;e)AMz*5_V3(q1NCyQDy{m)daf{A--c%~gk^n;XOVE~*fJw;*)LjVr8Z5sp zjjVpOMGG6+1-(9n1k3u&Q@po6Z~R{_9+5Whc(|@rWj+F}G99=lU_D)ppZ&B{R&YsR z)`Rc#Z0|VR^+{QEI$%#+&cFnO`Yk?N=?rm+4q`3}uj#K-L6J5j;~d`;0>V^46a{n) zX4p|ZTBuRyi9K{hDn8%w*Qs|YyEysz`K9&dbj|tFLT`NCt2(EX^o27bGH7)KoxtgA zmF2JB{FW1sS$jBJGCKpq6ufn+tT)2#cjsjVO#A7b4^~GF z_}7-6m1N;#F2=uvbS^MN5_An zglSkGaET``&6h zzA06EHlT#6KmHxo7|QFqYjmOL(T3++4(zcp(eOQdhzz@uDyoaH&svVCd%hIks56p5 z4`miRzzU7y(&Fv&(z1)TZ41nCkqr&ZNr+|-+m2pHOyn~qp3+Q|M9COS6;iR-_4M;C0}`X z)?jALvh`d7wzH;xF&W}h>v6LCSYyUvL|*Up*`{;T+PZA_pEvzh0(fo6I^4iIGvp&e z8ODqy`GiUw)++C+HD$4hm4~{04%Dr88QL8CNuXk^u|5flY`nt(wFCu zR>JM(yS@xK+5~agcMYtK74PkRMh_roTm#W)GqbKAX(Ce%yFCvXW@kP2e$4P=wrQW< zU-n=kVuBJ57o%?C^MlDL$j}XqD@^dUWYS&x(kl8;wqIZ1uYQ9Ce0mZZW}e8pCrivC zhAIs^BM-Wt=}HY5lhLpaOwmg&J}}IO>v`iGo6hVOdMKQGA5|SM5R(d(w|&R+WdTel znLk8`59vZ?)70cp!=NyO(*gkqvTpM(Rb2Sgj}~8j;49Mc5XTUi4ZW^ z<>3&^czHk@n?!YkfC=OaCMXXja^;M3Jln>y6+D){Tf+tGqNC}SXv!An7wmXm6iQTc zF#ERk&iU-OH{;L!Ow(LDFz#CoGbFRe9Rg@{<`y0+!Wi95-{ZjaIs~B(Gw5Cvt7Kmx z`E`p$j>o2cr{jxxuU#j0t4Z-AI{IDa?;2((C*ccch)?qc83HvC8u5^7@6h4dUla$xyYtgx zqHI=Z(qHVB{s;*oa(TTwsU@xzy{8fdeR21l70Ja@)-Oz4DoAz zlv{hpR3`wQ=5=cBJnMP1(;@BQ{M3S=4VgV1PzEuh^yQ4zs4%gdn&iY~s@*)%yXo5S zAsYi&eB%&a5rk01SA&-32Uh0n72ue!-k+pF8soFr#Xj)L|J&b2yB`*mJz9Pq2hUge z%H$%Iz=HOp2BKgVY4FGTESl$kF?8++sXy>KtVlnxVOTJuy7bkvAC}BY4}BB53&%ok zcfEEzmWUG;J=>PJ*hpN@rH*dyetf45Nhp(oj|V1aRg&K2G|>{)8C@VLAm0e%X+vT? z`oo!(=zMEqHLS=ijuDR7OG|j}$6V;$EFix_l1ADmjk;awoyb6N`T`M474!(-@ggcK z>Ym7IQ+yVq`va4UPVY%Sc=H(Gz!Np7JNOVE9}0Y2zDjU{;9;S@w%Fq@PF!cDbqr|q zYfNt&69^AH!J_2+5g(=;bwNsY@YaMJ3|zGkFxc#)_{?!|4S8TfKF2!b-*03f(VPJu z?u!H7=X++ea=!ohni*2@;y)iQiy;ch6-7^Sy!WyTuECTkx@T}q(ABxV`SgixuoyhI>v>F2P9GVns}4#GzMO^R!6B<>!-X7eNRbqp>%hW51a^0$ zIopA&3Rh)HS**eYwG-!Yd4|qn5=;Vc2gC>Afv+fX@L^U$*@5Renx!$y2=I~r9{d6_ z$QiJn+hxxL-5uog;znl)3TumBjf)`2dBHp@XM4Oo*JtAaXle^VXWi=$2X+lG4h3n~ z%UlFXmB7>A`hOTEKiWtAfsy?u)rZ5xZODK@cZYz4{2;#UOLO`l2=L2=UY^IVn!7)M zkdbCMuiS+xlbA~R0?P;a$`H7z0gq!cIBF$PKU))(FZTt(#OrIVcC!s>w0tox@;~N( z3XxHNfDQ>M)F`b57GIZszMto%jy>F1TgSMZEW~3p8h0r!PO2IY9#wf_oh~X;E)J(` z>MoByZc(7gw!(BkfqnOMt+dX#H?9VLz7u7Aes;3i2|q8ZI~dbR-ET1LBtDArK5PAv zA~fQ8P^j-%=e)If#%+k3Bk{g3;-#m-4Bfr^$K%RV9xEa)Obxto=>pSb`nYmCcjKsz zlf5NOt>fA>fE})aAc@!ZHRqLK_9NVu_bhHZ2<||}p|Fqt8&J#np+?F28JCqC!|-fX z6V7V33QSB&;*Fh}t-nv2$&}70__i72x|FJ?>)%uej^R(ot()G8&?lfoXMvwiUfI^i z75~jxku8?wtO~n!(xhtr=03Ege_@YG-(!g5d?;HQYs51f#w;#D&`H&}9gt>|E~BsQ zIcVVz_Lsir+9qfhAsGIwm(sLJECIh|;J#`5K#TN-dS6j|`A#OQ9h z*tnbi^tG1EX!ObMLhD2qZvfQ(Br;sDmyQ$akcbqnT`Q^ccq(2%DO(0PPb25!L{vbTaoR*Z9QVwO- z;yo8Yb@p~{ie_$#{uCOxo*w-pLF>KiF@F%Z?Q&qR9$MUT<$zyyY-jHo6)Yy6dP(Ba zyM1WzF$JWK!9;wZaB9DV$G^dSm**PfyAC)uYf)IpytAtRVW3*@`jb;{C_3vxv>E?O z`!2E9ITrqouo%OF2q6mC5VeNE{mXGqWL?@32-HtTj^L2fL_QYa86J3^Ejo|ZH=&ED z%5Q{}i~^B!>qCmbRf8LFdInNRnXgXIHQ%gJFmTycQ>6PPE+?+xa^3#P_|Go+Q5Famv;O2m{{NE1|8LL3w ztn+R0e^1veL{$hVmk+N8t^__)G@z}rw+l0L)wDnOatE|!GfMsz zxQlKP5>UMu44(*V9*lk!A5P28+-eX4+Cg{I9a?=B)>|!+Vr%#JjJh2dpojX&p!?33 zyPu0qOXo1i5&PSv-MVxPXMho{ON%hn!kjNlKV=~na$^Vqq0wyOPcIra!$h|Bi>Bh8 zMdjhMweX^;afgegZ%y3Z`xUe?@kC1+5+IoJoz5P&&ueHNi!YAY(Eh^D8%5aGT2J`{ zcjdaZ?Hi-PSu^HN!fpPri&5LM_{4HYY)@Zth*j}br+rtNWW&>U7D$?bCryvgEeRxs zdh2Sa(OOTnzA{-)%_Bg~r1>>Y%LpLkk*c}m{qkgjhM(|^wd;*ODW1|8eV#lV<>;+h zRr;p;_G>ojDL&!SL zG#1+L8Cr~OguzUEJ+&`8abfubt<;VIrEUX~6t5)NmUX7YI%3iN81(Fw#ll1*%wKAG zD@6paq?#N>Bs^YhDyMQ^U(u}MIXo*qu)#2nZB#4{OdOIQ0JBY`nR{IX6B&wfxs{DS zy|Z>W+Wx&q@Ji4_ucF>amh%i+<#^{L5n{ju7oocAjtbvvJsr!xJdV3u+<+g++E?Z% zc)rxnmBhuU0ARM(=Tz=+RF$B4Nc<*R z2++5`+-vWfbvn#hN}%&{`>8Nu9fV-qPV+UA>bx|!o^Dg`(~V)bVydY<|J>~cw=>r+IhTSwG8Ih!i~}^Z_z+&QlB~YSYc{M8J;W|Ik z#GDO%a3NJ!M@w189H;Oahc@Zo8P42cDKh<$v{0g3^V_;+f%2$dc*}*<7C^+r!w!jZ z5dfrn_4=sxbNivZww=rITo7jQM=~Y>W;oq+eo#7Ll5+@G*)8XSqHo1}2^uT?z7A7X z+&8QD*tAc#FKs-vFjXQY52Ji!hdtUV?Sx3t2;%3q`R5-29M|o>PNk*ooAHu^+;V^! z*Zjj*W#2;`MRgiIYJ|Pd>q}9nWPU5n>#QeiOp2xUOZ^9)a+h4WW+YFs?%fTN({$;m zdmHJN1F6DG*pWPDZhIx}Bda;ogq```Mu1dNbkc zd(nUlO9jhABMaF_c|h z)jy{_VRx`BnUV+XT$CWoT^4v0x}El9c!@TA(xslPtnI;UK>%NzwFy7nZY7a^*1+SR&! z(>R>GBZFqyH>T!kXhT-puoi^}9Cp97(B}H2#u4i--f}LUnf0+`Fmli-!t=zPw?iB@ zbeX;gdF<2Mfht{7pzcqH&%z0EM2`I{{dc0B;IRdn_1d3dN#Sc;Snkzu3JS8)`rS}~ z=^jl3hb|R-ix{jb26)Y2vR~ZvH+pKl!CIpXI_|yT5TgfhEl%Qi;bDHK)unPH(tGH& zzC^wttD-&@Xl#h2%{>)K2yAHMbB&qt?;&Z%4OM84iorKtW~YGOSKY>Jq(Q#2ATRXZc>@io!+N|R*MNWpfYT2D=l&(1}t_bzyfgZX&e4mb+ zXJt1(w46%MUoxyx%Vz!nD<{h;68x}(uW?mJYg8&b#sk<(-bH|RB8XY{US^t^i zLdva$^cYK!J?0BV}@4RPTegYWB$fph=W5Xp7+zboZ7XKK=S?oY9YQAEU z_uL&i|C?ky1xN&*l-cE`v1a7lybGoEG}VU3e{I_q7cYuq3guO>C)(VIV48NFW5OMq287_+Ow_cotQ)zjRua5IU6T&gfE_d%x$3oc^lLJT%?^y__s3xkO-hm5|qd zHlyqJajR^TU8~5)u7T|4D-vT~>@)xX?;_-rhi3A;SNrtUIC7ruxunzb+TEZ=t3f$b zm#$lbP_!CKGlHsEYFUSW*If<9>cKwdAj_h=s8rI;iI_Y&b*-urdP2CL_EHKJ{K`PL zGn}FgIcuFm!^6(W6^QUD7qjzq)T)!wmz1 ze1Ci8EOZfQKZHt2jimE&O?$Zin6CB2?Py;}ecL2iJ3JV$$I;Mu?+3ZlNlBup_sS2# zFtwf?(=-_RD?H|O>N%=N_+=^7t|V6og+59hObuW5;!$36Z7(e&L1yE?a4yjh#W!~^AkHHRRg1%_Ze(+Db zpBWnbH!TA=$qP~h>O+gP;T6RK+7Pm~OGbA!F!yrcTsG{Vb}$k8k_BEdJ{6#6?)DJ% zzuI_h1hBsc0XdKd_`xA6$7Z*$gaG~j&Zk`o%^(M|XAeA{1pYG#Ozid*xd+-xp&)b? zxhU-w)dbo;2~33l(gbo}5@0Mh86&PJAD{#R1-bUOCV1cvEItYM1EMP{%||{N z(^d^4uG3vomQ+F87Ll{<*3IuX826p}6Z!I9+dO#(A}9jtL7E4*ou0-aQqe5_9~`Hj zeJ@Cu`1U5APQ=Y2Rf6xby2%RodBhi*0tSMX98vROHhm!g-1|*c+h*VRV%$qvWzb5P z_c7!@%4E(3Gh>dvQ^(!FOGgzVZd0K_|Qx=aG@^Ld@Q zRJrV+ozISUO8!IwSUa7kv->@VFvYd9H$z#Hn*h8?wUq=rROBTi2KS6a_0M=jpZ}=*Hn}=ai)$futZ~d&SJTv4AkQb<*IK4S9*3g9R|X0=S-*4l`0p-5 zp-)x!! zbSsH~e&P_+jKL7jfrZ%zNJ3iynU#yuotUd4|)TPD4sgP@es_RatH^3@B%Ks1z>_|&zV5A zID8A*kjMTdhe5x^dM?b-;_+a%2feOZk|?15Cg2{cTLA#oh+rZL7FSNP%=_;blDYOI zP(NvK;iKxnJzVLQ<|@sq6aauK;;u$~jAD#O%K#^oEpv0CEW5&nshD@jN*9lv?s+SF z;#+U&>ON-saklhIQxgO= zY2jFaQvUMXa-UuE81SF`Y?EerSrtB+f1kQFHONjuBXQoa*DstZLN>=MUA;g2#w$1^ z^jgX$(_jD)Prd*_2z6H}^F!szaLy3*b>BGPOn&{jgQCNyBw`a#+f9W~Hu%d!js=0h nqzrtw@ON_l{~|E`<&DRqc~QammVD diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/ser_error.xml b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/ser_error.xml deleted file mode 100644 index e0023da8ce..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/diagrams/ser_error.xml +++ /dev/null @@ -1 +0,0 @@ -7Vhdb9owFP01PHYKcWjhsQXabajVOia1faqs+Dbx5tjMMRD262cT59NAWwkVtIEQso/ta/vcc+8l6aBhkt1IPItvBQHW8T2SddCo4/uDwUD/GmCVA4FngUhSkkPdCpjSP2BBz6JzSiBtTFRCMEVnTTAUnEOoGhiWUiyb014Ea+46wxE4wDTEzEUfKFFxjvZ7XoV/BhrFxc5dz44kuJhsgTTGRCxrEBp30FAKofJWkg2BGe4KXvJ111tGy4NJ4OotC8Z3PzI+uZ/G/vD3RD3cJk/Z1zNrZYHZ3F44VXr+pf5CFsJMUcHt8dWq4ESKOSdgzHoddLWMqYLpDIdmdKlFoLFYJUz3urrpHrPYE6SCrAbZY9+ASEDJlZ5iR88tg1ZC3YLRZeWQkva45ozAYthqICotVzTphmXqHaxdbGdNzrmiCTyD1p48OHO+d2zU9R3qHJKAk0sTuboXMpymNGzyAhlVj5ZC034y7U892xtltaHRquhwffjHwoDp1FaZbrVs3SvWbfVBKuYyhNc1orCMQL0egUAaecj1aM1jvQ0OKzAJDCu6aGavTV60O3wTVN+sFEzQ1IuPWjrIr20X1ZNNy86gpbt+y05Oi2NnLany0m9SWbS4jf3J3SP+DtHZw/0EFhPYldbSVaogOZL4RMEB43Mjc/4/GJ+7JPJqfF6c4nP/8Ync+AS5j5h8oYwNBdNmzFoE64/GUyXFL6iNoHM0QGQ/UdxrVdm+G8SDj4zhwGH3Zyr48/H8oWunveDQWa/3v2Q95Ga9XRI6Zb19Zr3zk8o21uAjURlqPWeWz53vlVnQbRnqfqzO3MfTLzwGSRXm2m2+RyiOJE4c8ensrZpya5ZNLji0aqyFMKMRN5rVUgGNX5laQEPMLu1AQgkx22wsLs3ys4f60n5jUDqgXmDQBiX57y8wulu9w8k9WL0IQ+O/ \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/encode_json.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/encode_json.md deleted file mode 100644 index 358c9ef696..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/encode_json.md +++ /dev/null @@ -1,200 +0,0 @@ -### jsoncons::encode_json - -Encode a C++ data structure to a JSON formatted string or stream. `encode_json` will work for all types that -have [json_type_traits](https://github.com/danielaparker/jsoncons/blob/master/doc/ref/json_type_traits.md) defined. - -#### Header -```c++ -#include - -template -void encode_json(const T& val, - std::basic_ostream& os, - const basic_json_encode_options& options = basic_json_options::default_options(), - indenting line_indent = indenting::no_indent); // (1) - -template -void encode_json(const T& val, - std::basic_ostream& os, - indenting line_indent); // (2) - -template -void encode_json(const T& val, - std::basic_string& s, - const basic_json_encode_options& options = basic_json_options::default_options(), - indenting line_indent = indenting::no_indent); // (3) - -template -void encode_json(const T& val, - std::basic_string& s, - indenting line_indent); // (4) - -template -void encode_json(const T& val, - basic_json_content_handler& receiver); // (5) - -template -void encode_json(const basic_json& j, - const T& val, - std::basic_ostream& os, - const basic_json_encode_options& options = basic_json_options::default_options(), - indenting line_indent = indenting::no_indent); // (6) - -template -void encode_json(const basic_json& j, - const T& val, - std::basic_ostream& os, - indenting line_indent); // (7) - -template -void encode_json(const basic_json& j, - const T& val, - std::basic_string& s, - const basic_json_encode_options& options = basic_json_options::default_options(), - indenting line_indent = indenting::no_indent); // (8) - -template -void encode_json(const basic_json& j, - const T& val, - std::basic_string& s, - indenting line_indent); // (9) - -template -void encode_json(const basic_json& j, - const T& val, - basic_json_content_handler& receiver); // (10) -``` - -(1) Encode `val` to output stream with the specified options and line indenting. - -(2) Encode `val` to output stream with the specified line indenting. - -(3) Encode `val` to string with the specified options and line indenting. - -(4) Encode `val` to string with the specified line indenting. - -(5) Convert `val` to json events and stream through content handler. - -Functions (1)-(5) perform encodings using the default json type `basic_json`. -Functions (6)-(10) are the same but perform encodings using the supplied `basic_json`. - -#### Parameters - - - - - - - - - - - - - - - - - - - - - - -
valC++ data structure
handlerJSON output handler
optionsSerialization options
osOutput stream
indentingindenting::indent to pretty print, indenting::no_indent for compact output
- -#### Return value - -None - -#### See also - -- [json_content_handler](json_content_handler.md) -- [json_options](json_options.md) - -### Examples - -#### Map with string-tuple pairs - -```c++ -#include -#include -#include -#include - -using namespace jsoncons; - -int main() -{ - typedef std::map> employee_collection; - - employee_collection employees = - { - {"John Smith",{"Hourly","Software Engineer",10000}}, - {"Jane Doe",{"Commission","Sales",20000}} - }; - - std::cout << "(1)\n" << std::endl; - encode_json(employees,std::cout); - std::cout << "\n\n"; - - std::cout << "(2) Again, with pretty print\n" << std::endl; - encode_json(employees, std::cout, jsoncons::indenting::indent); -} -``` -Output: -``` -(1) - -{"Jane Doe":["Commission","Sales",20000.0],"John Smith":["Hourly","Software Engineer",10000.0]} - -(2) Again, with pretty print - -{ - "Jane Doe": ["Commission","Sales",20000.0], - "John Smith": ["Hourly","Software Engineer",10000.0] -} -``` - -#### Contain JSON output in an object - -```c++ -#include -#include -#include -#include - -using namespace jsoncons; - -int main() -{ - std::map> employees = - { - {"John Smith",{"Hourly","Software Engineer",10000}}, - {"Jane Doe",{"Commission","Sales",20000}} - }; - - json_encoder encoder(std::cout, jsoncons::indenting::indent); - - encoder.begin_object(); - encoder.write_name("Employees"); - encode_json(employees, encoder); - encoder.end_object(); - encoder.flush(); -} -``` -Output: -```json -{ - "Employees": { - "Jane Doe": ["Commission","Sales",20000.0], - "John Smith": ["Hourly","Software Engineer",10000.0] - } -} -``` - -#### See also - -- [decode_json](decode_json.md) - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/indenting.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/indenting.md deleted file mode 100644 index d52d953a1e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/indenting.md +++ /dev/null @@ -1,13 +0,0 @@ -### jsoncons::indenting - -```c++ -enum class indenting {no_indent, indent} -``` - -#### Header -```c++ -#include -``` - -Specifies indentation options for the [json_encoder](json_encoder.md) - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json.md deleted file mode 100644 index 08ccf62c1b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json.md +++ /dev/null @@ -1,397 +0,0 @@ -### jsoncons::json - -```c++ -typedef basic_json> json -``` -The class `json` is an instantiation of the `basic_json` class template that uses `char` as the character type. The order of an object's name/value pairs is not preserved, they are sorted alphabetically by name. If you want to preserve the original insertion order, use [ojson](ojson.md) instead. - -The class `json` resembles a union. An instance of `json` holds a data item of one of its alternative types: - -- null -- bool -- int64 -- uint64 -- double -- string -- byte string -- array -- object - -The data item may be tagged with a [semantic_tag](semantic_tag.md) that provides additional -information about the data item. - -When assigned a new value, the old value is overwritten. The type of the new value may be different from the old value. - -The `jsoncons` library will rebind the supplied allocator from the template parameter to internal data structures. - -#### Header -```c++ -#include -``` - -#### Member types - -Member type |Definition -------------------------------------|------------------------------ -`json_type`|json -`reference`|json& -`const_reference`|const json& -`pointer`|json* -`const_pointer`|const json* -`allocator_type`|Allocator type -`char_allocator`|String allocator type -`array_allocator`|Array allocator type -`object_allocator`|Object allocator -`string_view_type`|A non-owning view of a string, holds a pointer to character data and length. Supports conversion to and from strings. Will be typedefed to the C++ 17 [string view](http://en.cppreference.com/w/cpp/string/basic_string_view) if `JSONCONS_HAS_STRING_VIEW` is defined in `jsoncons_config.hpp`, otherwise proxied. -`key_value_type`|[key_value_type](json/key_value.md) is a class that stores a name and a json value -`object`|json object type -`array`|json array type -`object_iterator`|A [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator) to [key_value_type](json/key_value.md) -`const_object_iterator`|A const [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator) to const [key_value_type](json/key_value.md) -`array_iterator`|A [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator) to `json` -`const_array_iterator`|A const [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator) to `const json` - -### Static member functions - - - - - - - - - - - - - - -
parseParses JSON.
make_arrayMakes a multidimensional json array.
const json& null()Returns a null value
- -### Member functions - - - - - - - - - - - - - -
(constructor)constructs the json value
(destructor)destructs the json value
operator=assigns values
- - allocator_type get_allocator() const -Returns the allocator associated with the json value. - -#### Ranges and Iterators - - - - - - - - - - -
array_rangeReturns a "range" that supports a range-based for loop over the elements of a `json` array.
obect_rangeReturns a "range" that supports a range-based for loop over the key-value pairs of a `json` object.
- -#### Capacity - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
size_t size() const noexceptReturns the number of elements in a json array, or the number of members in a json object, or `zero`
bool empty() const noexceptReturns true if a json string, object or array has no elements, otherwise false
size_t capacity() constReturns the size of the storage space currently allocated for a json object or array
void reserve(size_t n)Increases the capacity of a json object or array to allow at least `n` members or elements
void resize(size_t n)Resizes a json array so that it contains `n` elements
void resize(size_t n, const json& val)Resizes a json array so that it contains `n` elements that are initialized to `val`
void shrink_to_fit()Requests the removal of unused capacity
- -#### Accessors - - - - - - - - - - - - - - - - - - -
bool contains(const string_view_type& key) constReturns true if an object has a member with the given `key`, otherwise false
bool count(const string_view_type& key) constReturns the number of object members that match `key`
isChecks if a json value matches a type.
asAttempts to convert a json value to a value of a type.
- - semantic_tag get_semantic_tag() const -Returns the [semantic_tag](semantic_tag.md) associated with this value - - json& operator[](size_t i) - const json& operator[](size_t i) const -Returns a reference to the value at position i in a json object or array. -Throws `std::runtime_error` if not an object or array. - - json& operator[](const string_view_type& name) -Returns a proxy to a keyed value. If written to, inserts or updates with the new value. If read, evaluates to a reference to the keyed value, if it exists, otherwise throws. -Throws `std::runtime_error` if not an object. -If read, throws `std::out_of_range` if the object does not have a member with the specified name. - - const json& operator[](const string_view_type& name) const -If `name` matches the name of a member in the json object, returns a reference to the json object, otherwise throws. -Throws `std::runtime_error` if not an object. -Throws `std::out_of_range` if the object does not have a member with the specified name. - - object_iterator find(const string_view_type& name) - const_object_iterator find(const string_view_type& name) const -Returns an object iterator to a member whose name compares equal to `name`. If there is no such member, returns `end_member()`. -Throws `std::runtime_error` if not an object. - - json& at(const string_view_type& name) - const json& at(const string_view_type& name) const -Returns a reference to the value with the specifed name in a json object. -Throws `std::runtime_error` if not an object. -Throws `std::out_of_range` if the object does not have a member with the specified name. - - json& at(size_t i) - const json& at(size_t i) const -Returns a reference to the element at index `i` in a json array. -Throws `std::runtime_error` if not an array. -Throws `std::out_of_range` if the index is outside the bounds of the array. - - template - T get_with_default(const string_view_type& name, - const T& default_val) const -If `name` matches the name of a member in the json object, returns the member value converted to the default's data type, otherwise returns `default_val`. -Throws `std::runtime_error` if not an object. - - template - T get_with_default(const string_view_type& name, - const char_type* default_val) const -Make `get_with_default` do the right thing for string literals. -Throws `std::runtime_error` if not an object. - -#### Modifiers - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
void clear()Remove all elements from an array or members from an object, otherwise do nothing
eraseErases array elements and object members
push_backAdds a value to the end of a json array
insertInserts elements
emplace_backConstructs a value in place at the end of a json array
emplaceConstructs a value in place before a specified position in a json array
try_emplaceConstructs a key-value pair in place in a json object if the key does not exist, does nothing if the key exists
insert_or_assignInserts a key-value pair in a json object if the key does not exist, or assigns a new value if the key already exists
mergeInserts another json object's key-value pairs into a json object, if they don't already exist.
merge_or_updateInserts another json object's key-value pairs into a json object, or assigns them if they already exist.
void swap(json& val)Exchanges the content of the json value with the content of val, which is another json value
- -#### Serialization - - - - - - -
dumpSerializes json value to a string, stream, or output handler.
- -#### Non member functions - - - - - - - - - - - - - - - - - - - - - - - - - - -
bool operator==(const json& lhs, const json& rhs)Returns true if two json objects compare equal, false otherwise.
bool operator!=(const json& lhs, const json& rhs)Returns true if two json objects do not compare equal, false otherwise.
bool operator<(const json& lhs, const json& rhs)Compares the contents of lhs and rhs lexicographically.
bool operator<=(const json& lhs, const json& rhs)Compares the contents of lhs and rhs lexicographically.
bool operator>(const json& lhs, const json& rhs)Compares the contents of lhs and rhs lexicographically.
bool operator>=(const json& lhs, const json& rhs)Compares the contents of lhs and rhs lexicographically.
- - std::istream& operator>> (std::istream& os, json& val) -Reads a `json` value from a stream. - - std::ostream& operator<< (std::ostream& os, const json& val) -Inserts json value into stream. - - std::ostream& print(const json& val) - std::ostream& print(const json& val, const json_options& options) -Inserts json value into stream using the specified [json_options](json_options.md) if supplied. - - std::ostream& pretty_print(const json& val) - std::ostream& pretty_print(const json& val, const json_options& options) -Inserts json value into stream using the specified [json_options](json_options.md) if supplied. - - void swap(json& a, json& b) -Exchanges the values of `a` and `b` - -#### Deprecated names - -As the `jsoncons` library has evolved, names have sometimes changed. To ease transition, jsoncons deprecates the old names but continues to support many of them. See the [deprecated list](deprecated.md) for the status of old names. The deprecated names can be suppressed by defining macro JSONCONS_NO_DEPRECATED, which is recommended for new code. - -#### See also - -- [ojson](ojson.md) constructs a json value that preserves the original name-value insertion order - -- [wjson](wjson.md) constructs a wide character json value that sorts name-value members alphabetically - -- [wojson](wojson.md) constructs a wide character json value that preserves the original name-value insertion order - -### Examples - -#### Accessors and defaults -```c++ -json val; - -val["field1"] = 1; -val["field3"] = "Toronto"; - -double x1 = obj.contains("field1") ? val["field1"].as() : 10.0; -double x2 = obj.contains("field2") ? val["field2"].as() : 20.0; - -std::string x3 = obj.get_with_default("field3","Montreal"); -std::string x4 = obj.get_with_default("field4","San Francisco"); - -std::cout << "x1=" << x1 << '\n'; -std::cout << "x2=" << x2 << '\n'; -std::cout << "x3=" << x3 << '\n'; -std::cout << "x4=" << x4 << '\n'; -``` -Output: -```c++ -x1=1 -x2=20 -x3=Toronto -x4=San Francisco -``` -#### Nulls -```c++ -json obj; -obj["field1"] = json::null(); -std::cout << obj << std::endl; -``` -Output: -```json -{"field1":null} -``` -#### Constructing json structures -```c++ -json root; - -root["persons"] = json::array(); - -json person; -person["first_name"] = "John"; -person["last_name"] = "Smith"; -person["birth_date"] = "1972-01-30"; - -json address; -address["city"] = "Toronto"; -address["country"] = "Canada"; - -person["address"] = std::move(address); - -root["persons"].push_back(std::move(person)); - -std::cout << pretty_print(root) << std::endl; -``` -Output: -```c++ -{ - "persons": - [ - { - "address": - { - "city":"Toronto", - "country":"Canada" - }, - "birth_date":"1972-01-30", - "first_name":"John", - "last_name":"Smith" - } - ] -} -``` diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/array_range.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/array_range.md deleted file mode 100644 index d7f2b9cfb0..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/array_range.md +++ /dev/null @@ -1,45 +0,0 @@ -### jsoncons::json::array_range - -```c++ -range array_range(); -range array_range() const; -``` -Returns a "range" that supports a range-based for loop over the elements of a `json` array -Throws `std::runtime_error` if not an array. - -### Examples - -#### Range-based for loop - -```c++ -json j = json::array{"Toronto", "Vancouver", "Montreal"}; - -for (const auto& val : j.array_range()) -{ - std::cout << val.as() << std::endl; -} -``` -Output: -```json -Toronto -Vancouver -Montreal -``` - -#### Array iterator -```c++ -json j = json::array{"Toronto", "Vancouver", "Montreal"}; - -for (auto it = j.array_range().begin(); it != j.array_range().end(); ++it) -{ - std::cout << it->as() << std::endl; -} -``` -Output: -```json -Toronto -Vancouver -Montreal -``` - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/as.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/as.md deleted file mode 100644 index 4c62964feb..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/as.md +++ /dev/null @@ -1,120 +0,0 @@ -### jsoncons::json::as - -```c++ -template -T as(Args&&... args) const; // (1) - -bool as_bool() const; // (2) - -template -T as_integer() const; // (3) - -double as_double() const; // (4) - -string_view_type as_string_view() const; // (5) - -std::string as_string() const; // (6) - -byte_string as_byte_string() const; // (7) - -bignum as_bignum() const; // (8) -``` - -(1) Generic get `as` type `T`. Attempts to convert the json value to the template value type using [json_type_traits](../json_type_traits.md). - - std::string as() const noexcept - std::string as(const char_allocator& allocator) const noexcept -If value is string, returns value, otherwise returns result of [dump](dump.md). - - as>() -If the type `X` is not `std::basic_string` but otherwise satisfies [SequenceContainer](http://en.cppreference.com/w/cpp/concept/SequenceContainer), `as>()` returns the `json` value as an `X` if the `json` value is an array and each element is convertible to type `T`, otherwise throws. - - as>() -If the type 'X' satisfies [AssociativeContainer](http://en.cppreference.com/w/cpp/concept/AssociativeContainer) or [UnorderedAssociativeContainer](http://en.cppreference.com/w/cpp/concept/UnorderedAssociativeContainer), `as>()` returns the `json` value as an `X` if the `json` value is an object and if each member value is convertible to type `T`, otherwise throws. - -(2) Same as `as()`. -Returns `true` if value is `bool` and `true`, or if value is integral and non-zero, or if value is floating point and non-zero, or if value is string and parsed value evaluates as `true`. -Returns `false` if value is `bool` and `false`, or if value is integral and zero, or if value is floating point and zero, or if value is string and parsed value evaluates as `false`. -Otherwise throws `std::runtime_exception` - -(3) Same as `as()` for integral type T. -Returns integer value if value is integral, performs cast if value has double type, returns 1 or 0 if value has bool type, attempts conversion if value is string, otherwise throws. - -(4) Same as `as()`. -Returns value cast to double if value is integral, returns `NaN` if value is `null`, attempts conversion if value is string, otherwise throws. - -### Examples - -#### Accessing integral, floating point, and boolean values - -```c++ -json j = json::parse(R"( -{ - "k1" : 2147483647, - "k2" : 2147483648, - "k3" : -10, - "k4" : 10.5, - "k5" : true, - "k6" : "10.5" -} -)"); - -std::cout << "(1) " << j["k1"].as() << '\n'; -std::cout << "(2) " << j["k2"].as() << '\n'; -std::cout << "(3) " << j["k2"].as() << '\n'; -std::cout << "(4) " << j["k3"].as() << '\n'; -std::cout << "(5) " << j["k3"].as() << '\n'; -std::cout << "(6) " << j["k4"].as() << '\n'; -std::cout << "(7) " << j["k4"].as() << '\n'; -std::cout << std::boolalpha << "(8) " << j["k5"].as() << '\n'; -std::cout << std::boolalpha << "(9) " << j["k5"].as() << '\n'; -std::cout << "(10) " << j["k6"].as() << '\n'; - -``` -Output: -``` - -(1) 2147483647 -(2) -2147483648 -(3) 2147483648 -(4) -(5) 4294967286 -(6) 10 -(7) 10.5 -(8) 1 -(9) true -(10) 10.5 -``` - -#### Accessing a `json` array value as a `std::vector` -```c++ -std::string s = "{\"my-array\" : [1,2,3,4]}"; -json val = json::parse(s); -std::vector v = val["my-array"].as>(); -for (size_t i = 0; i < v.size(); ++i) -{ - if (i > 0) - { - std::cout << ","; - } - std::cout << v[i]; -} -std::cout << std::endl; -``` -Output: -``` -1,2,3,4 -``` - -#### Accessing a `json` byte string as a [byte_string](../byte_string.md) -```c++ -json j(byte_string("Hello")); -byte_string bs = j.as(); - -std::cout << bs << std::endl; -``` -Output: -``` -0x480x650x6c0x6c0x6f -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/constructor.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/constructor.md deleted file mode 100644 index ac3abbed65..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/constructor.md +++ /dev/null @@ -1,150 +0,0 @@ -### `jsoncons::json::json` - -```c++ -json(); // (1) - -json(const allocator_type& allocator); // (2) - -json(std::initializer_list list, const allocator_type& allocator); // (3) - -json(const json& val); // (4) - -json(const json& val, const allocator_type& allocator); // (5) - -json(json&& val) noexcept; // (6) - -json(json&& val, const allocator_type& allocator) noexcept; // (7) - -json(const array& val); // (8) - -json(array&& val) noexcept; // (9) - -json(const object& val); // (10) - -json(object&& val) noexcept; // (11) - -template -json(const T& val); // (12) - -template -json(const T& val, const allocator_type& allocator); // (13) - -json(const char* val); // (14) - -json(const char* val, const allocator_type& allocator); // (15) - -json(const byte_string_view& bs, - semantic_tag tag = semantic_tag::none); // (16) - -json(const byte_string_view& bs, - semantic_tag tag = semantic_tag::none, - const allocator_type& allocator); // (17) - -json(const bignum& n); // (18) - -json(const bignum& n, const allocator_type& allocator); // (19) -``` - -(1) Constructs a `json` value that holds an empty json object. - -(2) Constructs a `json` value that holds a json object. - -(3) Constructs a `json` array with the elements of the initializer-list `init`. - -(4) Constructs a copy of val - -(5) Copy with allocator - -(6) Acquires the contents of val, leaving val a `null` value - -(7) Move with allocator - -(8) Constructs a `json` value from a json array - -(9) Acquires the contents of a json array - -(10) Constructs a `json` value from a json object - -(11) Acquires the contents of a json object - -(12) Constructs a `json` value for types supported in [json_type_traits](json_type_traits.md). - -(13) Constructs a `json` value for types supported in [json_type_traits](json_type_traits.md) with allocator. - -(14) Constructs a `json` value for a text string. - -(15) Constructs a `json` value for a text string with supplied allocator. - -(16) Constructs a `json` value for a [byte_string](../byte_string.md). - -(17) Constructs a `json` value for a [byte_string](../byte_string.md) with supplied allocator. - -(18) Constructs a `json` value for a [bignum](../bignum.md). - -(19) Constructs a `json` value for a [bignum](../bignum.md) with supplied allocator. - -### Examples - -```c++ -#include -#include -#include -#include -#include - -using namespace jsoncons; -int main() -{ - json j1; // An empty object - std::cout << "(1) " << j1 << std::endl; - - json j2 = json::object({{"baz", "qux"}, {"foo", "bar"}}); // An object - std::cout << "(2) " << j2 << std::endl; - - json j3 = json::array({"bar", "baz"}); // An array - std::cout << "(3) " << j3 << std::endl; - - json j4(json::null()); // A null value - std::cout << "(4) " << j4 << std::endl; - - json j5(true); // A boolean value - std::cout << "(5) " << j5 << std::endl; - - double x = 1.0/7.0; - - json j6(x); // A double value - std::cout << "(6) " << j6 << std::endl; - - json j7(x,4); // A double value with specified precision - std::cout << "(7) " << j7 << std::endl; - - json j8("Hello"); // A text string - std::cout << "(8) " << j8 << std::endl; - - const uint8_t bs[] = {'H','e','l','l','o'}; - json j9(byte_string("Hello")); // A byte string - std::cout << "(9) " << j9 << std::endl; - - std::vector v = {10,20,30}; - json j10 = v; // From a sequence container - std::cout << "(10) " << j10 << std::endl; - - std::map m{ {"one", 1}, {"two", 2}, {"three", 3} }; - json j11 = m; // From an associative container - std::cout << "(11) " << j11 << std::endl; -} -``` - -``` -(1) {} -(2) {"baz":"qux","foo":"bar"} -(3) ["bar","baz"] -(4) null -(5) true -(6) 0.142857142857143 -(7) 0.1429 -(8) "Hello" -(9) "SGVsbG8_" -(10) [10,20,30] -(11) {"one":1,"three":3,"two":2} -``` diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/destructor.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/destructor.md deleted file mode 100644 index 1b6d7489d1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/destructor.md +++ /dev/null @@ -1,9 +0,0 @@ -### `jsoncons::json::json` - -```c++ -~json(); -``` - -Destroys all values and deletes all memory allocated for strings, arrays, and objects. - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/dump.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/dump.md deleted file mode 100644 index 71fb4553d8..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/dump.md +++ /dev/null @@ -1,177 +0,0 @@ -### `jsoncons::json::dump` - -```c++ -template -void dump(std::basic_string& s) const; // (1) - -template -void dump(std::basic_string& s, - indenting line_indent) const; // (2) - -template -void dump(std::basic_string& s, - const json_options& options) const; // (3) - -template -void dump(std::basic_string& s, - const basic_json_options& options, - indenting line_indent) const; // (4) - -void dump(std::ostream& os) const; // (5) - -void dump(std::ostream os, indenting line_indent) const; // (6) - -void dump(std::ostream os, const json_options& options) const; // (7) - -void dump(std::ostream os, const json_options& options, indenting line_indent) const; // (8) - -void dump(basic_json_content_handler& content_handler) const; // (9) -``` - -(1) Dumps json value to string using default json_options. - -(2) Dumps json value to string using default serialization options and the specified [indenting](../indenting.md). - -(3) Dumps json value to string using specified [json_options](../json_options.md). - -(4) Dumps json value to string using the specified [json_options](../json_options.md) and [indenting](../indenting.md). - -(5) Dumps json value to stream with default serialization options. - -(6) Dumps json value to stream using default serialization options and the specified [indenting](../indenting.md). - -(7) Dumps json value to stream using specified [json_options](../json_options.md). - -(8) Dumps json value to stream using the specified [json_options](../json_options.md) and [indenting](../indenting.md). - -(9) Emits json value to the [json_content_handler](../json_content_handler.md). - -### Examples - -#### Dump json value to csv file - -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - const json books = json::parse(R"( - [ - { - "title" : "Kafka on the Shore", - "author" : "Haruki Murakami", - "price" : 25.17 - }, - { - "title" : "Women: A Novel", - "author" : "Charles Bukowski", - "price" : 12.00 - }, - { - "title" : "Cutter's Way", - "author" : "Ivan Passer" - } - ] - )"); - - csv_options options; - options.column_names("author,title,price"); - - csv_encoder encoder(std::cout, options); - - books.dump(encoder); -} -``` - -Output: - -```csv -author,title,price -Haruki Murakami,Kafka on the Shore,25.17 -Charles Bukowski,Women: A Novel,12.0 -Ivan Passer,Cutter's Way, -``` - -#### Dump json content into a larger document - -```c++ -#include - -using namespace jsoncons; - -int main() -{ - const json some_books = json::parse(R"( - [ - { - "title" : "Kafka on the Shore", - "author" : "Haruki Murakami", - "price" : 25.17 - }, - { - "title" : "Women: A Novel", - "author" : "Charles Bukowski", - "price" : 12.00 - } - ] - )"); - - const json more_books = json::parse(R"( - [ - { - "title" : "A Wild Sheep Chase: A Novel", - "author" : "Haruki Murakami", - "price" : 9.01 - }, - { - "title" : "Cutter's Way", - "author" : "Ivan Passer", - "price" : 8.00 - } - ] - )"); - - json_encoder encoder(std::cout, jsoncons::indenting::indent); // pretty print - encoder.begin_array(); - for (const auto& book : some_books.array_range()) - { - book.dump(encoder); - } - for (const auto& book : more_books.array_range()) - { - book.dump(encoder); - } - encoder.end_array(); - encoder.flush(); -} -``` - -Output: - -```json -[ - { - "author": "Haruki Murakami", - "price": 25.17, - "title": "Kafka on the Shore" - }, - { - "author": "Charles Bukowski", - "price": 12.0, - "title": "Women: A Novel" - }, - { - "author": "Haruki Murakami", - "price": 9.01, - "title": "A Wild Sheep Chase: A Novel" - }, - { - "author": "Ivan Passer", - "price": 8.0, - "title": "Cutter's Way" - } -] -``` diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/emplace.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/emplace.md deleted file mode 100644 index b94fe964fa..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/emplace.md +++ /dev/null @@ -1,44 +0,0 @@ -### jsoncons::json::emplace - -```c++ -template -array_iterator emplace(Args&&... args); - -template -array_iterator emplace(const_array_iterator pos, Args&&... args); -``` - -Constructs a new json element at the specified position of a json array, shifting all elements currently at or above that position to the right. - -#### Parameters - - pos -Iterator that identifies the position in the array to construct the new json value - - args -Arguments to forward to the constructor of the json value - -#### Return value - -Array iterator pointing to the emplaced value. - -#### Exceptions - -Throws `std::runtime_error` if not a json array. - -### Example - -```c++ -json a = json::array(); -a.emplace_back("Toronto"); -a.emplace_back("Vancouver"); -a.emplace(a.array_range().begin(),"Montreal"); - -std::cout << a << std::endl; -``` -Output: - -```json -["Montreal","Toronto","Vancouver"] -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/emplace_back.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/emplace_back.md deleted file mode 100644 index f01713f87d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/emplace_back.md +++ /dev/null @@ -1,35 +0,0 @@ -### jsoncons::json::emplace_back -```c++ -template -json& emplace_back(Args&&... args); -``` - -#### Parameters - - args -Arguments to forward to the constructor of the json value - -#### Return value - -A reference to the emplaced json value. - -#### Exceptions - -Throws `std::runtime_error` if not a json array. - -### Example - -```c++ -json arr = json::array(); -arr.emplace_back(10); -arr.emplace_back(20); -arr.emplace_back(30); - -std::cout << arr << std::endl; -``` -Output: - -```json -[10,20,30] -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/erase.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/erase.md deleted file mode 100644 index 8f37d6f31b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/erase.md +++ /dev/null @@ -1,29 +0,0 @@ -### `jsoncons::json::erase` - -```c++ -void erase(const_array_iterator pos); // (1) - -void erase(const_array_iterator first, const_array_iterator last); // (2) - -void erase(const_object_iterator pos); // (3) - -void erase(const_object_iterator first, const_object_iterator last); // (4) - -void erase(const string_view_type& name); // (5) -``` - -(1) Remove an element from an array at the specified position. -Throws `std::runtime_error` if not an array. - -(2) Remove the elements from an array in the range '[first,last)'. -Throws `std::runtime_error` if not an array. - -(3) Remove a member from an object at the specified position. -Throws `std::runtime_error` if not an object. - -(4) Remove the members from an object in the range '[first,last)'. -Throws `std::runtime_error` if not an object. - -(5) Remove a member with the specified name from an object -Throws `std::runtime_error` if not an object. - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/insert.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/insert.md deleted file mode 100644 index 698c2138c9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/insert.md +++ /dev/null @@ -1,125 +0,0 @@ -### jsoncons::json::insert - -```c++ -template -array_iterator insert(const_array_iterator pos, T&& val); // (1) - -template -array_iterator insert(const_array_iterator pos, InputIt first, InputIt last); // (2) -``` -(1) Adds a new json element at the specified position of a json array, shifting all elements currently at or above that position to the right. -The argument `val` is forwarded to the `json` constructor as `std::forward(val)`. -Returns an `array_iterator` that points to the new value -Throws `std::runtime_error` if not an array. - -(2) Inserts elements from range [first, last) before pos. - -``` -template -void insert(InputIt first, InputIt last); // (3) -``` - -(3) Inserts elements from range `[first, last)` into a json object. - If multiple elements in the range have the same key, the first element in the range is inserted. - The function template parameter `InputIt` represents an input - iterator type that iterates over elements of type `key_value`, - or alternatively over elements of type `std::pair` where `T1` is convertible to `string_type` and `T2` is convertible to `json_type`. - -#### See also - -- [push_back](json/push_back.md) - -### Examples - -#### Creating an array of elements -```c++ -json cities = json::array(); // an empty array -std::cout << cities << std::endl; // output is "[]" - -cities.push_back("Toronto"); -cities.push_back("Vancouver"); -// Insert "Montreal" at beginning of array -cities.insert(cities.array_range().begin(),"Montreal"); - -std::cout << cities << std::endl; -``` -Output: -``` -[] -["Montreal","Toronto","Vancouver"] -``` -#### Creating an array of elements with reserved storage -```c++ -json cities = json::array(); -cities.reserve(10); // storage is reserved -std::cout << "capacity=" << cities.capacity() - << ", size=" << cities.size() << std::endl; - -cities.push_back("Toronto"); -cities.push_back("Vancouver"); -cities.insert(cities.array_range().begin(),"Montreal"); -std::cout << "capacity=" << cities.capacity() - << ", size=" << cities.size() << std::endl; - -std::cout << cities << std::endl; -``` -Output: -``` -capacity=10, size=0 -capacity=10, size=3 -["Montreal","Toronto","Vancouver"] -``` - -### Copy two std::map's into a json - -```c++ -std::map m1 = {{"f",4},{"e",5},{"d",6}}; -std::map m2 = {{"c",1},{"b",2},{"a",3}}; - -json j; -j.insert(m1.begin(),m1.end()); -j.insert(m2.begin(),m2.end()); - -std::cout << j << "\n"; -``` -Output: -``` -{"a":3.0,"b":2.0,"c":1.0,"d":6.0,"e":5.0,"f":4.0} -``` - -### Copy two std::map's into an ojson - -```c++ -std::map m1 = {{"f",4},{"e",5},{"d",6}}; -std::map m2 = {{"c",1},{"b",2},{"a",3}}; - -ojson j; -j.insert(m1.begin(),m1.end()); -j.insert(m2.begin(),m2.end()); - -std::cout << j << "\n"; -``` -Output: -``` -{"d":6.0,"e":5.0,"f":4.0,"a":3.0,"b":2.0,"c":1.0} -``` - -### Move two std::map's into a json - -```c++ -std::map m1 = {{"a",1},{"b",2},{"c",3}}; -std::map m2 = {{"d",4},{"e",5},{"f",6}}; - -json j; -j.insert(std::make_move_iterator(m1.begin()),std::make_move_iterator(m1.end())); -j.insert(std::make_move_iterator(m2.begin()),std::make_move_iterator(m2.end())); - -std::cout << j << "\n"; -``` -Output: -``` -{"a":1.0,"b":2.0,"c":3.0,"d":4.0,"e":5.0,"f":6.0} -``` - - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/insert_or_assign.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/insert_or_assign.md deleted file mode 100644 index 07255a207f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/insert_or_assign.md +++ /dev/null @@ -1,36 +0,0 @@ -### jsoncons::json::insert_or_assign - -```c++ -template -pair insert_or_assign(const string_view_type& key, T&& value); // (1) - -template -object_iterator insert_or_assign(const_object_iterator hint, const string_view_type& key, - T&& value); // (2) -``` - -#### Parameters - - key -The member name used to look up and, if not found, to insert - - hint -An object iterator that provides a hint where to insert the new json value - - value -Value to insert or assign - -#### Return value - -(1) returns a pair consisting of first, an iterator to the inserted value -or the already existing value, -and second, a bool indicating whether the insertion took place -(true for insertion, false for no insertion.) - -(2) returns an iterator to the inserted value -or the already existing value. - -#### Exceptions - -Throws `std::runtime_error` if not a json object. - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/is.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/is.md deleted file mode 100644 index de8d45ad09..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/is.md +++ /dev/null @@ -1,107 +0,0 @@ -### jsoncons::json::is - -```c++ -template -bool is(Args&&... args) const noexcept; // (1) - -bool is_null() const noexcept; // (2) - -bool is_string() const noexcept; // (3) - -bool is_int64() const noexcept; // (4) - -bool is_uint64() const noexcept; // (5) - -bool is_double() const noexcept; // (6) - -bool is_number() const noexcept; // (7) - -bool is_bool() const noexcept; // (8) - -bool is_byte_string() const; // (9) - -bool is_bignum() const; // (10) - -bool is_array() const noexcept; // (11) - -bool is_object() const noexcept; // (12) -``` - -(1) Generic `is` equivalent to type `T`. Returns `true` if the json value is the same as type `T` according to [json_type_traits](../json_type_traits.md), `false` otherwise. - - bool is> const noexcept -If the type `X` is not `std::basic_string` but otherwise satisfies [SequenceContainer](http://en.cppreference.com/w/cpp/concept/SequenceContainer), `is>()` returns `true` if the json value is an array and each element is the "same as" type `T` according to [json_type_traits](json_type_traits.md), `false` otherwise. - bool is> const noexcept -If the type 'X' satisfies [AssociativeContainer](http://en.cppreference.com/w/cpp/concept/AssociativeContainer) or [UnorderedAssociativeContainer](http://en.cppreference.com/w/cpp/concept/UnorderedAssociativeContainer), `is>()` returns `true` if the json value is an object and each mapped value is the "same as" `T` according to [json_type_traits](json_type_traits.md), `false` otherwise. - -(2) Same as `is()`. -Returns `true` if the json value is null, `false` otherwise. - -(3) Same as `is()`. -Returns `true` if the json value is of string type, `false` otherwise. - -(4) Same as `is()`. -Returns `true` if the json value is integral and within the range of `int64_t`, `false` otherwise. - -(5) Same as `is()`. -Returns `true` if the json value is integral and within the range of `uint64_t`, `false` otherwise. - -(6) Same as `is()`. -Returns `true` if the json value is floating point and within the range of `double`, `false` otherwise. - -(7) Same as `is() || is() || is()`. - -(8) Same as `is()`. -Returns `true` if the json value is of boolean type, `false` otherwise. - -(10) Same as `is()`. -Returns `true` if `is() || is() is `true`, or if -`is()` is `true` and the string holds an integer value, -otherwise `false`. - -(11) Same as `is()`. -Returns `true` if the json value is an array, `false` otherwise. - -(12) Same as `is()`. -Returns `true` if the json value is an object, `false` otherwise. - -### Examples - -```c++ -json j = json::parse(R"( -{ - "k1" : 2147483647, - "k2" : 2147483648, - "k3" : -10, - "k4" : 10.5, - "k5" : true, - "k6" : "10.5" -} -)"); - -std::cout << std::boolalpha << "(1) " << j["k1"].is() << '\n'; -std::cout << std::boolalpha << "(2) " << j["k2"].is() << '\n'; -std::cout << std::boolalpha << "(3) " << j["k2"].is() << '\n'; -std::cout << std::boolalpha << "(4) " << j["k3"].is() << '\n'; -std::cout << std::boolalpha << "(5) " << j["k3"].is() << '\n'; -std::cout << std::boolalpha << "(6) " << j["k4"].is() << '\n'; -std::cout << std::boolalpha << "(7) " << j["k4"].is() << '\n'; -std::cout << std::boolalpha << "(8) " << j["k5"].is() << '\n'; -std::cout << std::boolalpha << "(9) " << j["k5"].is() << '\n'; -std::cout << std::boolalpha << "(10) " << j["k6"].is() << '\n'; - -``` -Output: -``` -(1) true -(2) false -(3) true -(4) true -(5) false -(6) false -(7) true -(8) false -(9) true -(10) false - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/key_value.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/key_value.md deleted file mode 100644 index 44526a44e1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/key_value.md +++ /dev/null @@ -1,53 +0,0 @@ -### jsoncons::json::key_value - -```c++ -template -class key_value -``` - -`key_value` stores a key (name) and a json value - -#### Member types - -Member type |Definition -------------------------------------|------------------------------ -`string_view_type`|A non-owning view of a string, holds a pointer to character data and length. Supports conversion to and from strings. Will be typedefed to the C++ 17 [string view](http://en.cppreference.com/w/cpp/string/basic_string_view) if `JSONCONS_HAS_STRING_VIEW` is defined in `jsoncons_config.hpp`, otherwise proxied. - -#### Accessors - - string_view_type key() const - - const json& value() const - - json& value() - -#### Non member functions - - - - - - - - - - - - - - - - - - - - - - - - - - -
bool operator==(const key_value& lhs, const key_value& rhs)Returns true if two key_value objects compare equal, false otherwise.
bool operator!=(const key_value& lhs, const key_value& rhs)Returns true if two key_value objects do not compare equal, false otherwise.
bool operator<(const key_value& lhs, const key_value& rhs)Compares the contents of lhs and rhs lexicographically.
bool operator<=(const key_value& lhs, const key_value& rhs)Compares the contents of lhs and rhs lexicographically.
bool operator>(const key_value& lhs, const key_value& rhs)Compares the contents of lhs and rhs lexicographically.
bool operator>=(const key_value& lhs, const key_value& rhs)Compares the contents of lhs and rhs lexicographically.
- - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/make_array.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/make_array.md deleted file mode 100644 index b5e12f70b6..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/make_array.md +++ /dev/null @@ -1,94 +0,0 @@ -### jsoncons::json::make_array - -```c++ -template -static json make_array(size_ n, const T& val) - -template -static json make_array(size_ n, const T& val, - const allocator_type& alloc = allocator_type()) - -template -static json make_array(size_t size1 ... size_t sizeN) - -template -static json make_array(size_t size1 ... size_t sizeN, const T& val) - -template -static json make_array(size_t size1 ... size_t sizeN, const T& val, - const allocator_type& alloc) -``` -Makes a multidimensional array with the number of dimensions specified as a template parameter. The size of each dimension is passed as a parameter, and optionally an inital value. If no initial value, the default is an empty json object. The elements may be accessed using familiar C++ native array syntax. - -### Examples - -#### Make an array of size 10 initialized with zeros -```c++ -json a = json::make_array<1>(10,0); -a[1] = 1; -a[2] = 2; -std::cout << pretty_print(a) << std::endl; -``` -Output: -```json -[0,1,2,0,0,0,0,0,0,0] -``` -#### Make a two dimensional array of size 3x4 initialized with zeros -```c++ -json a = json::make_array<2>(3,4,0); -a[0][0] = "Tenor"; -a[0][1] = "ATM vol"; -a[0][2] = "25-d-MS"; -a[0][3] = "25-d-RR"; -a[1][0] = "1Y"; -a[1][1] = 0.20; -a[1][2] = 0.009; -a[1][3] = -0.006; -a[2][0] = "2Y"; -a[2][1] = 0.18; -a[2][2] = 0.009; -a[2][3] = -0.005; - -std::cout << pretty_print(a) << std::endl; -``` -Output: -```json -[ - ["Tenor","ATM vol","25-d-MS","25-d-RR"], - ["1Y",0.2,0.009,-0.006], - ["2Y",0.18,0.009,-0.005] -] -``` -#### Make a three dimensional array of size 4x3x2 initialized with zeros -```c++ -json a = json::make_array<3>(4,3,2,0); -a[0][2][0] = 2; -a[0][2][1] = 3; -std::cout << pretty_print(a) << std::endl; -``` -Output: -```json -[ - [ - [0,0], - [0,0], - [2,3] - ], - [ - [0,0], - [0,0], - [0,0] - ], - [ - [0,0], - [0,0], - [0,0] - ], - [ - [0,0], - [0,0], - [0,0] - ] -] -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/merge.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/merge.md deleted file mode 100644 index 2603436ddf..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/merge.md +++ /dev/null @@ -1,60 +0,0 @@ -### jsoncons::json::merge - -```c++ -void merge(const json& source); // (1) -void merge(json&& source); // (2) -void merge(object_iterator hint, const json& source); // (3) -void merge(object_iterator hint, json&& source); // (4) -``` - -Copies the key-value pairs in source json object into json object. If there is a member in source json object with key equivalent to the key of a member in json object, -then that member is not copied. - -The `merge` function performs only a one-level-deep shallow merge, not a deep merge of nested objects. - -#### Parameters - - - - - - -
source`json` object value
- -#### Return value - -None - -#### Exceptions - -Throws `std::runtime_error` if source or *this are not json objects. - -### Examples - -#### Merge `json` - -```c++ -json j = json::parse(R"( -{ - "a" : 1, - "b" : 2 -} -)"); - -const json source = json::parse(R"( -{ - "a" : 2, - "c" : 3 -} -)"); - -j1.merge(source); - -std::cout << j << endl; -``` -Output: - -```json -{"a":1,"b":2,"c":3} -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/merge_or_update.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/merge_or_update.md deleted file mode 100644 index eaeb41e7ea..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/merge_or_update.md +++ /dev/null @@ -1,59 +0,0 @@ -### jsoncons::json::merge_or_update - -```c++ -void merge_or_update(const json& source); // (1) -void merge_or_update(json&& source); // (2) -void merge_or_update(object_iterator hint, const json& source); // (3) -void merge_or_update(object_iterator hint, json&& source); // (4) -``` - -Inserts another json object's key-value pairs into a json object, or assigns them if they already exist. - -The `merge_or_update` function performs only a one-level-deep shallow merge, not a deep merge of nested objects. - -#### Parameters - - - - - - -
source`json` object value
- -#### Return value - -None - -#### Exceptions - -Throws `std::runtime_error` if source or *this are not json objects. - -### Examples - -#### Merge or update `json` - -```c++ -json j = json::parse(R"( -{ - "a" : 1, - "b" : 2 -} -)"); - -const json source = json::parse(R"( -{ - "a" : 2, - "c" : 3 -} -)"); - -j1.merge_or_update(source); - -std::cout << j << endl; -``` -Output: - -```json -{"a":2,"b":2,"c":3} -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/object_range.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/object_range.md deleted file mode 100644 index 6733e0e60e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/object_range.md +++ /dev/null @@ -1,63 +0,0 @@ -### jsoncons::json::object_range - -```c++ -range object_range(); -range object_range() const; -``` -Returns a "range" that supports a range-based for loop over the key-value pairs of a `json` object -Throws `std::runtime_error` if not an object. - -### Examples - -#### Range-based for loop over key-value pairs of an object - -```c++ -#include - -int main() -{ - json j = json::parse(R"( -{ - "category" : "Fiction", - "title" : "Pulp", - "author" : "Charles Bukowski", - "date" : "2004-07-08", - "price" : 22.48, - "isbn" : "1852272007" -} -)"); - - for (const auto& member : j.object_range()) - { - std::cout << member.key() << " => " << member.value().as() << std::endl; - } -} -``` -Output: -```json -author => Charles Bukowski -category => Fiction -date => 2004-07-08 -isbn => 1852272007 -price => 22.48 -title => Pulp -``` -#### Object iterator -```c++ -json j; -j["city"] = "Toronto"; -j["province"] = "Ontario"; -j["country"] = "Canada"; - -for (auto it = j.object_range().begin(); it != j.object_range().end(); ++it) -{ - std::cout << it->key() << " => " << it->value().as() << std::endl; -} -``` -Output: -```c++ -city => Toronto -country => Canada -province => Ontario -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/operator=.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/operator=.md deleted file mode 100644 index 4e9d2c8337..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/operator=.md +++ /dev/null @@ -1,16 +0,0 @@ -### `jsoncons::json::operator=` - -```c++ -json& operator=(const json& rhs); -json& operator=(json&& rhs) noexcept; // (1) - -template -json& operator=(const T& rhs); // (2) - -json& operator=(const char_type* rhs); // (3) -``` - -(1) Assigns a new `json` value to a `json` variable, replacing it's current contents. - -(2) Assigns the templated value to a `json` variable using [json_type_traits](json_type_traits.md). - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/parse.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/parse.md deleted file mode 100644 index c229598cbf..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/parse.md +++ /dev/null @@ -1,113 +0,0 @@ -### jsoncons::json::parse - -```c++ -static json parse(const string_view_type& s); // (1) - -static json parse(const string_view_type& s, - const json_options& options); // (2) - -static json parse(const string_view_type& s, - parse_error_handler& err_handler); // (3) - -static json parse(const string_view_type& s, - const json_options& options, - parse_error_handler& err_handler); // (4) - -static json parse(std::istream& is); // (5) - -static json parse(std::istream& is, - const json_options& options); // (6) - -static json parse(std::istream& is, - const json_options& options, - parse_error_handler& err_handler); // (7) - -static json parse(std::istream& is, - const json_options& options, - parse_error_handler& err_handler); // (8) -``` -(1) - (4) Parses a string of JSON text and returns a json object or array value. -Throws [ser_error](ser_error.md) if parsing fails. - -(5) - (8) Parses an input stream of JSON text and returns a json object or array value. -Throws [ser_error](ser_error.md) if parsing fails. - -### Examples - -#### Parse from stream - -Input example.json: - -```json -{"File Format Options":{"Color Spaces":["sRGB","AdobeRGB","ProPhoto RGB"]}} -``` - -```c++ -std::ifstream is("example.json"); -json j = json::parse(is); - -std::cout << pretty_print(j) << std::endl; -``` - -Output: - -```json -{ - "File Format Options": { - "Color Spaces": ["sRGB","AdobeRGB","ProPhoto RGB"] - } -} -``` - -#### Parse from string - -```c++ -try -{ - json val = json::parse("[1,2,3,4,]"); -} -catch(const jsoncons::ser_error& e) -{ - std::cout << e.what() << std::endl; -} -``` -Output: -``` -Extra comma at line 1 and column 10 -``` - -#### Parse from string with serializing options - -```c++ -std::string s = R"({"field1":"NaN","field2":"PositiveInfinity","field3":"NegativeInfinity"})"; - -json_options options; -options.nan_to_str("NaN") - .inf_to_str("PositiveInfinity") - .neginf_to_str("NegativeInfinity"); - -json j = json::parse(s,options); - -std::cout << "\n(1)\n" << pretty_print(j) << std::endl; - -std::cout << "\n(2)\n" << pretty_print(j,options) << std::endl; -``` -Output: -``` -(1) -{ - "field1": null, - "field2": null, - "field3": null -} - -(2) -{ - "field1": "NaN", - "field2": "PositiveInfinity", - "field3": "NegativeInfinity" -} -``` - - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/push_back.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/push_back.md deleted file mode 100644 index 070bde16f1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/push_back.md +++ /dev/null @@ -1,50 +0,0 @@ -### jsoncons::json::push_back - -```c++ -template -void push_back(T&& val) -``` -Adds a new json element at the end of a json array. The argument `val` is forwarded to the `json` constructor as `std::forward(val)`. -Throws `std::runtime_error` if not an array. - -### Examples - -#### Creating an array of elements -```c++ -json cities = json::array(); // an empty array -std::cout << cities << std::endl; // output is "[]" - -cities.push_back("Toronto"); -cities.push_back("Vancouver"); -// Insert "Montreal" at beginning of array -cities.insert(cities.array_range().begin(),"Montreal"); - -std::cout << cities << std::endl; -``` -Output: -``` -[] -["Montreal","Toronto","Vancouver"] -``` -#### Creating an array of elements with reserved storage -```c++ -json cities = json::array(); -cities.reserve(10); // storage is reserved -std::cout << "capacity=" << cities.capacity() - << ", size=" << cities.size() << std::endl; - -cities.push_back("Toronto"); -cities.push_back("Vancouver"); -cities.insert(cities.array_range().begin(),"Montreal"); -std::cout << "capacity=" << cities.capacity() - << ", size=" << cities.size() << std::endl; - -std::cout << cities << std::endl; -``` -Output: -``` -capacity=10, size=0 -capacity=10, size=3 -["Montreal","Toronto","Vancouver"] -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/try_emplace.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/try_emplace.md deleted file mode 100644 index cf25de74bb..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json/try_emplace.md +++ /dev/null @@ -1,57 +0,0 @@ -### jsoncons::json::try_emplace - -```c++ -template -pair try_emplace(const string_view_type& key, - Args&&... args); // (1) - -template -object_iterator try_emplace(const_object_iterator hint, - const string_view_type& key, - Args&&... args); // (2) -``` - -#### Parameters - - key -The key used both to look up and to insert if not found - - hint -Iterator to the position before which the new element will be inserted - - args -Arguments to forward to the constructor of the element - -#### Return value - -(1) returns a pair consisting of first, an iterator to the inserted value -or the already existing value, -and second, a bool indicating whether the insertion took place -(true for insertion, false for no insertion.) - -(2) returns an iterator to the inserted value -or the already existing value. - -#### Exceptions - -Throws `std::runtime_error` if not a json object. - -### Example - -```c++ -json a; - -a.try_emplace("object1",json()); -a.try_emplace("field1","value1"); -a["object1"].try_emplace("field2","value2"); - -std::cout << a << std::endl; -``` -Output: - -```json -{"field1":"value1","object1":{"field2":"value2"}} -``` - - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_content_handler.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_content_handler.md deleted file mode 100644 index e44c6ee118..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_content_handler.md +++ /dev/null @@ -1,245 +0,0 @@ -### jsoncons::json_content_handler - -```c++ -typedef basic_json_content_handler json_content_handler -``` - -Defines an interface for receiving JSON events. The `json_content_handler` class is an instantiation of the `basic_json_content_handler` class template that uses `char` as the character type. - -#### Header -```c++ -#include -``` -#### Member types - -Member type |Definition -------------------------------------|------------------------------ -`string_view_type`|A non-owning view of a string, holds a pointer to character data and length. Supports conversion to and from strings. Will be typedefed to the C++ 17 [string view](http://en.cppreference.com/w/cpp/string/basic_string_view) if `JSONCONS_HAS_STRING_VIEW` is defined in `jsoncons_config.hpp`, otherwise proxied. - -#### Public producer interface - - bool begin_object(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); -Indicates the begining of an object of indefinite length. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Throws a [ser_error](ser_error.md) to indicate an error. - - bool begin_object(size_t length, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); -Indicates the begining of an object of known length. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Throws a [ser_error](ser_error.md) to indicate an error. - - bool end_object(const ser_context& context = null_ser_context()) -Indicates the end of an object. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Throws a [ser_error](ser_error.md) to indicate an error. - - bool begin_array(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); -Indicates the beginning of an indefinite length array. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Throws a [ser_error](ser_error.md) to indicate an error. - - bool begin_array(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); -Indicates the beginning of an array of known length. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. - - bool end_array(const ser_context& context=null_ser_context()); -Indicates the end of an array. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Throws a [ser_error](ser_error.md) to indicate an error. - - bool name(const string_view_type& name, - const ser_context& context=null_ser_context()); -Writes the name part of a name-value pair inside an object. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Throws a [ser_error](ser_error.md) to indicate an error. - - bool string_value(const string_view_type& value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); -Writes a string value. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Throws a [ser_error](ser_error.md) to indicate an error. - - bool byte_string_value(const byte_string_view& b, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); -Writes a byte string value. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Throws a [ser_error](ser_error.md) to indicate an error. - - bool byte_string_value(const uint8_t* p, size_t size, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); -Writes a byte string value. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Throws a [ser_error](ser_error.md) to indicate an error. - - bool int64_value(int64_t value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); -Writes a signed integer value. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Throws a [ser_error](ser_error.md) to indicate an error. - - bool uint64_value(uint64_t value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); -Writes a non-negative integer value. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Throws a [ser_error](ser_error.md) to indicate an error. - - bool double_value(double value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); -Writes a floating point value. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Throws a [ser_error](ser_error.md) to indicate an error. - - bool bool_value(bool value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); -Writes a boolean value. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Throws a [ser_error](ser_error.md) to indicate an error. - - bool null_value(semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); -Writes a null value. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Throws a [ser_error](ser_error.md) to indicate an error. - - void flush() -Flushes whatever is buffered to the destination. - -#### Private virtual consumer interface - - virtual bool do_begin_object(semantic_tag tag, - const ser_context& context) = 0; -Handles the beginning of an object of indefinite length. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Sets `ec` to indicate an error. - - virtual bool do_begin_object(size_t length, - semantic_tag tag, - const ser_context& context) = 0; -Handles the beginning of an object of known length. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Sets `ec` to indicate an error. - - virtual bool do_end_object(const ser_context& context) = 0; -Handles the end of an object. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Sets `ec` to indicate an error. - - virtual bool do_begin_array(semantic_tag tag, - const ser_context& context) = 0; -Handles the beginning of an array of indefinite length. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Sets `ec` to indicate an error. - - virtual bool do_begin_array(size_t length, - const ser_context& context); -Handles the beginning of an array of known length. Defaults to calling `do_begin_array(semantic_tag, const ser_context& context)`. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Sets `ec` to indicate an error. - - virtual bool do_end_array(const ser_context& context) = 0; -Handles the end of an array. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Sets `ec` to indicate an error. - - virtual bool do_name(const string_view_type& name, - const ser_context& context) = 0; -Handles the name part of a name-value pair inside an object. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Sets `ec` to indicate an error. - - virtual bool do_string_value(const string_view_type& val, - semantic_tag tag, - const ser_context& context) = 0; -Handles a string value. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Sets `ec` to indicate an error. - - virtual bool do_byte_string_value(const byte_string_view& b, - semantic_tag tag, - const ser_context& context) = 0; -Handles a byte string value. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Sets `ec` to indicate an error. - - virtual bool do_int64_value(int64_t value, - semantic_tag tag, - const ser_context& context) = 0; -Handles a signed integer value. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Sets `ec` to indicate an error. - - virtual bool do_uint64_value(uint64_t value, - semantic_tag tag, - const ser_context& context) = 0; -Handles a non-negative integer value. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Sets `ec` to indicate an error. - - virtual bool do_double_value(double value, - semantic_tag tag, - const ser_context& context) = 0; -Handles a floating point value. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Sets `ec` to indicate an error. - - virtual bool do_bool_value(bool value, - semantic_tag tag, - const ser_context& context) = 0; -Handles a boolean value. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Sets `ec` to indicate an error. - - virtual bool do_null_value(semantic_tag tag, - const ser_context& context) = 0; -Handles a null value. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. -Returns `true` if the producer should continue streaming events, `false` otherwise. -Sets `ec` to indicate an error. - - virtual void do_flush() = 0; -Allows producers of json events to flush whatever they've buffered. - -#### See also - -- [semantic_tag](../semantic_tag.md) - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_cursor.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_cursor.md deleted file mode 100644 index 23c170c309..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_cursor.md +++ /dev/null @@ -1,358 +0,0 @@ -### jsoncons::json_cursor - -```c++ -typedef basic_json_cursor json_cursor -``` - -A pull parser for parsing json events. A typical application will -repeatedly process the `current()` event and call the `next()` -function to advance to the next event, until `done()` returns `true`. - -`json_cursor` is noncopyable and nonmoveable. - -#### Header -```c++ -#include -``` - -### Implemented interfaces - -[staj_reader](staj_reader.md) - -#### Constructors - - template - basic_json_cursor(Source&& source); // (1) - - template - basic_json_cursor(Source&& source, - basic_staj_filter& filter); // (2) - - template - basic_json_cursor(Source&& source, - parse_error_handler& err_handler); // (3) - - template - basic_json_cursor(Source&& source, - basic_staj_filter& filter, - parse_error_handler& err_handler); // (4) - - template - basic_json_cursor(Source&& source, - const basic_json_decode_options& options); // (5) - - template - basic_json_cursor(Source&& source, - basic_staj_filter& filter, - const basic_json_decode_options& options); // (6) - - template - basic_json_cursor(Source&& source, - basic_staj_filter& filter, - const basic_json_decode_options& options, - parse_error_handler& err_handler); // (7) - -Constructors (1)-(7) read from a character sequence or stream and throw a -[ser_error](ser_error.md) if a parsing error is encountered -while processing the initial event. - -(1) Constructs a `json_cursor` that reads from a character sequence or stream `source`, uses default [json_decode_options](json_decode_options.md) -and a default [parse_error_handler](parse_error_handler.md). - -(2) Constructs a `json_cursor` that reads from a character sequence or stream `source`, applies a [staj_filter](staj_filter.md) to the JSON events, uses default [json_decode_options](json_decode_options.md) -and a default [parse_error_handler](parse_error_handler.md). - -(3) Constructs a `json_cursor` that reads from a character sequence or stream `source`, -uses the specified [json_decode_options](json_decode_options.md) -and a default [parse_error_handler](parse_error_handler.md). - -(4) Constructs a `json_cursor` that reads from a character sequence or stream `source`, -applies a [staj_filter](staj_filter.md) to the JSON events, -uses the specified [json_decode_options](json_decode_options.md) -and a default [parse_error_handler](parse_error_handler.md). - -(5) Constructs a `json_cursor` that reads from a character sequence or stream `source`, -uses default [json_decode_options](json_decode_options.md) -and a specified [parse_error_handler](parse_error_handler.md). - -(6) Constructs a `json_cursor` that reads from a character sequence or stream `source`, -applies a [staj_filter](staj_filter.md) to the JSON events, -uses default [json_decode_options](json_decode_options.md) -and a specified [parse_error_handler](parse_error_handler.md). - -(7) Constructs a `json_cursor` that reads from a character sequence or stream `source`, -applies a [staj_filter](staj_filter.md) to the JSON events, -uses the specified [json_decode_options](json_decode_options.md) -and a specified [parse_error_handler](parse_error_handler.md). - - template - basic_json_cursor(Source&& source, - std::error_code& ec); // (8) - - template - basic_json_cursor(Source&& source, - basic_staj_filter& filter, - std::error_code& ec) // (9) - - template - basic_json_cursor(Source&& source, - parse_error_handler& err_handler, - std::error_code& ec) // (10) - - template - basic_json_cursor(Source&& source, - basic_staj_filter& filter, - parse_error_handler& err_handler, - std::error_code& ec) // (11) - - template - basic_json_cursor(Source&& source, - const basic_json_decode_options& options, - std::error_code& ec) // (12) - - template - basic_json_cursor(Source&& source, - basic_staj_filter& filter, - const basic_json_decode_options& options, - std::error_code& ec) // (13) - - template - basic_json_cursor(Source&& source, - basic_staj_filter& filter, - const basic_json_decode_options& options, - parse_error_handler& err_handler, - std::error_code& ec) // (14) - -Constructors (8)-(14) read from a character sequence or stream and set `ec` -if a parsing error is encountered while processing the initial event. - -Note: It is the programmer's responsibility to ensure that `basic_json_cursor` does not outlive any source, -content handler, and error handler passed in the constuctor, as `basic_json_cursor` holds pointers to but does not own these resources. - -#### Parameters - -`source` - a value from which a `jsoncons::basic_string_view` is constructible, -or a value from which a `source_type` is constructible. In the case that a `jsoncons::basic_string_view` is constructible -from `source`, `source` is dispatched immediately to the parser. Otherwise, the `json_cursor` reads from a `source_type` in chunks. - -#### Member functions - - bool done() const override; -Checks if there are no more events. - - const staj_event& current() const override; -Returns the current [staj_event](staj_event.md). - - void accept(json_content_handler& handler) override -Sends the parse events from the current event to the -matching completion event to the supplied [handler](json_content_handler.md) -E.g., if the current event is `begin_object`, sends the `begin_object` -event and all inbetween events until the matching `end_object` event. -If a parsing error is encountered, throws a [ser_error](ser_error.md). - - void accept(json_content_handler& handler, - std::error_code& ec) override -Sends the parse events from the current event to the -matching completion event to the supplied [handler](json_content_handler.md) -E.g., if the current event is `begin_object`, sends the `begin_object` -event and all inbetween events until the matching `end_object` event. -If a parsing error is encountered, sets `ec`. - - void next() override; -Advances to the next event. If a parsing error is encountered, throws a -[ser_error](ser_error.md). - - void next(std::error_code& ec) override; -Advances to the next event. If a parsing error is encountered, sets `ec`. - - const ser_context& context() const override; -Returns the current [context](ser_context.md) - -### Examples - -The example JSON text, `book_catalog.json`, is used by the examples below. - -```json -[ - { - "author" : "Haruki Murakami", - "title" : "Hard-Boiled Wonderland and the End of the World", - "isbn" : "0679743464", - "publisher" : "Vintage", - "date" : "1993-03-02", - "price": 18.90 - }, - { - "author" : "Graham Greene", - "title" : "The Comedians", - "isbn" : "0099478374", - "publisher" : "Vintage Classics", - "date" : "2005-09-21", - "price": 15.74 - } -] -``` - -#### Reading a JSON stream - -```c++ -#include -#include -#include - -using namespace jsoncons; - -int main() -{ - std::ifstream is("book_catalog.json"); - - json_cursor reader(is); - - for (; !reader.done(); reader.next()) - { - const auto& event = reader.current(); - switch (event.event_type()) - { - case staj_event_type::begin_array: - std::cout << "begin_array\n"; - break; - case staj_event_type::end_array: - std::cout << "end_array\n"; - break; - case staj_event_type::begin_object: - std::cout << "begin_object\n"; - break; - case staj_event_type::end_object: - std::cout << "end_object\n"; - break; - case staj_event_type::name: - // If underlying type is string, can return as string_view - std::cout << "name: " << event.as() << "\n"; - break; - case staj_event_type::string_value: - std::cout << "string_value: " << event.as() << "\n"; - break; - case staj_event_type::null_value: - std::cout << "null_value: " << event.as() << "\n"; - break; - case staj_event_type::bool_value: - std::cout << "bool_value: " << event.as() << "\n"; - // or std::cout << "bool_value: " << event.as() << "\n"; - break; - case staj_event_type::int64_value: - std::cout << "int64_value: " << event.as() << "\n"; - // or std::cout << "int64_value: " << event.as() << "\n"; - break; - case staj_event_type::uint64_value: - std::cout << "uint64_value: " << event.as() << "\n"; - // or std::cout << "int64_value: " << event.as() << "\n"; - break; - case staj_event_type::double_value: - std::cout << "double_value: " << event.as() << "\n"; - // or std::cout << "double_value: " << event.as() << "\n"; - break; - default: - std::cout << "Unhandled event type\n"; - break; - } - } -} -``` -Output: -``` -begin_array -begin_object -name: author -string_value: Haruki Murakami -name: title -string_value: Hard-Boiled Wonderland and the End of the World -name: isbn -string_value: 0679743464 -name: publisher -string_value: Vintage -name: date -string_value: 1993-03-02 -name: price -double_value: 18.90 -end_object -begin_object -name: author -string_value: Graham Greene -name: title -string_value: The Comedians -name: isbn -string_value: 0099478374 -name: publisher -string_value: Vintage Classics -name: date -string_value: 2005-09-21 -name: price -double_value: 15.74 -end_object -end_array -``` - -#### Filtering a JSON stream - -```c++ -#include -#include -#include - -using namespace jsoncons; - -class author_filter : public staj_filter -{ - bool accept_next_ = false; -public: - bool accept(const staj_event& event, const ser_context&) override - { - if (event.event_type() == staj_event_type::name && - event.as() == "author") - { - accept_next_ = true; - return false; - } - else if (accept_next_) - { - accept_next_ = false; - return true; - } - else - { - accept_next_ = false; - return false; - } - } -}; - -int main() -{ - std::ifstream is("book_catalog.json"); - - author_filter filter; - json_cursor reader(is, filter); - - for (; !reader.done(); reader.next()) - { - const auto& event = reader.current(); - switch (event.event_type()) - { - case staj_event_type::string_value: - std::cout << event.as() << "\n"; - break; - } - } -} -``` -Output: -``` -Haruki Murakami -Graham Greene -``` - -#### See also - -- [staj_array_iterator](staj_array_iterator.md) -- [staj_object_iterator](staj_object_iterator.md) - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_decode_options.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_decode_options.md deleted file mode 100644 index 5858a8d99f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_decode_options.md +++ /dev/null @@ -1,47 +0,0 @@ -### jsoncons::json_decode_options - -```c++ -typedef basic_json_decode_options json_decode_options -``` - -An abstract class that defines accessors for JSON decode options. The `json_decode_options` class is an instantiation of the `basic_json_decode_options` class template that uses `char` as the character type. - -#### Header -```c++ -#include -``` - -#### Implementing classes - -[json_options](json_options.md) - -#### Destructor - - virtual ~json_decode_options(); - -#### Accessors - - virtual bool is_str_to_nan() const = 0; -Indicates `NaN` replacement for string when parsing. - - virtual std::string nan_to_str() const = 0; -When parsing JSON text, replace string with a `NaN` if `is_nan_to_str()` returns `true`. - - virtual bool is_str_to_inf() const = 0; -Indicates `Infinity` replacement for string when parsing. - - virtual const std::string& inf_to_str() const = 0; -When parsing JSON text, replace string with infinity if `is_inf_to_str()` returns `true`. - - virtual bool is_str_to_neginf() const = 0; -Indicates `Negative Infinity` replacement for string when parsing. - - virtual const std::string& neginf_to_str() const = 0; -When parsing JSON text, replace string with minus infinity if `is_neginf_to_str()` returns true. - - virtual bool lossless_number() const = 0; -If set to `true`, parse decimal numbers as strings with semantic tagging `semantic_tag::bigdec` instead of double. - - virtual size_t max_nesting_depth() = 0; - Maximum nesting depth when parsing JSON. - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_decoder.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_decoder.md deleted file mode 100644 index 3d930dd83d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_decoder.md +++ /dev/null @@ -1,39 +0,0 @@ -### jsoncons::json_decoder - -```c++ -template -json_decoder -``` - -#### Header -```c++ -#include -``` - -![json_decoder](./diagrams/json_decoder.png) - -#### Member types - -Member type |Definition -------------------------------------|------------------------------ -`allocator_type`|Json::allocator_type - -#### Constructors - - json_decoder(const allocator_type allocator = allocator_type()) - -#### Member functions - - allocator_type get_allocator() const -Returns the allocator associated with the json value. - - bool is_valid() const -Checks if the `deserializer` contains a valid json_type value. The initial `is_valid()` is false, becomes `true` when a `do_end_document` event is received, and becomes false when `get_result()` is called. - - Json get_result() -Returns the json value `v` stored in the `deserializer` as `std::move(v)`. If before calling this function `is_valid()` is false, the behavior is undefined. After `get_result()` is called, 'is_valid()' becomes false. - -### See also - -- [json_content_handler](json_content_handler.md) - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_encode_options.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_encode_options.md deleted file mode 100644 index f76f2575e9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_encode_options.md +++ /dev/null @@ -1,105 +0,0 @@ -### jsoncons::json_encode_options - -```c++ -typedef basic_json_encode_options json_encode_options -``` - -An abstract class that defines accessors for JSON encoding options. The `json_encode_options` class is an instantiation of the `basic_json_encode_options` class template that uses `char` as the character type. - -#### Header -```c++ -#include -``` - -#### Implementing classes - -[json_options](json_options.md) - -#### Destructor - - virtual ~json_encode_options(); - -#### Accessors - - virtual size_t indent_size() const = 0; -The indent size, the default is 4. - - virtual spaces_option spaces_around_colon() const = 0; -Indicates [space option](spaces_option.md) for name separator (`:`). Default -is space after. - - virtual spaces_option spaces_around_comma() const = 0; -Indicates [space option](spaces_option.md) for array value and object name/value pair separators (`,`). Default -is space after. - - virtual bool pad_inside_object_braces() const = 0; -Default is `false` - - virtual bool pad_inside_array_brackets() const = 0; -Default is `false` - - virtual chars_format floating_point_format() const = 0 -Overrides [floating point format](chars_format.md) when serializing json. -The default, for a floating point value that was previously decoded from json text, is to preserve the original format when serializing. -The default, for a floating point value that was directly inserted into a json value, to serialize with [chars_format::general](chars_format.md). - - virtual bigint_chars_format bigint_format() const = 0 -Overrides [bignum format](bigint_chars_format.md) when serializing json. -The default is [bigint_chars_format::base10](bigint_chars_format.md). - - virtual byte_string_chars_format byte_string_format() const = 0 -Overrides [byte string format](byte_string_chars_format.md) when serializing json. -The default is [byte_string_chars_format::base64url](byte_string_chars_format.md). - - virtual int precision() const = 0 -Overrides floating point precision when serializing json. -The default, for a floating point value that was previously decoded from json text, is to preserve the original precision. -The fefault, for a floating point value that was directly inserted into a json value, to serialize with shortest representation. - - virtual bool escape_all_non_ascii() const = 0 -Escape all non-ascii characters. The default is `false`. - - virtual bool escape_solidus() const = 0 -Escape the solidus ('/') character. The default is `false`. - - virtual bool is_nan_to_num() const = 0; - virtual bool is_nan_to_str() const = 0; - virtual const std::string& nan_to_num() const = 0 - virtual const std::string& nan_to_str() const = 0 -Replace `NaN` with a number, if `is_nan_to_num()` returns `true`, -or a string, if `is_nan_to_str()` returns `true`. If both -return `false`, replace `NaN` with `null`. - - virtual bool is_inf_to_num() const = 0; - virtual bool is_inf_to_str() const = 0; - virtual const std::string& inf_to_num() const = 0 - virtual const std::string& inf_to_str() const = 0 -Replace positive infinity with a number, if `is_inf_to_num()` returns `true`, -or a string, if `is_inf_to_str()` returns `true`. If both -return `false`, replace positive infinity with `null`. - - virtual bool is_neginf_to_num() const = 0 - virtual bool is_neginf_to_str() const = 0 - virtual const std::string& neginf_to_num() const = 0 - virtual const std::string& neginf_to_str() const = 0 -Replace negative infinity with a number, if `is_neginf_to_num()` returns `true`, -or a string, if `is_neginf_to_str()` returns true. If both -return `false`, replace negative infinity with `null`. - - virtual std::string new_line_chars() const = 0 -Defaults to "\n" - - virtual size_t line_length_limit() const = 0 - - virtual line_split_kind object_object_line_splits() const = 0; -For an object whose parent is an object, indicates whether that object is split on a new line, or if its members are split on multiple lines. The default is [line_split_kind::multi_line](line_split_kind.md). - - virtual line_split_kind array_object_line_splits() const = 0; -For an object whose parent is an array, indicates whether that object is split on a new line, or if its members are split on multiple lines. The default is [line_split_kind::multi_line](line_split_kind.md). - - virtual line_split_kind object_array_line_splits() const = 0; -For an array whose parent is an object, indicates whether that array is split on a new line, or if its elements are split on multiple lines. The default is [line_split_kind::same_line](line_split_kind.md). - - virtual line_split_kind array_array_line_splits() const = 0; -For an array whose parent is an array, indicates whether that array is split on a new line, or if its elements are split on multiple lines. The default is [line_split_kind::new_line](line_split_kind.md). - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_encoder.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_encoder.md deleted file mode 100644 index 36ae6eff88..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_encoder.md +++ /dev/null @@ -1,160 +0,0 @@ -### jsoncons::basic_json_encoder - -```c++ -template< - class CharT, - class Result> -> basic_json_encoder : public jsoncons::basic_json_content_handler - -template< - class CharT, - class Result> -> basic_json_compressed_encoder : public jsoncons::basic_json_content_handler -``` - -`basic_json_encoder` and `basic_json_compressed_encoder` are noncopyable and nonmoveable. - -#### Header - - #include - -![json_encoder](./diagrams/json_encoder.png) - -Four specializations for common character types and result types are defined -for both the pretty print and compressed serializers: - -Type |Definition ----------------------------|------------------------------ -json_encoder |basic_json_encoder> -json_string_encoder |basic_json_encoder> -wjson_encoder |basic_json_encoder> -wjson_string_encoder |basic_json_encoder> -json_compressed_encoder |basic_json_compressed_encoder> -json_compressed_string_encoder |basic_json_compressed_encoder> -wjson_compressed_encoder |basic_json_compressed_encoder> -wjson_compressed_string_encoder |basic_json_compressed_encoder> - -#### Member types - -Type |Definition ----------------------------|------------------------------ -char_type |CharT -result_type |Result -string_view_type | - -#### Constructors - - explicit basic_json_encoder(result_type result) -Constructs a new encoder that is associated with the output adaptor `result`. - - basic_json_encoder(result_type result, - const basic_json_encode_options& options) -Constructs a new encoder that is associated with the output adaptor `result` -and uses the specified [json options](json_options.md). - -#### Destructor - - virtual ~basic_json_encoder() - -### Inherited from [basic_json_content_handler](../json_content_handler.md) - -#### Member functions - - bool begin_object(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool begin_object(size_t length, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool end_object(const ser_context& context = null_ser_context()) - - bool begin_array(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool begin_array(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool end_array(const ser_context& context=null_ser_context()); - - bool name(const string_view_type& name, - const ser_context& context=null_ser_context()); - - bool string_value(const string_view_type& value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool byte_string_value(const byte_string_view& b, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool byte_string_value(const uint8_t* p, size_t size, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool int64_value(int64_t value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool uint64_value(uint64_t value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool double_value(double value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool bool_value(bool value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool null_value(semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - void flush() - -### Examples - -### Feeding json events directly to a `json_encoder` -```c++ -#include -#include -#include - -using namespace jsoncons; -using boost::numeric::ublas::matrix; - -int main() -{ - matrix A(2, 2); - A(0, 0) = 1; - A(0, 1) = 2; - A(1, 0) = 3; - A(1, 1) = 4; - - json_options options; - json_encoder os(std::cout, options); - os.begin_array(); - for (size_t i = 0; i < A.size1(); ++i) - { - os.begin_array(); - for (size_t j = 0; j < A.size2(); ++j) - { - os.double_value(A(i, j)); - } - os.end_array(); - } - os.end_array(); - - return 0; -} -``` - -Output: - -```json -[ - [1,2], - [3,4] -] -``` diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_error.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_error.md deleted file mode 100644 index 1ed89614dd..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_error.md +++ /dev/null @@ -1,38 +0,0 @@ -### jsoncons::json_errc - -The constant integer values scoped by `jsoncons::json_errc` define the values for json text error codes. - -#### Header - - #include - -### Member constants - -constant |Description -------------------------------------|------------------------------ -`unexpected_eof` |Unexpected end of file -`invalid_json_text` |Invalid JSON text -`extra_character` |Unexpected non-whitespace character after JSON text -`max_depth_exceeded` |Maximum JSON depth exceeded -`single_quote` |JSON strings cannot be quoted with single quotes -`illegal_character_in_string` |Illegal character in string -`extra_comma` |Extra comma -`expected_name` |Expected object member name -`expected_value` |Expected value -`invalid_value` |Invalid value -`expected_colon` |Expected name separator ':' -`illegal_control_character` |Illegal control character in string -`illegal_escaped_character` |Illegal escaped character in string -`expected_codepoint_surrogate_pair` |Invalid codepoint, expected another \\u token to begin the second half of a codepoint surrogate pair. -`invalid_hex_escape_sequence` |Invalid codepoint, expected hexadecimal digit. -`invalid_unicode_escape_sequence` |Invalid codepoint, expected four hexadecimal digits. -`leading_zero` |A number cannot have a leading zero -`invalid_number` |Invalid number -`expected_comma_or_right_brace` |Expected comma or right brace ']' -`expected_comma_or_right_bracket` |Expected comma or right bracket '}' -`unexpected_right_brace` |Unexpected right brace '}' -`unexpected_right_bracket` |Unexpected right bracket ']' - - - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_filter.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_filter.md deleted file mode 100644 index ea22808cbc..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_filter.md +++ /dev/null @@ -1,210 +0,0 @@ -### jsoncons::json_filter - -```c++ -typedef basic_json_filter json_filter -``` - -The `json_filter` class is an instantiation of the `basic_json_filter` class template that uses `char` as the character type. - -`json_filter` is noncopyable and nonmoveable. - -#### Header - - #include - -![json_filter](./diagrams/json_filter.png) - -#### Member types - -Member type |Definition -------------------------------------|------------------------------ -`string_view_type`|A non-owning view of a string, holds a pointer to character data and length. Supports conversion to and from strings. Will be typedefed to the C++ 17 [string view](http://en.cppreference.com/w/cpp/string/basic_string_view) if `JSONCONS_HAS_STRING_VIEW` is defined in `jsoncons_config.hpp`, otherwise proxied. - -#### Constructors - - json_filter(json_content_handler& handler) -All JSON events that pass through the `json_filter` go to the specified `json_content_handler` (e.g. another filter.) -You must ensure that the `handler` exists as long as does `json_filter`, as `json_filter` holds a pointer to but does not own this object. - -#### Accessors - - json_content_handler& to_handler() -Returns a reference to the JSON handler that sends json events to a destination handler. - -### See also - -- [json_content_handler](json_content_handler.md) - -### Examples - -#### Rename object member names with the built in filter [rename_object_member_filter](rename_object_member_filter.md) - -```c++ -#include -#include -#include - -using namespace jsoncons; - -int main() -{ - std::string s = R"({"first":1,"second":2,"fourth":3,"fifth":4})"; - - json_encoder encoder(std::cout); - - // Filters can be chained - rename_object_member_filter filter2("fifth", "fourth", encoder); - rename_object_member_filter filter1("fourth", "third", filter2); - - // A filter can be passed to any function that takes - // a json_content_handler ... - std::cout << "(1) "; - std::istringstream is(s); - json_reader reader(is, filter1); - reader.read(); - std::cout << std::endl; - - // or a json_content_handler - std::cout << "(2) "; - ojson j = ojson::parse(s); - j.dump(filter1); - std::cout << std::endl; -} -``` -Output: -```json -(1) {"first":1,"second":2,"third":3,"fourth":4} -(2) {"first":1,"second":2,"third":3,"fourth":4} -``` - -#### Fix up names in an address book JSON file - -Example address book file (`address-book.json`): -```json -{ - "address-book" : - [ - { - "name":"Jane Roe", - "email":"jane.roe@example.com" - }, - { - "name":"John", - "email" : "john.doe@example.com" - } - ] -} -``` - -Suppose you want to break the name into a first name and last name, and report a warning when `name` does not contain a space or tab separated part. - -You can achieve the desired result by subclassing the [json_filter](json_filter.md) class, overriding the default methods for receiving name and string value events, and passing modified events on to the parent [json_content_handler](json_content_handler.md) (which in this example will forward them to a [json_encoder](json_encoder.md).) -```c++ -#include -#include -#include - -using namespace jsoncons; - - -class name_fix_up_filter : public json_filter -{ - std::string member_name_; - -public: - name_fix_up_filter(json_content_handler& handler) - : json_filter(handler) - { - } - -private: - bool do_name(const string_view_type& name, - const ser_context& context) override - { - member_name_ = name; - if (member_name_ != "name") - { - this->to_handler().write_name(name, context); - } - return true; - } - - bool do_string_value(const string_view_type& s, - const ser_context& context) override - { - if (member_name_ == "name") - { - size_t end_first = val.find_first_of(" \t"); - size_t start_last = val.find_first_not_of(" \t", end_first); - this->to_handler().write_name("first-name", context); - string_view_type first = val.substr(0, end_first); - this->to_handler().value(first, context); - if (start_last != string_view_type::npos) - { - this->to_handler().write_name("last-name", context); - string_view_type last = val.substr(start_last); - this->to_handler().value(last, context); - } - else - { - std::cerr << "Incomplete name \"" << s - << "\" at line " << context.line() - << " and column " << context.column() << std::endl; - } - } - else - { - this->to_handler().value(s, context); - } - return true; - } -}; -``` -Configure a [rename_object_member_filter](rename_object_member_filter.md) to emit json events to a [json_encoder](json_encoder.md). -```c++ -std::ofstream os("output/new-address-book.json"); -json_encoder encoder(os, jsoncons::indenting::indent); -name_fix_up_filter filter(encoder); -``` -Parse the input and send the json events into the filter ... -```c++ -std::cout << "(1) "; -std::ifstream is("input/address-book.json"); -json_reader reader(is, filter); -reader.read(); -std:: << "\n"; -``` -or read into a json value and write to the filter -```c++ -std::cout << "(2) "; -json j; -is >> j; -j.dump(filter); -std:: << "\n"; -``` -Output: -``` -(1) Incomplete name "John" at line 9 and column 26 -(2) Incomplete name "John" at line 0 and column 0 -``` -Note that when filtering `json` events written from a `json` value to an output handler, contexual line and column information in the original file has been lost. -``` - -The new address book (`address-book-new.json`) with name fixes is -```json -{ - "address-book": - [ - { - "first-name":"Jane", - "last-name":"Roe", - "email":"jane.roe@example.com" - }, - { - "first-name":"John", - "email":"john.doe@example.com" - } - ] -} -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_options.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_options.md deleted file mode 100644 index 3048d3117d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_options.md +++ /dev/null @@ -1,390 +0,0 @@ -### jsoncons::json_options - -```c++ -typedef basic_json_options json_options -``` - -Specifies options for encoding and decoding JSON text. The `json_options` class is an instantiation of the `basic_json_options` class template that uses `char` as the character type. - -The default floating point formatting for a floating point value that was previously decoded from json text is to preserve the original format and precision. This ensures round-trip for both format and precision, e.g. 1.1 read will remain `1.1` when written, and not become `1.1000000000000001` (an equivalent but longer representation.) - -The default floating point formatting for a floating point value that was directly inserted into a json value is [chars_format::general](chars_format.md) with shortest representation. Trailing zeros are removed, except one immediately following the decimal point. The period character (.) is always used as the decimal point, non English locales are ignored. - -#### Header -```c++ -#include -``` - -![json_options](./diagrams/json_options.png) - -#### Member constants - - static const size_t indent_size_default = 4; -The default size indent is 4 - - static const size_t line_length_limit_default = 120; -The default line length limit is 120 - -#### Constructors - - json_options() -Constructs an `json_options` with default values. - -#### Modifiers - - json_options& indent(size_t value) -The indent size, the default is 4. - - json_options & spaces_around_colon(spaces_option value) -Indicates [space option](spaces_option.md) for name separator (`:`). Default -is space after. - - json_options & spaces_around_comma(spaces_option value) -Indicates [space option](spaces_option.md) for array value and object name/value pair separators (`,`). Default -is space after. - - json_options & pad_inside_object_braces(bool value) -Default is `false` - - json_options & pad_inside_array_brackets(bool value) -Default is `false` - - json_options& bigint_format(bigint_chars_format value) -Overrides [bignum format](bigint_chars_format.md) when serializing json. -The default is [bigint_chars_format::base10](bigint_chars_format.md). - - json_options& byte_string_format(byte_string_chars_format value) -Overrides [byte string format](byte_string_chars_format.md) when serializing json. -The default is [byte_string_chars_format::base64url](byte_string_chars_format.md). - - json_options& precision(int value) -Overrides floating point precision when serializing json. -The default, for a floating point value that was previously decoded from json text, is to preserve the original precision. -The default, for a floating point value that was directly inserted into a json value, to serialize with shortest representation. - - json_options& escape_all_non_ascii(bool value) -Escape all non-ascii characters. The default is `false`. - - json_options& escape_solidus(bool value) -Escape the solidus ('/') character. The default is `false`. - - json_options& nan_to_num(const std::string& value); -Sets a number replacement for `NaN` when writing JSON - - json_options& inf_to_num(const std::string& value); -Sets a number replacement for `Infinity` when writing JSON - - json_options& neginf_to_num(const std::string& value); -Sets a number replacement for `Negative Infinity` when writing JSON - - json_options& nan_to_str(const std::string& value, bool is_str_to_nan = true); -Sets a string replacement for `NaN` when writing JSON, and indicate whether it is also -to be used when reading JSON. - - json_options& inf_to_str(const std::string& value, bool is_str_to_inf = true); -Sets a string replacement for infinity when writing JSON, and indicate whether it is also -to be used when reading JSON. - - json_options& neginf_to_str(const std::string& value, bool is_str_to_neginf = true); // (4) -Sets a string replacement for negative infinity when writing JSON, and indicate whether it is also -to be used when reading JSON. - - json_options& lossless_number(bool value); -If set to `true`, parse numbers with exponents and fractional parts as strings with semantic tagging `semantic_tag::bigdec`. -Defaults to `false`. - - json_options& new_line_chars(const std::string& value) -Defaults to "\n" - - json_options & line_length_limit(size_t value) - - void max_nesting_depth(size_t depth) -The maximum nesting depth allowed when parsing JSON. By default `jsoncons` can read a `JSON` text of arbitrarily large depth. - - json_options& object_object_line_splits(line_split_kind value) -For an object whose parent is an object, set whether that object is split on a new line, or if its members are split on multiple lines. The default is [line_split_kind::multi_line](line_split_kind.md). - - json_options& array_object_line_splits(line_split_kind value) -For an object whose parent is an array, set whether that object is split on a new line, or if its members are split on multiple lines. The default is [line_split_kind::multi_line](line_split_kind.md). - - json_options& object_array_line_splits(line_split_kind value) -For an array whose parent is an object, set whether that array is split on a new line, or if its elements are split on multiple lines. The default is [line_split_kind::same_line](line_split_kind.md). - - json_options& array_array_line_splits(line_split_kind value) -For an array whose parent is an array, set whether that array is split on a new line, or if its elements are split on multiple lines. The default is [line_split_kind::new_line](line_split_kind.md). - -#### Static member functions - - static const basic_json_options& default_options() -Default JSON encode and decode options. - -### See also - -[json_decode_options](json_decode_options.md) -[json_encode_options](json_encode_options.md) - -### Examples - -#### Default NaN and inf replacement -```c++ -json obj; -obj["field1"] = std::sqrt(-1.0); -obj["field2"] = 1.79e308*1000; -obj["field3"] = -1.79e308*1000; -std::cout << obj << std::endl; -``` -Output: -```json -{"field1":null,"field2":null,"field3":null} -``` -#### User specified `Nan` and `Inf` replacement - -```c++ -json obj; -obj["field1"] = std::sqrt(-1.0); -obj["field2"] = 1.79e308*1000; -obj["field3"] = -1.79e308*1000; - -json_options options; -format.nan_to_num("null"); // default is "null" -format.inf_to_num("1e9999"); // default is "null" - -std::cout << pretty_print(obj,options) << std::endl; -``` - -Output: -```json - { - "field1":null, - "field2":1e9999, - "field3":-1e9999 - } -``` - -#### Decimal precision - -By default, jsoncons parses a number with an exponent or fractional part -into a double precision floating point number. If you wish, you can -keep the number as a string with semantic tagging `bigdec`, -using the `lossless_number` option. You can then put it into a `float`, -`double`, a boost multiprecision number, or whatever other type you want. - -```c++ -int main() -{ - std::string s = R"( - { - "a" : 12.00, - "b" : 1.23456789012345678901234567890 - } - )"; - - // Default - json j = json::parse(s); - - std::cout.precision(15); - - // Access as string - std::cout << "(1) a: " << j["a"].as() << ", b: " << j["b"].as() << "\n"; - // Access as double - std::cout << "(2) a: " << j["a"].as() << ", b: " << j["b"].as() << "\n\n"; - - // Using lossless_number option - json_options options; - options.lossless_number(true); - - json j2 = json::parse(s, options); - // Access as string - std::cout << "(3) a: " << j2["a"].as() << ", b: " << j2["b"].as() << "\n"; - // Access as double - std::cout << "(4) a: " << j2["a"].as() << ", b: " << j2["b"].as() << "\n\n"; -} -``` -Output: -``` -(1) a: 12.0, b: 1.2345678901234567 -(2) a: 12, b: 1.23456789012346 - -(3) a: 12.00, b: 1.23456789012345678901234567890 -(4) a: 12, b: 1.23456789012346 -``` - -#### Object-array block formatting - -```c++ - json val; - - val["verts"] = json::array{1, 2, 3}; - val["normals"] = json::array{1, 0, 1}; - val["uvs"] = json::array{0, 0, 1, 1}; - - std::cout << "Default (same line)" << std::endl; - std::cout << pretty_print(val) << std::endl; - - std::cout << "New line" << std::endl; - json_options options1; - format1.object_array_line_splits(line_split_kind::new_line); - std::cout << pretty_print(val,options1) << std::endl; - - std::cout << "Multi line" << std::endl; - json_options options2; - format2.object_array_line_splits(line_split_kind::multi_line); - std::cout << pretty_print(val,options2) << std::endl; -``` - -Output: - -Default (same line) - -```json -{ - "normals": [1,0,1], - "uvs": [0,0,1,1], - "verts": [1,2,3] -} -``` - -New line - -```json -{ - "normals": [ - 1,0,1 - ], - "uvs": [ - 0,0,1,1 - ], - "verts": [ - 1,2,3 - ] -} -``` -Multi line -```json -{ - "normals": [ - 1, - 0, - 1 - ], - "uvs": [ - 0, - 0, - 1, - 1 - ], - "verts": [ - 1, - 2, - 3 - ] -} -``` - -#### Array-array block formatting - -```c++ - json val; - val["data"]["id"] = json::array{0,1,2,3,4,5,6,7}; - val["data"]["item"] = json::array{json::array{2}, - json::array{4,5,2,3}, - json::array{4}, - json::array{4,5,2,3}, - json::array{2}, - json::array{4,5,3}, - json::array{2}, - json::array{4,3}}; - - std::cout << "Default (new line)" << std::endl; - std::cout << pretty_print(val) << std::endl; - - std::cout << "Same line" << std::endl; - json_options options1; - format1.array_array_line_splits(line_split_kind::same_line); - std::cout << pretty_print(val, options1) << std::endl; - - std::cout << "Multi line" << std::endl; - json_options options2; - format2.array_array_line_splits(line_split_kind::multi_line); - std::cout << pretty_print(val, options2) << std::endl; -``` - -Output: - -Default (new line) - -```json -{ - "data": { - "id": [0,1,2,3,4,5,6,7], - "item": [ - [2], - [4,5,2,3], - [4], - [4,5,2,3], - [2], - [4,5,3], - [2], - [4,3] - ] - } -} -``` -Same line - -```json -{ - "data": { - "id": [0,1,2,3,4,5,6,7], - "item": [[2],[4,5,2,3],[4],[4,5,2,3],[2],[4,5,3],[2],[4,3]] - } -} -``` - -Multi line - -```json -{ - "data": { - "id": [ - 0,1,2,3,4,5,6,7 - ], - "item": [ - [ - 2 - ], - [ - 4, - 5, - 2, - 3 - ], - [ - 4 - ], - [ - 4, - 5, - 2, - 3 - ], - [ - 2 - ], - [ - 4, - 5, - 3 - ], - [ - 2 - ], - [ - 4, - 3 - ] - ] - } -} -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_parser.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_parser.md deleted file mode 100644 index b86c77bdc2..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_parser.md +++ /dev/null @@ -1,272 +0,0 @@ -### jsoncons::json_parser - -```c++ -typedef basic_json_parser json_parser -``` -`json_parser` is an incremental json parser. It can be fed its input -in chunks, and does not require an entire file to be loaded in memory -at one time. - -A buffer of text is supplied to the parser with a call to `update(buffer)`. -If a subsequent call to `parse_some` reaches the end of the buffer in the middle of parsing, -say after digesting the sequence 'f', 'a', 'l', member function `stopped()` will return `false` -and `source_exhausted()` will return `true`. Additional JSON text can be supplied to the parser, -`parse_some` called again, and parsing will resume from where it left off. - -A typical application will repeatedly call the `parse_some` function -until `stopped()` returns true. A stopped state indicates that a content -handler function returned `false`, an error occured, or a complete JSON -text has been consumed. If the latter, `done() `will return `true`. - -As an alternative to repeatedly calling `parse_some()` until `stopped()` -returns `true`, when `source_exhausted()` is `true` and there is -no more input, `finish_parse` may be called. - -`check_done` can be called to check if the input has any unconsumed -non-whitespace characters, which would normally be considered an error. - -`json_parser` is used by the push parser [json_reader](json_reader.md), -and by the pull parser [json_cursor](json_cursor.md). - -`json_parser` is noncopyable and nonmoveable. - -#### Header -```c++ -#include -``` -#### Constructors - - json_parser(); // (1) - - json_parser(const json_decode_options& options); // (2) - - json_parser(parse_error_handler& err_handler); // (3) - - json_parser(const json_decode_options& options, - parse_error_handler& err_handler); // (4) - -(1) Constructs a `json_parser` that uses default [json_decode_options](json_decode_options.md) -and a default [parse_error_handler](parse_error_handler.md). - -(2) Constructs a `json_parser` that uses the specified [json_decode_options](json_decode_options.md) -and a default [parse_error_handler](parse_error_handler.md). - -(3) Constructs a `json_parser` that uses default [json_decode_options](json_decode_options.md) -and a specified [parse_error_handler](parse_error_handler.md). - -(4) Constructs a `json_parser` that uses the specified [json_decode_options](json_decode_options.md) -and a specified [parse_error_handler](parse_error_handler.md). - -Note: It is the programmer's responsibility to ensure that `json_reader` does not outlive any error handler passed in the constuctor. - -#### Member functions - - void update(const string_view_type& sv) - void update(const char* data, size_t length) -Update the parser with a chunk of JSON - - bool done() const -Returns `true` when the parser has consumed a complete JSON text, `false` otherwise - - bool stopped() const -Returns `true` if the parser is stopped, `false` otherwise. -The parser may enter a stopped state as a result of a content handler -function returning `false`, an error occurred, -or after having consumed a complete JSON text. - - bool finished() const -Returns `true` if the parser is finished parsing, `false` otherwise. - - bool source_exhausted() const -Returns `true` if the input in the source buffer has been exhausted, `false` otherwise - - void parse_some(json_content_handler& handler) -Parses the source until a complete json text has been consumed or the source has been exhausted. -Parse events are sent to the supplied `handler`. -Throws [ser_error](ser_error.md) if parsing fails. - - void parse_some(json_content_handler& handler, - std::error_code& ec) -Parses the source until a complete json text has been consumed or the source has been exhausted. -Parse events are sent to the supplied `handler`. -Sets `ec` to a [json_errc](jsoncons::json_errc.md) if parsing fails. - - void finish_parse(json_content_handler& handler) -Called after `source_exhausted()` is `true` and there is no more input. -Repeatedly calls `parse_some(handler)` until `finished()` returns `true` -Throws [ser_error](ser_error.md) if parsing fails. - - void finish_parse(json_content_handler& handler, - std::error_code& ec) -Called after `source_exhausted()` is `true` and there is no more input. -Repeatedly calls `parse_some(handler)` until `finished()` returns `true` -Sets `ec` to a [json_errc](jsoncons::json_errc.md) if parsing fails. - - void skip_bom() -Reads the next JSON text from the stream and reports JSON events to a [json_content_handler](json_content_handler.md), such as a [json_decoder](json_decoder.md). -Throws [ser_error](ser_error.md) if parsing fails. - - void check_done() -Throws if there are any unconsumed non-whitespace characters in the input. -Throws [ser_error](ser_error.md) if parsing fails. - - void check_done(std::error_code& ec) -Sets `ec` to a [json_errc](jsoncons::json_errc.md) if parsing fails. - - size_t reset() const -Resets the state of the parser to its initial state. In this state -`stopped()` returns `false` and `done()` returns `false`. - - size_t restart() const -Resets the `stopped` state of the parser to `false`, allowing parsing -to continue. - -### Examples - -#### Incremental parsing - -```c++ -int main() -{ - json_parser parser; - jsoncons::json_decoder decoder; - try - { - parser.update("10"); - parser.parse_some(decoder); - std::cout << "(1) done: " << std::boolalpha << parser.done() << ", source_exhausted: " << parser.source_exhausted() << "\n\n"; - - parser.update(".5"); - parser.parse_some(decoder); // This is the end, but the parser can't tell - std::cout << "(2) done: " << std::boolalpha << parser.done() << ", source_exhausted: " << parser.source_exhausted() << "\n\n"; - - parser.finish_parse(decoder); // Indicates that this is the end - std::cout << "(3) done: " << std::boolalpha << parser.done() << ", source_exhausted: " << parser.source_exhausted() << "\n\n"; - - parser.check_done(); // Checks if there are any unconsumed - // non-whitespace characters in the input - std::cout << "(4) done: " << std::boolalpha << parser.done() << ", source_exhausted: " << parser.source_exhausted() << "\n\n"; - - json j = decoder.get_result(); - std::cout << "(5) " << j << "\n"; - } - catch (const ser_error& e) - { - std::cout << e.what() << std::endl; - } -} -``` - -Output: - -``` -(1) done: false, source_exhausted: true - -(2) done: false, source_exhausted: true - -(3) done: true, source_exhausted: true - -(4) done: true, source_exhausted: true - -(5) 10.5 -``` - -#### Incremental parsing with unconsumed non-whitespace characters - -```c++ -int main() -{ - json_parser parser; - jsoncons::json_decoder decoder; - try - { - parser.update("[10"); - parser.parse_some(decoder); - std::cout << "(1) done: " << std::boolalpha << parser.done() << ", source_exhausted: " << parser.source_exhausted() << "\n\n"; - - parser.update(".5]{}"); - parser.parse_some(decoder); // The parser reaches the end at ']' - std::cout << "(2) done: " << std::boolalpha << parser.done() << ", source_exhausted: " << parser.source_exhausted() << "\n\n"; - - parser.finish_parse(decoder); // Indicates that this is the end - std::cout << "(3) done: " << std::boolalpha << parser.done() << ", source_exhausted: " << parser.source_exhausted() << "\n\n"; - - parser.check_done(); // Checks if there are any unconsumed - // non-whitespace characters in the input - // (there are) - } - catch (const ser_error& e) - { - std::cout << "(4) " << e.what() << std::endl; - } -} -``` - -Output: - -``` -(1) done: false, source_exhausted: true - -(2) done: true, source_exhausted: false - -(3) done: true, source_exhausted: false - -(4) Unexpected non-whitespace character after JSON text at line 1 and column 7 -``` - -#### nan, inf, and -inf substitition - -```c++ -int main() -{ - std::string s = R"( - { - "A" : "NaN", - "B" : "Infinity", - "C" : "-Infinity" - } - )"; - - json_options options; // Implements json_decode_options - options.nan_to_str("NaN") - .inf_to_str("Infinity"); - - json_parser parser(options); - jsoncons::json_decoder decoder; - try - { - parser.update(s); - parser.parse_some(decoder); - parser.finish_parse(decoder); - parser.check_done(); - } - catch (const ser_error& e) - { - std::cout << e.what() << std::endl; - } - - json j = decoder.get_result(); // performs move - if (j["A"].is()) - { - std::cout << "A: " << j["A"].as() << std::endl; - } - if (j["B"].is()) - { - std::cout << "B: " << j["B"].as() << std::endl; - } - if (j["C"].is()) - { - std::cout << "C: " << j["C"].as() << std::endl; - } -} -``` - -Output: - -``` -A: nan -B: inf -C: -inf -``` - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_reader.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_reader.md deleted file mode 100644 index ee7cad527f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_reader.md +++ /dev/null @@ -1,239 +0,0 @@ -### jsoncons::basic_json_reader - -```c++ -template< - class CharT, - class Src=jsoncons::stream_source, - class Allocator=std::allocator -> -class basic_json_reader -``` -`basic_json_reader` uses the incremental parser [basic_json_parser](json_parser.md) -to read arbitrarily large files in chunks. -A `basic_json_reader` can read a sequence of JSON texts from a stream, using `read_next()`, -which omits the check for unconsumed non-whitespace characters. - -`basic_json_reader` is noncopyable and nonmoveable. - -#### Header -```c++ -#include -``` - -Four specializations for common character types and result types are defined: - -Type |Definition ----------------------------|------------------------------ -json_reader |basic_json_reader -wjson_reader |basic_json_reader - -#### Member types - -Type |Definition ----------------------------|------------------------------ -char_type |CharT -source_type |Src -string_view_type | - -#### Constructors - - template - explicit basic_json_reader(Source&& source); // (1) - - template - basic_json_reader(Source&& source, - const basic_json_decode_options& options); // (2) - - template - basic_json_reader(Source&& source, - parse_error_handler& err_handler); // (3) - - template - basic_json_reader(Source&& source, - const basic_json_decode_options& options, - parse_error_handler& err_handler); // (4) - - template - basic_json_reader(Source&& source, - basic_json_content_handler& handler); // (5) - - template - basic_json_reader(Source&& source, - basic_json_content_handler& handler, - const basic_json_decode_options& options); // (6) - - template - basic_json_reader(Source&& source, - basic_json_content_handler& handler, - parse_error_handler& err_handler); // (7) - - template - basic_json_reader(Source&& source, - basic_json_content_handler& handler, - const basic_json_decode_options& options, - parse_error_handler& err_handler); // (8) - -Constructors (1)-(4) use a default [json_content_handler](json_content_handler.md) that discards the JSON parse events, and are for validation only. - -(1) Constructs a `basic_json_reader` that reads from a character sequence or stream `source`, uses default [options](json_decode_options.md) and a default [parse_error_handler](parse_error_handler.md). - -(2) Constructs a `basic_json_reader` that reads from a character sequence or stream `source`, -uses the specified [options](json_decode_options.md) -and a default [parse_error_handler](parse_error_handler.md). - -(3) Constructs a `basic_json_reader` that reads from a character sequence or stream `source`, -uses default [options](json_decode_options.md) -and a specified [parse_error_handler](parse_error_handler.md). - -(4) Constructs a `basic_json_reader` that reads from a character sequence or stream `source`, -uses the specified [options](json_decode_options.md) -and a specified [parse_error_handler](parse_error_handler.md). - -Constructors (5)-(8) take a user supplied [json_content_handler](json_content_handler.md) that receives JSON parse events, such as a [json_decoder](json_decoder). - -(5) Constructs a `basic_json_reader` that reads from a character sequence or stream `source`, -emits JSON parse events to the specified -[json_content_handler](json_content_handler.md), and uses default [options](json_decode_options.md) -and a default [parse_error_handler](parse_error_handler.md). - -(6) Constructs a `basic_json_reader` that reads from a character sequence or stream `source`, -emits JSON parse events to the specified [json_content_handler](json_content_handler.md) -and uses the specified [options](json_decode_options.md) -and a default [parse_error_handler](parse_error_handler.md). - -(7) Constructs a `basic_json_reader` that reads from a character sequence or stream `source`, -emits JSON parse events to the specified [json_content_handler](json_content_handler.md) -and uses default [options](json_decode_options.md) -and a specified [parse_error_handler](parse_error_handler.md). - -(8) Constructs a `basic_json_reader` that reads from a character sequence or stream `source`, -emits JSON parse events to the specified [json_content_handler](json_content_handler.md) and -uses the specified [options](json_decode_options.md) -and a specified [parse_error_handler](parse_error_handler.md). - -Note: It is the programmer's responsibility to ensure that `basic_json_reader` does not outlive any source, -content handler, and error handler passed in the constuctor, as `basic_json_reader` holds pointers to but does not own these resources. - -#### Parameters - -`source` - a value from which a `jsoncons::basic_string_view` is constructible, -or a value from which a `source_type` is constructible. In the case that a `jsoncons::basic_string_view` is constructible -from `source`, `source` is dispatched immediately to the parser. Otherwise, the `json_reader` reads from a `source_type` in chunks. - -#### Member functions - - bool eof() const -Returns `true` when there are no more JSON texts to be read from the stream, `false` otherwise - - void read(); // (1) - void read(std::error_code& ec); // (2) -Reads the next JSON text from the stream and reports JSON events to a [json_content_handler](json_content_handler.md), such as a [json_decoder](json_decoder.md). -Override (1) throws if parsing fails, or there are any unconsumed non-whitespace characters left in the input. -Override (2) sets `ec` to a [json_errc](jsoncons::json_errc.md) if parsing fails or if there are any unconsumed non-whitespace characters left in the input. - - void read_next() - void read_next(std::error_code& ec) -Reads the next JSON text from the stream and reports JSON events to a [json_content_handler](json_content_handler.md), such as a [json_decoder](json_decoder.md). -Override (1) throws [ser_error](ser_error.md) if parsing fails. -Override (2) sets `ec` to a [json_errc](jsoncons::json_errc.md) if parsing fails. - - void check_done(); // (1) - void check_done(std::error_code& ec); // (2) -Override (1) throws if there are any unconsumed non-whitespace characters in the input. -Override (2) sets `ec` to a [json_errc](jsoncons::json_errc.md) if there are any unconsumed non-whitespace characters left in the input. - - size_t buffer_length() const - - void buffer_length(size_t length) - - size_t line() const - - size_t column() const - -### Examples - -#### Parsing JSON text with exceptions -``` -std::string input = R"({"field1"{}})"; - -json_decoder decoder; -json_reader reader(input,decoder); - -try -{ - reader.read(); - json j = decoder.get_result(); -} -catch (const ser_error& e) -{ - std::cout << e.what() << std::endl; -} - -``` -Output: -``` -Expected name separator ':' at line 1 and column 10 -``` - -#### Parsing JSON text with error codes -``` -std::string input = R"({"field1":ru})"; -std::istringstream is(input); - -json_decoder decoder; -json_reader reader(is,decoder); - -std::error_code ec; -reader.read(ec); - -if (!ec) -{ - json j = decoder.get_result(); -} -else -{ - std::cerr << ec.message() - << " at line " << reader.line() - << " and column " << reader.column() << std::endl; -} -``` -Output: -``` -Expected value at line 1 and column 11 -``` - -#### Reading a sequence of JSON texts from a stream - -`jsoncons` supports reading a sequence of JSON texts, such as shown below (`json-texts.json`): -```json -{"a":1,"b":2,"c":3} -{"a":4,"b":5,"c":6} -{"a":7,"b":8,"c":9} -``` -This is the code that reads them: -```c++ -std::ifstream is("json-texts.json"); -if (!is.is_open()) -{ - throw std::runtime_error("Cannot open file"); -} - -json_decoder decoder; -json_reader reader(is,decoder); - -while (!reader.eof()) -{ - reader.read_next(); - if (!reader.eof()) - { - json val = decoder.get_result(); - std::cout << val << std::endl; - } -} -``` -Output: -```json -{"a":1,"b":2,"c":3} -{"a":4,"b":5,"c":6} -{"a":7,"b":8,"c":9} -``` diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_type_traits.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_type_traits.md deleted file mode 100644 index 602ed487f2..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/json_type_traits.md +++ /dev/null @@ -1,826 +0,0 @@ -### jsoncons::json_type_traits - -`json_type_traits` defines a compile time template based interface for accessing and modifying `basic_json` values. - -#### Header -```c++ -#include -``` - -The default definition provided by the `jsoncons` library is - -```c++ -template -struct json_type_traits -{ - typedef typename Json::allocator_type allocator_type; - - static constexpr bool is_compatible = false; - - static constexpr bool is(const Json&) - { - return false; - } - - static T as(const Json&); - - static Json to_json(const T&, allocator_type = allocator_type()); -}; -``` - -You can interact with a new type using `is`, `as`, construction and assignment by specializing `json_type_traits` in the `jsoncons` namespace. - -If you try to specialize `json_type_traits` for a type that is already -specialized in the jsoncons library, for example, a custom container -type that satisfies the conditions for a sequence container, you -may see a compile error "more than one partial specialization matches the template argument list". -For these situations `jsoncons` provides the traits class -```c++ -template -struct is_json_type_traits_declared : public false_type {}; -``` -which inherits from [false_type](http://www.cplusplus.com/reference/type_traits/false_type/). -This traits class may be specialized for a user-defined type with a [true_type](http://www.cplusplus.com/reference/type_traits/true_type/) value to -inform the `jsoncons` library that the type is already specialized. - -`JSONCONS_MEMBER_TRAITS_DECL` is a macro that simplifies the creation of the necessary boilerplate -from member data. If used, it must be placed outside any namespace blocks. - -`JSONCONS_GETTER_CTOR_TRAITS_DECL` is a macro that simplifies the creation of the necessary boilerplate -from getter functions and a constructor. If used, it must be placed outside any namespace blocks. - -### Specializations - -`T`|`j.is()`|`j.as()`|j is assignable from `T` ---------|-----------|--------------|--- -`Json`|`true`|self| -`Json::object`|`true` if `j.is_object()`, otherwise `false`|Compile-time error| -`Json::array`|`true` if `j.is_array()`, otherwise `false`|Compile-time error| -`bool`|`true` if `j.is_bool()`, otherwise `false`|as `bool`| -`null_type`|`true` if `j.is_null()`, otherwise `false`|`null_type()` value if j.is_null(), otherwise throws| -`const char_type*`|`true` if string, otherwise `false`|as `const char_type*`| -`char_type*`|`true` if `j.is_string()`, otherwise `false`|Compile-time error| -`integral types`|`true` if `j.is_int64()` or `j.is_uint64()` and value is in range, otherwise `false`|j numeric value cast to `T`| -`floating point types`|`true` if j.is_double() and value is in range, otherwise `false`|j numeric value cast to `T`| -`string`|`true` if j.is_string(), otherwise `false`|as string| -STL sequence container (other than string) e.g. std::vector|`true` if array and each value is assignable to a `Json` value, otherwise `false`|if array and each value is convertible to `value_type`, as container, otherwise throws| -STL associative container e.g. std::map|`true` if object and each `mapped_type` is assignable to `Json`, otherwise `false`|if object and each member value is convertible to `mapped_type`, as container| -`std::tuple`|`true` if `j.is_array()` and each array element is assignable to the corresponding `tuple` element, otherwise false|tuple with array elements converted to tuple elements| -`std::pair`|`true` if `j.is_array()` and `j.size()==2` and each array element is assignable to the corresponding pair element, otherwise false|pair with array elements converted to pair elements| - -### Examples - -[Convert from and to standard library sequence containers](#A1) -[Convert from and to standard library associative containers](#A2) -[Convert from and to std::tuple](#A3) -[Extend json_type_traits to support `boost::gregorian` dates.](#A4) -[Specialize json_type_traits to support a book class.](#A5) -[Using JSONCONS_MEMBER_TRAITS_DECL to generate the json_type_traits](#A6) -[A polymorphic example using JSONCONS_GETTER_CTOR_TRAITS_DECL to generate the json_type_traits](#A7) -[Specialize json_type_traits for a container type that the jsoncons library also supports](#A8) -[Convert JSON to/from boost matrix](#A9) - -
- -#### Convert from and to standard library sequence containers - -```c++ - std::vector v{1, 2, 3, 4}; - json j(v); - std::cout << "(1) "<< j << std::endl; - std::deque d = j.as>(); -``` -Output: -``` -(1) [1,2,3,4] -``` - -
- -#### Convert from and to standard library associative containers - -```c++ - std::map m{{"one",1},{"two",2},{"three",3}}; - json j(m); - std::cout << j << std::endl; - std::unordered_map um = j.as>(); -``` -Output: -``` -{"one":1,"three":3,"two":2} -``` - -
- -#### Convert from and to std::tuple - -```c++ - auto t = std::make_tuple(false,1,"foo"); - json j(t); - std::cout << j << std::endl; - auto t2 = j.as>(); -``` -Output: -``` -[false,1,"foo"] -``` - -
- -#### Extend json_type_traits to support `boost::gregorian` dates. - -```c++ -#include -#include "boost/datetime/gregorian/gregorian.hpp" - -namespace jsoncons -{ - template - struct json_type_traits - { - static const bool is_assignable = true; - - static bool is(const Json& val) noexcept - { - if (!val.is_string()) - { - return false; - } - std::string s = val.template as(); - try - { - boost::gregorian::from_simple_string(s); - return true; - } - catch (...) - { - return false; - } - } - - static boost::gregorian::date as(const Json& val) - { - std::string s = val.template as(); - return boost::gregorian::from_simple_string(s); - } - - static Json to_json(boost::gregorian::date val) - { - return Json(to_iso_extended_string(val)); - } - }; -} -``` -```c++ -namespace ns -{ - using jsoncons::json; - using boost::gregorian::date; - - json deal = json::parse(R"( - { - "Maturity":"2014-10-14", - "ObservationDates": ["2014-02-14","2014-02-21"] - } - )"); - - deal["ObservationDates"].push_back(date(2014,2,28)); - - date maturity = deal["Maturity"].as(); - std::cout << "Maturity: " << maturity << std::endl << std::endl; - - std::cout << "Observation dates: " << std::endl << std::endl; - - for (auto observation_date: deal["ObservationDates"].array_range()) - { - std::cout << observation_date << std::endl; - } - std::cout << std::endl; -} -``` -Output: -``` -Maturity: 2014-Oct-14 - -Observation dates: - -2014-Feb-14 -2014-Feb-21 -2014-Feb-28 -``` - -
- -#### Specialize json_type_traits to support a book class. - -```c++ -#include -#include -#include -#include - -namespace ns { - struct book - { - std::string author; - std::string title; - double price; - }; -} // namespace ns - -namespace jsoncons { - - template - struct json_type_traits - { - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - return j.is_object() && j.contains("author") && - j.contains("title") && j.contains("price"); - } - static ns::book as(const Json& j) - { - ns::book val; - val.author = j.at("author").template as(); - val.title = j.at("title").template as(); - val.price = j.at("price").template as(); - return val; - } - static Json to_json(const ns::book& val, - allocator_type allocator=allocator_type()) - { - Json j(allocator); - j.try_emplace("author", val.author); - j.try_emplace("title", val.title); - j.try_emplace("price", val.price); - return j; - } - }; -} // namespace jsoncons -``` - -To save typing and enhance readability, the jsoncons library defines macros, -so we could have written - -```c++ -JSONCONS_MEMBER_TRAITS_DECL(ns::book, author, title, price) -``` - -which expands to the code above. - -```c++ -using namespace jsoncons; // for convenience - -int main() -{ - const std::string s = R"( - [ - { - "author" : "Haruki Murakami", - "title" : "Kafka on the Shore", - "price" : 25.17 - }, - { - "author" : "Charles Bukowski", - "title" : "Pulp", - "price" : 22.48 - } - ] - )"; - - std::vector book_list = decode_json>(s); - - std::cout << "(1)\n"; - for (const auto& item : book_list) - { - std::cout << item.author << ", " - << item.title << ", " - << item.price << "\n"; - } - - std::cout << "\n(2)\n"; - encode_json(book_list, std::cout, indenting::indent); - std::cout << "\n\n"; -} -``` -Output: -``` -(1) -Haruki Murakami, Kafka on the Shore, 25.17 -Charles Bukowski, Pulp, 22.48 - -(2) -[ - { - "author": "Haruki Murakami", - "price": 25.17, - "title": "Kafka on the Shore" - }, - { - "author": "Charles Bukowski", - "price": 22.48, - "title": "Pulp" - } -] -``` - -
- -#### Using JSONCONS_MEMBER_TRAITS_DECL to generate the json_type_traits - -`JSONCONS_MEMBER_TRAITS_DECL` is a macro that can be used to generate the `json_type_traits` boilerplate -for your own types. - -```c++ -#include -#include -#include - -using namespace jsoncons; - -namespace ns { - - struct reputon - { - std::string rater; - std::string assertion; - std::string rated; - double rating; - - friend bool operator==(const reputon& lhs, const reputon& rhs) - { - return lhs.rater == rhs.rater && lhs.assertion == rhs.assertion && - lhs.rated == rhs.rated && lhs.rating == rhs.rating; - } - - friend bool operator!=(const reputon& lhs, const reputon& rhs) - { - return !(lhs == rhs); - }; - }; - - class reputation_object - { - std::string application; - std::vector reputons; - - // Make json_type_traits specializations friends to give accesses to private members - JSONCONS_TYPE_TRAITS_FRIEND; - public: - reputation_object() - { - } - reputation_object(const std::string& application, const std::vector& reputons) - : application(application), reputons(reputons) - {} - - friend bool operator==(const reputation_object& lhs, const reputation_object& rhs) - { - return (lhs.application == rhs.application) && (lhs.reputons == rhs.reputons); - } - - friend bool operator!=(const reputation_object& lhs, const reputation_object& rhs) - { - return !(lhs == rhs); - }; - }; - - -} // namespace ns - -// Declare the traits. Specify which data members need to be serialized. -JSONCONS_MEMBER_TRAITS_DECL(ns::reputon, rater, assertion, rated, rating) -JSONCONS_MEMBER_TRAITS_DECL(ns::reputation_object, application, reputons) - -int main() -{ - ns::reputation_object val("hiking", { ns::reputon{"HikingAsylum.example.com","strong-hiker","Marilyn C",0.90} }); - - std::string s; - encode_json(val, s, indenting::indent); - std::cout << s << "\n"; - - auto val2 = decode_json(s); - - assert(val2 == val); -} -``` -Output: -``` -{ - "application": "hiking", - "reputons": [ - { - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rater": "HikingAsylum.example.com", - "rating": 0.9 - } - ] -} -``` - -
- -#### A polymorphic example using JSONCONS_GETTER_CTOR_TRAITS_DECL to generate the json_type_traits - -`JSONCONS_GETTER_CTOR_TRAITS_DECL` is a macro that can be used to generate the `json_type_traits` boilerplate -from getter functions and a constructor. - -```c++ -#include -#include -#include -#include - -using namespace jsoncons; - -namespace ns { - -class Employee -{ - std::string firstName_; - std::string lastName_; -public: - Employee(const std::string& firstName, const std::string& lastName) - : firstName_(firstName), lastName_(lastName) - { - } - virtual ~Employee() = default; - - virtual double calculatePay() const = 0; - - const std::string& firstName() const {return firstName_;} - const std::string& lastName() const {return lastName_;} -}; - -class HourlyEmployee : public Employee -{ - double wage_; - unsigned hours_; -public: - HourlyEmployee(const std::string& firstName, const std::string& lastName, - double wage, unsigned hours) - : Employee(firstName, lastName), - wage_(wage), hours_(hours) - { - } - HourlyEmployee(const HourlyEmployee&) = default; - HourlyEmployee(HourlyEmployee&&) = default; - HourlyEmployee& operator=(const HourlyEmployee&) = default; - HourlyEmployee& operator=(HourlyEmployee&&) = default; - - double wage() const {return wage_;} - - unsigned hours() const {return hours_;} - - double calculatePay() const override - { - return wage_*hours_; - } -}; - -class CommissionedEmployee : public Employee -{ - double baseSalary_; - double commission_; - unsigned sales_; -public: - CommissionedEmployee(const std::string& firstName, const std::string& lastName, - double baseSalary, double commission, unsigned sales) - : Employee(firstName, lastName), - baseSalary_(baseSalary), commission_(commission), sales_(sales) - { - } - CommissionedEmployee(const CommissionedEmployee&) = default; - CommissionedEmployee(CommissionedEmployee&&) = default; - CommissionedEmployee& operator=(const CommissionedEmployee&) = default; - CommissionedEmployee& operator=(CommissionedEmployee&&) = default; - - double baseSalary() const - { - return baseSalary_; - } - - double commission() const - { - return commission_; - } - - unsigned sales() const - { - return sales_; - } - - double calculatePay() const override - { - return baseSalary_ + commission_*sales_; - } -}; -} // ns - -JSONCONS_GETTER_CTOR_TRAITS_DECL(ns::HourlyEmployee, firstName, lastName, wage, hours) -JSONCONS_GETTER_CTOR_TRAITS_DECL(ns::CommissionedEmployee, firstName, lastName, baseSalary, commission, sales) - -namespace jsoncons { - -template -struct json_type_traits> -{ - static bool is(const Json& j) noexcept - { - return j.is() || j.is(); - } - static std::shared_ptr as(const Json& j) - { - if (j.at("type").as() == "Hourly") - { - return std::make_shared(j.as()); - } - else if (j.at("type").as() == "Commissioned") - { - return std::make_shared(j.as()); - } - else - { - throw std::runtime_error("Not an employee"); - } - } - static Json to_json(const std::shared_ptr& ptr) - { - if (ns::HourlyEmployee* p = dynamic_cast(ptr.get())) - { - Json j(*p); - j.try_emplace("type","Hourly"); - return j; - } - else if (ns::CommissionedEmployee* p = dynamic_cast(ptr.get())) - { - Json j(*p); - j.try_emplace("type","Commissioned"); - return j; - } - else - { - throw std::runtime_error("Not an employee"); - } - } -}; - -} // jsoncons - -int main() -{ - std::vector> v; - - v.push_back(std::make_shared("John", "Smith", 40.0, 1000)); - v.push_back(std::make_shared("Jane", "Doe", 30000, 0.25, 1000)); - - json j(v); - std::cout << pretty_print(j) << "\n\n"; - - assert(j[0].is()); - assert(!j[0].is()); - assert(!j[1].is()); - assert(j[1].is()); - - - for (size_t i = 0; i < j.size(); ++i) - { - auto p = j[i].as>(); - assert(p->firstName() == v[i]->firstName()); - assert(p->lastName() == v[i]->lastName()); - assert(p->calculatePay() == v[i]->calculatePay()); - } -} -``` -Output: -``` -[ - { - "firstName": "John", - "hours": 1000, - "lastName": "Smith", - "type": "Hourly", - "wage": 40.0 - }, - { - "baseSalary": 30000.0, - "commission": 0.25, - "firstName": "Jane", - "lastName": "Doe", - "sales": 1000, - "type": "Commissioned" - } -] -``` - -
- -#### Specialize json_type_traits for a container type that the jsoncons library also supports - -```c++ -#include -#include -#include -#include - -//own vector will always be of an even length -struct own_vector : std::vector { using std::vector::vector; }; - -namespace jsoncons { - -template -struct json_type_traits -{ - static bool is(const Json& j) noexcept - { - return j.is_object() && j.size() % 2 == 0; - } - static own_vector as(const Json& j) - { - own_vector v; - for (auto& item : j.object_range()) - { - std::string s(item.key()); - v.push_back(std::strtol(s.c_str(),nullptr,10)); - v.push_back(item.value().template as()); - } - return v; - } - static Json to_json(const own_vector& val){ - Json j; - for(size_t i=0;i -struct is_json_type_traits_declared : public std::true_type -{}; -} // jsoncons - -using jsoncons::json; - -int main() -{ - json j = json::object{ {"1",2},{"3",4} }; - assert(j.is()); - auto v = j.as(); - json j2 = v; - - std::cout << j2 << "\n"; -} -``` -Output: -``` -{"1":2,"3":4} -``` - -
- -#### Convert JSON to/from boost matrix - -```c++ -#include -#include - -namespace jsoncons { - - template - struct json_type_traits> - { - static bool is(const Json& val) noexcept - { - if (!val.is_array()) - { - return false; - } - if (val.size() > 0) - { - size_t n = val[0].size(); - for (const auto& a: val.array_range()) - { - if (!(a.is_array() && a.size() == n)) - { - return false; - } - for (auto x: a.array_range()) - { - if (!x.template is()) - { - return false; - } - } - } - } - return true; - } - - static boost::numeric::ublas::matrix as(const Json& val) - { - if (val.is_array() && val.size() > 0) - { - size_t m = val.size(); - size_t n = 0; - for (const auto& a : val.array_range()) - { - if (a.size() > n) - { - n = a.size(); - } - } - - boost::numeric::ublas::matrix A(m,n,T()); - for (size_t i = 0; i < m; ++i) - { - const auto& a = val[i]; - for (size_t j = 0; j < a.size(); ++j) - { - A(i,j) = a[j].template as(); - } - } - return A; - } - else - { - boost::numeric::ublas::matrix A; - return A; - } - } - - static Json to_json(const boost::numeric::ublas::matrix& val) - { - Json a = Json::template make_array<2>(val.size1(), val.size2(), T()); - for (size_t i = 0; i < val.size1(); ++i) - { - for (size_t j = 0; j < val.size1(); ++j) - { - a[i][j] = val(i,j); - } - } - return a; - } - }; -} // jsoncons - -using namespace jsoncons; -using boost::numeric::ublas::matrix; - -int main() -{ - matrix A(2, 2); - A(0, 0) = 1.1; - A(0, 1) = 2.1; - A(1, 0) = 3.1; - A(1, 1) = 4.1; - - json a = A; - - std::cout << "(1) " << std::boolalpha << a.is>() << "\n\n"; - - std::cout << "(2) " << std::boolalpha << a.is>() << "\n\n"; - - std::cout << "(3) \n\n" << pretty_print(a) << "\n\n"; - - matrix B = a.as>(); - - std::cout << "(4) \n\n"; - for (size_t i = 0; i < B.size1(); ++i) - { - for (size_t j = 0; j < B.size2(); ++j) - { - if (j > 0) - { - std::cout << ","; - } - std::cout << B(i, j); - } - std::cout << "\n"; - } - std::cout << "\n\n"; -} -``` -Output: -``` -(1) true - -(2) false - -(3) - -[ - [1.1,2.1], - [3.1,4.1] -] - -(4) - -1.1,2.1 -3.1,4.1 -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/apply_patch.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/apply_patch.md deleted file mode 100644 index 098bda9985..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/apply_patch.md +++ /dev/null @@ -1,100 +0,0 @@ -### jsoncons::jsonpatch::apply_patch - -Applies a patch to a `json` document. - -#### Header -```c++ -#include - -template -void apply_patch(Json& target, const Json& patch); // (1) - -template -void apply_patch(Json& target, const Json& patch, std::error_code& ec); // (2) -``` - -#### Return value - -None - -#### Exceptions - -(1) Throws a [jsonpatch_error](jsonpatch_error.md) if `apply_patch` fails. - -(2) Sets the `std::error_code&` to the [jsonpatch_error_category](jsonpatch_errc.md) if `apply_patch` fails. - -### Examples - -#### Apply a JSON Patch with two add operations - -```c++ -#include -#include - -using namespace jsoncons::literals; -namespace jp = jsoncons::jsonpatch; - -int main() -{ - jsoncons::json target = R"( - { "foo": "bar"} - )"_json; - - jsoncons::json patch = R"( - [ - { "op": "add", "path": "/baz", "value": "qux" }, - { "op": "add", "path": "/foo", "value": [ "bar", "baz" ] } - ] - )"_json; - - std::error_code ec; - jp::apply_patch(target,patch,ec); - - std::cout << pretty_print(target) << std::endl; -} -``` -Output: -``` -{ - "baz": "qux", - "foo": ["bar","baz"] -} -``` - -#### Apply a JSON Patch with three add operations, the last one fails - -```c++ -#include -#include - -using namespace jsoncons::literals; -namespace jp = jsoncons::jsonpatch; - -int main() -{ - jsoncons::json target = R"( - { "foo": "bar"} - )"_json; - - jsoncons::json patch = R"( - [ - { "op": "add", "path": "/baz", "value": "qux" }, - { "op": "add", "path": "/foo", "value": [ "bar", "baz" ] }, - { "op": "add", "path": "/baz/bat", "value": "qux" } // nonexistent target - ] - )"_json; - - std::error_code ec; - jp::apply_patch(target, patch, ec); - - std::cout << "(1) " << ec.message() << std::endl; - std::cout << "(2) " << target << std::endl; -} -``` -Output: -``` -(1) JSON Patch add operation failed -(2) {"foo":"bar"} -``` -Note that all JSON Patch operations have been rolled back, and target is in its original state. - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/diagrams/jsonpatch_error.png b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/diagrams/jsonpatch_error.png deleted file mode 100644 index 6bfb6391bec85550ff6e13561b094615e5d135cc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9022 zcmeHtcT^K$_a*`&NC%}Fx+v1SgchYs6A-1>&^rW>(50zBsM4!I0O`F1uz>VxC{h9{ zpoGu`>97<1zO(22_W%87my?r|8QytsnLGDB_qp@xv5p$$b(ZS{1O${C>dN{A1cV^q z_j^(j;M;`Xou7b!gFr+1!IS3}o4Ib4hH7nnG0m0*hRF}JnC^0Bki3m&dQrgG%Kq}v z$oXSix4J8n>Rjv)I#Nf$hXGa1G@Qg)R@itl6>^5SSHd)I^hCu?PU<2dcc(quihJH& z3-AP6TW zR*h$QQ}FcNMihZTi12=E%QM0v5l|=BN(d>0BOxq&lMi84*B^t)oyzgiuk%8cSr<2e5nEnbwTNWhUl|=s0i?>E% zgc%5^V>uHcja5>EB*}Pp8k=?;33@_#3M)?R%=Llm8U0%FNP(Kh;~Sc)1n49R!JA41 zfNIUX1n{q4ie61vfwRiM!O;+HKrd%P^Lpm}yJW9z21u>I$%Q30q>wO`(P<_KwJBUe zltoL8-mr9xrtB3#teJL!rc?ym?FV`S%vZG3#2TMaTmg)jWg%L#)=c14jlep-UEgOW zl%%yvsvv_6*pC z_IOt5cKN_RtzOH$Q#Zc%OVR&1Zkn;sT)eyoj}u@vVSfS`{lG9S!F z?hn~-zjIoFSriD?6o~cd%If>0$agYCZ5Q4dRxES{p9VeGRX8p-ZFEe!Jf18!aNk;J zyZB`{l-8Cj>wl78au!KOzYtBydbXV{?cs-;sS)0=K%MC1D{NBTy8jKwCJ%3veZEs5 z>%V1I(Isfw+&Cu_uveTd?tnnS57uil4tzIfDvcW!)~6~k`&cxlP$_2NXm>d{AHlA; z#UNnxP0DR<#cy+NYH+`2t2C_ats3Y3>yQF+z!aeZFLGjz#|X$96(I>w9|GhexR)+u z?v0zS@|kaT^QpgD=lT6%i~qKTv~6!(+JsSqef*`dkmz+AmNa3@EylxggVLWQ{BG4| zZC^GQJLk2tB+3o*Gj58qcw>BL%(v?L*nGaIPKQN`YBIO2Scp>0b#?HHF8PefUwG(m zzoNerl*1@w=IB-)InQ5rT2An8O;5-A(SrgO;KoW?q(&1E-=hKCxtfWnl`$UfrieTf zGLAVvJw#QRwzLQQ{&~C+&h+Lc|I?$ZohevS$}pzawINQ2g)wa24x>4{#_=NAb>mbSEGvcm5nZFhfs?oLgrqHCeikZ0paW zn!QIRJj;+J?%|KCYH0^t z(4G*oU*ta5=(Kfly6x}37~nvp#&2F7L4Yix8f?509!Q zGP{HJ_1@_TnoOlP%&E;LXj!~*44wXGW`4l+Dt z&O#Ixa!LpOaXtJ*F5DRXYeaf!!xkNZ(q?bjR|Em~7Ti2Z5#CF`HVq@aA6%~#GJWU4 z$}b&VUswZE%mH@#(ep&Zxs83O_9B}9q$hq+)^Ee;eOb)Bbh|mX+mirzNp{F1)^~kF z!tXa)f`Qok;oi2&@P35LUcLEgD@S570h0T+`Fb>@n+)axCczGUT)z-P=qL$IF1uKs zW~FSzxmNUlG#u#ZIG8rwa!@?YMD6ynvMP9EL+$|&#S>^bBO%S*^G=#QKGRX=$fZ77 zqI`j)`X)}@0;upgf?B35bFbbR49#ARW_91WG)MW3a339yg>G$i9$M6NKkM3ilWJ6k zDg-hw<;^^Ir2;BKLotwV4|-p!9^P+!b9OJ_VCq>+VUk-~%Th>or5gI0){6^UFdd}i z5h>)`C7Bi2H9qnlJn^8m1XG|Kfn~aJ8+(nI0-sr!DMG(JTvw&8XCsoPP;FGoK+t~H zU8Z4jCBz7LM`%N#fiqQXPE4wbA#^g1Kkr6c`i3%~!Hi_=7!c*Sz-E^!%5C4|gLq;cX=KSGD>egO5r% ztRf(BDx<2pB;2nv;EvNuT{B8-sMQw`X25G=sdtI4kit|resawi5-do>L0si|aGXoP zr{U8fT;%H@%?DU&p)xZmq=28WjQj%v&sOdz?k*7yT@CjG+!;p>ls)F~)L66nf@`fm!25}AX@H(Hq1k*6@+8ki zd}k%K)Y8ATQUdpn_i-QBvs+hhr$Wr}GKN=y+9o0e6jdnky2(niuhoZp)uz+~@Q%a9 zz>7fnemKQftMz(WASr(IVg-_-7KA}Y$dq`+GQH|I*n8#WrA0#0eVpj2;zh$~^_Qn) zfX#G?cL}eMLI?;uX_;PC#$+ID-=C;>g!}kQ5c;3bV5rcvly`1EARq+Hx3;-385mycBLd>?%gr`Ccuj_(P!OGiuXaHEwTFqr@V0R_g?GB~}}Pp~J2 zZ@L7K-14?2z6&`Defi**6pyMw9N+tI0qHemfPCGLMEl)ZsS z;q;*3`b@;;d<$wj^zu~U=XS^;%6)UL>B7Ny4XCW^7yG4(ZO6ZVq1eyY)D-Ack!=6j0i-#s|gXg(Io4AC~Gh0ioxzr8BzV^wx z86s-AZy`Go?2RGmU5~N@&Kd0iC6_S6nc|pcxA}#&i6Wow53tD-zXH*JCPjo@6nR~q z4pwY^tqDn*hdP3b6Xg*66}P*NJ`2N%cB(vraYVZ?lbze{Z9VF*ut}G(oXMK&Deo)D$U@%-9eej(xX7y@ z)#g6JCPh!2i2eoLom%*TbIawcOTWv&i_m-72+!S7cLzs%h*pF^f4z~*Po?qh{C4?Hc!({>ZU@)LAhEn_TWZ^Z{YQ7uoH36McFGh4=@eAB zbW#0iuDzAm9L(b)&daetXP&_6qDpMPLg(9a{y~_!fZhMx^!B(AYN87J@_5y1x(hb^ z8t8oZAJJU%&Oy*($BzyW=g8!ihc%3bBR`%JDGN2?sYy zsJT>wG_g%q^{GRp-_aJrFn}L@>4hC@wloTz53Bl~H7TB3EVi>vFXQ~rD|#){*|;+? zWrLA>2LR}t975GLwev=Of_PsCwwez)zy(hq)UsK=Ute@v@LhZ0S7hp&<;9Nd&KUL8 zy50$!g$H%+SFvw5RmU70o#_3xu+b^MHXCn|>NMJ!L1k-FnsdVjHU$q7J^8BZYcvBW zdS2`sXJmkV;Kv;e9w{S2TWkG#lJc<9-6TQysp4q~+g}?jVThX!uJZlvJ^DzQ11oTx z6DUysB}N{%`+2M8;G_~~Ew6w(=|u=5*`U%k73jDV5l}A-A;VX^mUVEXzd!4pi&s0} z&ds2QL5aGal`qvvCl{|!U&al6zX5m<->G|@J6_hkel#?n3SOY!;^A&F) zfPD^^OwRxABPnmthX%l#v_3yN=R_3L+qmVYUsl07y<8C+@y743KK$V6n`+YVL`<1x zu6<(!Rs`uIo$@f(9PnPQfIy?2IW=zAc)WI$=^fsUI(AuT2EdKehP+Wpq!@S*_eUtt zd?VhCs`{uTN7;(L2x^5jw$L-LLwuz$x(}6BeF$v)3fQ>eKw}Ob)Vrd^hVYfI>`OP} zu%(QIn*ksK14CU z2zmpJu)ojGVbP)8joAi;3w&2TV-Z+&@Ed5(ll#mhG%mON-}~gBZ#bm%CCxzeg5Z|@ zB14<_274$?aO>O02q8&y(quhV((C=T^j)sX$~fj%D!so_`3%m2B-PUqjMnll(*pjh zgE#$$1Y26A7oT@$#|LhA`X{z)T2cUk+DKc1UP(9U?oe6z9>eaJE#>A7birPiXTKB; zVv3OBRdT>jfUhhAR52@Bu!(ya}^ z`b)zpS*4yo$2a;e)0Mk??ZKyi+OX(Slu`^;n??$6lZ7czNn}GWI>umHfIuTvp3{Mc z3Y25HvNJP+hnov+=v;;1InaI250`TAeaB0UnhfO`=j+;Q2cSnpnA5Vj=!%;;n%MsWk* zMK>{Y;wc2%)R48(1;4DJBOGkt=*z_;Ox9FVd+}xH>HN|{sp7fQooj#C0#=fyx51+g zww8Svd*faU9?L2xNzTLS_L7s2+A5~O9xtVI<#r=OJ%)1LRyLj0+4h-ii-NA@E*|3X zP=fp)BbSB#9G>5g%TQ5uT%!HYk5`35u~3)Qz15-aPszVQ+ntAbvH<3a)tDTJ806rv z9hZ@okdS!79Nj@SMbJM&Ex(^=3bei{DzXa}wBlYs@1N$BDBY@)5fDFET24)Q81dCteO^JVqy_2aiW-_|7c-){1d#14y5z=ir}vjj2)U zeQ%xlqK!_tL(!Y;6LKB`Ke34z)#0vdpXz848qEC8PxpEO-!`ARb*L|t_iFjMNayf> zk||&F1r@b-CQGw71uhL`&=u{K2nP?0QUCrqf@@VV_+_Mr_B!2ckrl0(sj>9;=sHH` zH{8+5S=(w|raG)k2p$eT9&GNXk;f7sv%@h9I##k-2GQ0a0;7r7hnMQ^a z`ErVGXMnA!@O+Yd1Kl(u3(H7OlviXO?rZvchdv`m@QG6gR9o$SSRhC|chUco=f&Ca zMfbzSYsDgIo^mTAYROTlYC67g_I3+x9gV}=jC!Qje#MqyTgXo^Mj-+9_=$6yW9V;>ra@|>_=|U@D1;8Xv3STi$ zZm$hW+9l)l%zK<)5me`sUjr{U>ceV$ohz~^zgP2}DEC41yh*!Y#m~)Ihx~(iN?TxZ(uPE1?T1F7n6*0@%rgCu+a27EsxlZX8BSY@~2k7UtLUzL;v!W?eMHnfa`yQ}+ z`I4Z`UWnf!b{ZCWQhJL&1M$&f__V~|JR7k?Dd^GFk!p7|piS*&D1+uqQEt7{W!s0K zF4ky@gQQnJYURL+|2k>kI3EgWgIa&`A+XNDi8?I|h?p??B2D&woDL#vt zQ~^JD7nZ8Mk=)q;tM*4dT4U^2nMRx&_>(wjwl1TkGkK(A;A?V}%2+H+j9JA2{8}sK zVE3&tmAo*m&oGoRSRPe)?SWB=!5I`w}*&c%Hu#3SP%+A7Y$# zO?I?eTMb3OWHcduLC-^s|1&uJwwxhb$Pdyx%;sM|l`_uBcfOs6z-ttC zY226AlsUdU3xMA)!H!!W%`aU{2XFc0v)1WloJxmzlbuy`B?v}>tl-WtSya9{}wkglPLkDwF$*%?=f3a|GUg}*Ja3z9q z#C^>`WiPN&lsm|-68luWwezi6gK<>H-U%F4`V2;~mV=JPCboX^%Jr_a0;{^t6@L$d z%pi-F_|i0X6O`G9La2MydwPo{^(TMN+jXSQ6c2wK-D^5<(7-L1D3;Ut-UQf^!}p;@ z*Bevqs&ubUFKd>^Pv?&8MY%=iKf%5H+|I`E!=O`{`n(zeZcLIAh`*^IFL#ijRZEE>dAC8Xzp`$mEv5C9O57nkzl;3#e&Q|=ZX za|yRWdRZXUwl5?^ZfV7cPW%-s&g-z}iAPdhD}>B01oClYUp{yOuui!}fe(Ob894*p zb$ED`Ob)HQvM<7`PSpJo#zc=fDYlYVu=VDp?!$vxO`*g(VBnF#K}m595Dfc&N^oB` zg5=LUEM4$2!dcA8YnJQ7^SY9l14n@bM%>2B8h^r~Xr}N^gJP1uZ57Z(T?hEg>$G}! zj^;17!vr)7qIdQGHE%$_5(OMby}XMjS^n}k6hN#n-6;Rp&I3KpL*V$K@{PY2p6jAPOj^JF9rnA0L(cziOTwa*B2xLOgatiReT$SU-J?O7TO3d-~X-;RwUNe2@hG@A|tu05_?u{2yrq#5Mqr*;Akn`e)4*(i;F$yutMr zZ`$8AqQLCH=5aCA|2O}yOaK4f|GyIa1uWs3=|XM5EUXnj!^F?!7K8q57+czp<;lA* zw0cdk1ias$s(6OM47|e$+H?sARqf+xhmu|3?qm;Z8R->Kc2{c%KX~J`-|T zmI7FeCE)2jF@NwwTCG3_P3za?it;5iK-c;F(vgPmuYt3Xf~1fS99Ohr9_W#lzbA`~ zy)4#93F5W+6iMdZ9ZlI~`=6nve-<`F(k0Eb6W){!@S3u)wa5Ij-s_Wm(!g9ZbgbTP zC`VwyTO+vU{YAU*>9F`%E6^mOPPb5Fm!V5t!Bz0y1omwPL6a3=s=3&V0Wki_o_DTX zK>eHW#8w6&v$fG28MN~=3MJI@Ic74!JLktho3;%MhxWp?)Qp}r2aDbVTADxSiKmMD zh16}5F4GjamLC4hT%B|Bwk-^5#%;F#bHn3{bNRuSY`6CPl?ez4kMZvU5T0ugpeOR% zX`8GGH??9S`H|m&DXaVO-s+-A|Lxe*xh9wMAAGUEtc)?Zjhs=a^$q{Cb_~iNHQu`R zgP&J5PIK|k>qT*oMLEkBCbFs;0~7$0f7S}JymA^$<#Yczoawfl#Pms%GFD&@XjgVr zA5?n=sthOga?R^nOYv;9Wbw`-25|nf7XZpA{w(J@zazpWK}x}MMeFZWMl~5Ci!O|{ zJ(7MUYDTb%z6>Y2)`7hb-!D@nFPx`4kv8vW>Z*4D4>t27)on=|m}cFQMW$=b{v5b2 zY>vX9r@>nbG9G0s$$qQ;lAyWm%C6j#QeX94hL!?V@5(Py#Ou6a*s|>Vo0M|len=(8 z`Hde;Y&;`EUG<-7EXWxIJWmg|{Y_ik-9Y^t0Qqgo)E><3>*p*jAv@RQ9spfVn(0=m z4fx3unD|8OZaap@cm2-4f!#X z7sIR64BFF*ao6tuA-Fo2m8+A9@Y{vn5wMIXw!~%Fu#lknfKkhOp5_{+#7Uo2lL51F< zx%ICmS$P7>NYpgp|Asa+BH934GGTjxXD;#FAQccv-yAh`|5Lz_WM}NJ(XAfek diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/diagrams/jsonpatch_error.xml b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/diagrams/jsonpatch_error.xml deleted file mode 100644 index 4d39d6a91e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/diagrams/jsonpatch_error.xml +++ /dev/null @@ -1 +0,0 @@ -7Vhdb9owFP01PHZK4kDpI6Uf2xDVOibRPlVWfJt4c2zmGAj79bOJ8x1Kq6KxaSAk7GP72j733HtJemgcp7cSL6KpIMB6nkPSHrrqeZ7reK7+McgmQ5DvZUAoKbGTSmBGf0G+0qJLSiCpTVRCMEUXdTAQnEOgahiWUqzr054Fq++6wCG0gFmAWRudU6KiDB32nRL/CDSM8p1dx47EOJ9sgSTCRKwrELruobEUQmWtOB0DM+TlvGTrbnaMFgeTwNVrFlzffUv55H4WeeOfEzWfxo/p5zNrZYXZ0l44UXr+SH8hDWChqOD2+GqTcyLFkhMwZp0eulxHVMFsgQMzutYq0FikYqZ7rm62j5nvCVJBWoHssW9BxKDkRk+xowPLoJWQmzO6Lh1S0B5VnOFbDFsNhIXlkibdsEy9gbXz3azJJVc0hifQ2pNHZ85z/jbqhi3qWiQBJyMTuboXMJwkNKjzAilVD5ZC03407Q9927tKK0NXm7zD9eEfcgOmU1lluuWybS9ft9MHiVjKAPZrRGEZgtofgUBqeajt0YrH+h0OyzEJDCu6qmevLi/aHb4Iqm9WCMav68VDDR1k17aLqsmmYeeiobthw05GS8vOVlLFpV+lsnA1jbzJ3QP+CuHZ/H4Cqwm8lNaSTaIgPlx8toKxQy474xP5R4zPTua8fzE+G8GzN2Bf0szegD0/xee74xO1VPY9EXyBVRAdIjKfKWNjwbQZsxbB9qPxREnxAyojaIAuEDlMLPcbtXZ47FD2O0l+Ouj/uoNmP//YjPX/2+SH2slvt6ZOye9dyW9wUtkelR31LzBqPG4Wj59vlZnvNgy5f1Zn7afUTzwCSRXm2kueQygOJY5b4tMJXdXlVq+bXHBoFFkLYUZDbjSrlQEavzTlgQaYjexATAkx23QWl3r5OUB9ab44KBxQLTCoQ0ne2wuM7pavcjIPli/E0PVv \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/from_diff.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/from_diff.md deleted file mode 100644 index edebaed1d3..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/from_diff.md +++ /dev/null @@ -1,72 +0,0 @@ -### jsoncons::jsonpatch::from_diff - -Create a JSON Patch from a diff of two json documents. - -#### Header -```c++ -#include - -template -Json from_diff(const Json& source, const Json& target) -``` - -#### Return value - -Returns a JSON Patch. - -### Examples - -#### Create a JSON Patch - -```c++ -#include -#include - -using namespace jsoncons::literals; -namespace jp = jsoncons::jsonpatch; - -int main() -{ - jsoncons::json source = R"( - {"/": 9, "foo": "bar"} - )"_json; - - jsoncons::json target = R"( - { "baz":"qux", "foo": [ "bar", "baz" ]} - )"_json; - - auto patch = jp::from_diff(source, target); - - std::error_code ec; - jp::apply_patch(source, patch, ec); - - std::cout << "(1) " << pretty_print(patch) << std::endl; - std::cout << "(2) " << pretty_print(source) << std::endl; -} -``` -Output: -``` -(1) -[ - { - "op": "remove", - "path": "/~1" - }, - { - "op": "replace", - "path": "/foo", - "value": ["bar","baz"] - }, - { - "op": "add", - "path": "/baz", - "value": "qux" - } -] -(2) -{ - "baz": "qux", - "foo": ["bar","baz"] -} -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/jsonpatch.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/jsonpatch.md deleted file mode 100644 index ab528d01a1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/jsonpatch.md +++ /dev/null @@ -1,87 +0,0 @@ -### jsonpatch extension - -The jsonpatch extension implements the IETF standard [JavaScript Object Notation (JSON) Patch](https://tools.ietf.org/html/rfc6902) - - - - - - - - - - -
apply_patchApply JSON Patch operations to a JSON document.
from_diffCreate a JSON patch from a diff of two JSON documents.
- -The JSON Patch IETF standard requires that the JSON Patch method is atomic, so that if any JSON Patch operation results in an error, the target document is unchanged. -The patch function implements this requirement by generating the inverse commands and building an undo stack, which is executed if any part of the patch fails. - -### Examples - -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - // Apply a JSON Patch - - json doc = R"( - { "foo": "bar"} - )"_json; - - json doc2 = doc; - - json patch = R"( - [ - { "op": "add", "path": "/baz", "value": "qux" }, - { "op": "add", "path": "/foo", "value": [ "bar", "baz" ] } - ] - )"_json; - - std::error_code ec; - jsonpatch::apply_patch(doc, patch, ec); - - std::cout << "(1)\n" << pretty_print(doc) << std::endl; - - // Create a JSON Patch from two JSON documents - - auto patch2 = jsonpatch::from_diff(doc2,doc); - - std::cout << "(2)\n" << pretty_print(patch2) << std::endl; - - jsonpatch::apply_patch(doc2, patch2, ec); - - std::cout << "(3)\n" << pretty_print(doc2) << std::endl; -} -``` -Output: -``` -(1) -{ - "baz": "qux", - "foo": ["bar","baz"] -} -(2) -[ - { - "op": "replace", - "path": "/foo", - "value": ["bar","baz"] - }, - { - "op": "add", - "path": "/baz", - "value": "qux" - } -] -(3) -{ - "baz": "qux", - "foo": ["bar","baz"] -} -``` - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/jsonpatch_errc.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/jsonpatch_errc.md deleted file mode 100644 index 01e1eb8f9e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/jsonpatch_errc.md +++ /dev/null @@ -1,25 +0,0 @@ -### jsoncons::jsonpatch::jsonpatch_errc - -The constant integer values scoped by `jsonpatch_errc` define the values for jsonpatch error codes. - -#### Header - -```c++ -#include -``` - -### Member constants - -constant |Description ----------------------|------------------------------ -`invalid_patch` |Invalid JSON Patch document -`test_failed` |JSON Patch test operation failed -`add_failed` |JSON Patch add operation failed -`remove_failed` |JSON Patch remove operation failed -`replace_failed` |JSON Patch replace operation failed -`move_failed` |JSON Patch move operation failed -`copy_failed` |JSON Patch copy operation failed - - - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/jsonpatch_error.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/jsonpatch_error.md deleted file mode 100644 index 19b7b36b35..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpatch/jsonpatch_error.md +++ /dev/null @@ -1,65 +0,0 @@ -### jsoncons::jsonpatch::jsonpatch_error - -#### Header - - #include - -`jsoncons::jsonpatch::jsonpatch_error` defines an exception type for reporting failures in jsonpatch operations. - -![jsonpatch_error](./diagrams/jsonpatch_error.png) - -#### Constructors - - jsonpatch_error(std::error_code ec) - - jsonpatch_error(const jsonpatch_error& other) - -#### Member functions - - const char* what() const noexcept -Returns a message for this exception - -#### Inherited from std::system_error - - const std::error_code code() const noexcept -Returns an error code for this exception - -### Example - -```c++ -#include -#include - -using jsoncons::json; - -int main() -{ - json target = R"( - { "foo": "bar"} - )"_json; - - json patch = R"( - [ - { "op": "add", "path": "/baz", "value": "qux" }, - { "op": "add", "path": "/foo", "value": [ "bar", "baz" ] }, - { "op": "add", "path": "/baz/bat", "value": "qux" } // nonexistent target - ] - )"_json; - - try - { - jsonpatch::apply_patch(target, patch); - } - catch (const jsonpatch::jsonpatch_error& e) - { - std::cout << "(1) " << e.what() << std::endl; - std::cout << "(2) " << target << std::endl; - } -} -``` - -Output: -``` -(1) JSON Patch add operation failed -(2) {"foo":"bar"} -``` diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/diagrams/jsonpath_error.png b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/diagrams/jsonpath_error.png deleted file mode 100644 index aa314a1f4ddeba82b6b36c04fc58582b020f2103..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8983 zcmeHtXH=6-w>Bcu5fG#&Gy#>~HAoXMlmt+kbZLSDK``_#O{IfUr6>@PB1)C66zMhe z`XDt_=^f6%^FHsl)_Z<`>#VcZ;YSup_PuBJo|(O`>zcgLREJZLGLhop;Zdk4D?Y-* z!-s(X>xc=#s{t>H7Y~mOPeoB)+s$M(#lAyZue4dxzD_RhMv$1W=6xXwMP^}w&nAEQ zu-z8l3wTr7M5sw~?Eo)EPR^FP&9*d1Ss|TT%>i-K;Oi5Fm%mLi16u)7;njx1x8N}~ z)qI1mF$RZ(-JPT5wnb^!#JhsUdx_#NeRzarx*Jk!hE7k0KAfbL?j=Srf=QdFX z;(j?lr7wgnT)}dY*S^{$1>#{iNpxXBcm#oXp&ttF$x#}5hzT10;}pjYI3`4}%D;36ex@=L^DTSnS0ZGX*z(`5Wai#>FnC-+YR9}W;y{w1{dkczf5 zhWZEkhJso%m?v<{^1s`yT%yw2%1AK^i$S8gQ>9V$fh6Ke!&7GUck6aXZCe*iA4HzU zm~ESJ`QaFJtrF=8$f=%0FALnCct9piUVV!RijT>dCi$_=9u)&z2^je`hchqe7MiT? zu8u7aFFGyc0N^rSmmJ;iBjzH>`f(6n}Zb+okojBk|(nKUbb8ejCCDzXY4`)a1J zwmrJzdbs_}^{*e7ruK%vSxo#4AeyLnxnOW&KT%mK-qjk*GcjKAa^gL&-h-xm-y!$T zMA7B~{o-GpAMRoyFmwy@!=2u4-2TQ4YPvoM1qNJ;hC`p}c$jx<$-V34q-UXE)AYx~ zWZ;XL;9nx9<{UYWbza^5Chx;-93io)o|M15#Ro1t*H0v6wurW0o|7HeUIOy3GeZFC%Pi-KXl1MKoY91TXVaty=fF6VkqDpShUNgeMZV9)kU>8-{ee|;1Y{DO z!+6sI7RRSAqRKe36y?3uV8t(Hp)874YGKnk2S zpOO=cy^|Bfa2oZLK4%~BL%oli%X%~z9bKg&dm=rywj(* z?>V3fA8`-H%na3?meOQ6bo~kNV;)uf^ln|pJF~fYcHX>2F`Dw+7Oiu%s&^S z&;>^%OB9sqJlaOUN1E=~5@%LAP1j%XoitzQN(#BwOuivVOfX6fHHH1B`=rP z$#g>q{KU9BwPpA1aW!fJCY#1!4QuNX6@zeca;`;4t-a`$J@ev~cBtNXz?uvlxA&M7 zIYWcMaKn9~1WX#~xj3p9hn6A7`5LcUx+hYXa*7y2PH=;&Zc#eFR(36s+1YyAZo>mPz*N$M*Y=+zeYl5l#@vGE5FS*@LY3(d_DwYnLFtIsjj`AJtto>r_?E+3zL zF{xRiK0aQMb#uVWQ>(%3Nq!33Hr{5TxPLjV(Vv zd3-Cny8+Xb#>zU|=}W@sXFWqCX0XGzX+PP8#tCxyoEngsb;WtPL^_eDOA?3-3s|rU zavuMZ1wpQW(=Uv$*z|U8h_>{k`vT;wpG@OBIntmXMvZ_Md}k!b@8sxN<{08Lz*#dZ zL}tJvD?QgLLU@e&+Qr{-K7j>4(&fH#iJ8Eh&q?B`2+PC9euX^fW({<6n4g~DE!7V+ z?MFpsV!8Bd{t2N*S$J-}H}SviMCFEf6^3|qlTIzm7hPxf!`;09MuN&ujtl=AeTAz0 zpHFoD+8&xaY^`s>PNd~hk^&CI{Ix@kaB^<`JW-D*3p|czd1N=H@Y}EWyMSFYdxd~4 zDon0TNkG3xVcZeo0TNiXUb|e^`K}nM@Bwi>I~bQk91$;%hkrn@^mukNBR%L^jvC10 ztPu}Ls?%kBD&+_1Uz7w!2!iuxD!~k!J=bd%>V^Vxo1?h2ns85Gw_nXzfjP=t(x3fq zM;&!mX_qn-%?Ur=^VpurZ-$(0q&cg<|3-QqvY#Riu^b*g)nd4uY%bIs7ICsfUvs4^ zFN6B~R$c9)5Ty4-A2k6R9%$5@nyv$vv1Yui!jQ6UHm38(!r9VwDBbJ1=IB^34Z1Kh zm>o`D?7ZhQDahqQH|;U9Cpe8*??R&g_!iD)b>S*S=>NI8k**WTljhYYi@Ua5k8b2( z%w|QvSt7PUZ3iP!6SoXyo7Q*?_$EIKD~2XS5Pwo!!N*cla~p$9fne7O%yp%OAT`U> z#Y6}=B5C`|#dE#H$^$c_rM9PIh`jUfduLzwQDZnsAHe+aKv@2?1gRcNw|swPEa_7s z;G{aadv9<#T@iZsFPtI?N|xBIrQ3`WB1B61%A3%c)*Zy>@v%|)`)NzhdP-k~z_j;C z=n-&&Kh#w}(29X7dKceErOGAd5@9%NpjHolpx=E|GQ3E$ki1+T)Tu=~mo-HI5kgX7 zAphjwIb+j&<;NldVPxExTJBw|N5AvaSI|PhDWVwhL4bG!ebkG(JD_HNF{=1ISZKO_ zCMD!L?g+2+BilXhQxmKRfqJw^AOC4s15LUh&7t;S{+0BwZ7}X@HVO6LIl6+snhUi; zz=POhYYDK?w1O!L=V9UzcyLA)c)SyajHrUvAT_>MyC==s<664~DF+4dtT@5nsy(ja zd@avw=o(!1FgkXrf%Wtc(cZ8{bGje+;-TeK3sXGc>L0PUQ&_z3_n$VlK> z8C2X%7m@y8qwYDg89%Gt%$3^BGo>Y!7K3c!yNl`3X;)avn-p2TNft7DC3(5%3MbKH ztBsRo)3c?qlZ{B^bqF*xU=eeTsLu6dn_Atf87#RuOvOb8MPFwOWr8fa-To+ zyRJ;5T=ns0J3n?KvD*uyGZEziI{e9xSV?p8{uN7P2-9`_#RiuYLG)a z>aZO*b)hMlM@$%OiM(B?d%l#*4Sq$P`+Q%RllPv^{^k^LWgu~HIg}K)rf#4 zB#ARx&Fgcs3qh=Gdg(596M`lx$b>w^#}_VW4MTq#A(j8WZ_NvtK!oVRN@Neb+uq&E zKp@Ih;n%SrgKt(*^nHnmPSNXik5eH=z!?onK;r@lDkdnYWDiy-Ja6l!ErYM^kgrv4 zdSEh=aZw4|`N1grKtqv7kmzDovnut-4!RMOXrFpfhzFkTDfd7OikovC0XKP{r4{o8 z)=}$COnx!5w^S{PY4$MA-c7eW(Y@{WXG%i4(q(>s=zmG<^AxaJ6cbit(Z%S8JproIypFv=@T7 zl(LIlRAe5_tV(~}*JO!*8~Q(>$uXGtfgVOOBSMVrU!SK!j+12jzkN1C+?a!U0$agQ zfPo|w3BiQYg+W=B1n#kfSTj9ljsUPUJ;eCm@|c19X92jcc@)v-CywDc{^#8P{LBFV z)_=2u;X7;phd&{0f~DMxal<7cT@}<->=L3y(bQEjVT_!&8m!*n9hj?79DVCg5p>p* zEW$L@a&*4!Nsg$D{i^@Ey;GOYx!vP&WFgDyPX~Kr%H<6TAHsLZ@)I808I=zshzpjv z{ct(SZe4E9Q?zaBQyyMc3!;mw1ir(OE+YG;4R(#(u+ z87x!xt6=9}=ctg``8M{$HzCNc5`mXmi}1z4nPs9_pNMDYE4ZUozaY(@)nkbAomZhL7N3ny;Mi;>j?qP|_aFMW<-QQxNtT_>D}?jGn1 zE=r3N%H<8f;pze)A5{qWbHohed7e9gty-G}>oD0wx%Ff{9o67p zrn;pODc?$OX2oTFgRZsv85VPJG$3T0Bv|j!_2KT?=3M(4CK=@fxJR%gGeG8hLdf_E zh8uo38&1{oWYgyqQv4j0GW+Rc)$Ya2lBf?y3$ka4u&{`e1%9VEB>Z^*mhxdRm#GkB z$1-xhJzHky)?PiCUfP#sxaK%+ztb;O}-!EKdv11@Z}J9x$$Dt)qzJ&hgi z%TlZPVy1*N;qeY*m7Uyrn8MWzdv@?URPfN~5a5s%mzCk^Qo0)5{KwN4@7x!&$6b=t zPMxc6)S+wp%5|MH1ryJvPIo2ua2md&t^PFtb>-|-&AUtNi8sFsncqmC3+>?4=~A{5 z?^mtJ8}yi>vWNN}JDyF(dA=UoY(=BkSa$QFy(&_hDJ&Z~&H!Awt%4{5_ zKIj+tp7a@Ke6_B5oRSiqaE0Z=??BTqIB82pq7Vz+X%1C2pU4&_YdllNZTf9;Mq*@~ zdwU#<3=!_>bC^F}QtSIcnw`G@`CB!0Y3DEBZ~C03=Mqg8p2pz6vpiI_I#wPz8W-bc zb&0-#?M(3QM@)v+jyyg0fUz&YIg0Jh%Y&tdLm!qiv-lK~LUXei%o8JzlTl}{=Tn@% z#aj)R(;nM5RN4E*jeYBu7s8t^S?8IGF$~?(i@A+tQN*wO&nT0*cnK&|W zQVna1uifVI^-YlV^%>$%g1uBiHW!pnE;Ux00J);(U_SXYRoZ>i(sj(fydu(j#bT}9 z^t9+`zgc#&dyL$IQrQ0{hV!_ezII~=n=3#&5|lxsS8*%nQdI32Bg=uWaNd&y!1q;MhtQ zx|^O#i+vt>9!uE2%BK{)$~&OyFzSxs>=)^XM*%TJ@0#vt{r;?$L$=Y1_G(M2KFJXo zMYwKP{hj(@@x8gJ;=6Q75%O*E`8Z;l??WGOyR|gIWORIgSm|Kia-qp-^JKkCYDR2Ui(b-%MXhJ;xZ5RaysKL^Zh~@sTp#oCrI|jed zmgQ7MTuU#Kv#^(RPO9wQ2gWzTJa<Yi2rx6{Q2q z+%zlgV|NT1vm06E6-JE&m4)FJEgwtavdAjt;gVSDc}+=dFF*ZQjef(U>eDKer0U!a z7D-!?A~K_FmUdkj;Vsigb79-eC|g`vep0!0s{1boYuw=5;~K4_e4I`E<>v(A@Vum} z6O$?_yw72|+GWc|?-*>qlWbeUXpuJD^~Fpz+A#3?k-(rnzR9SJIg@s@wF7-LOB07E z%`Zrcv#NH{D4f-SnI}g4!LS)#uk7UT{4&^uwk+)xap4IzCpAxq(5!orPYZUYfJ>#g zWw_$-7h|rBWRg~0Y|%jVipvn&TRl|zgRTTtk&dV!U;hXK7FPim7&?Qp-P+dqWkW9)n z3R|%FrpLeO)ud~#rQW8LbqthbG#;+crXQ8A`7|bXXyE;ueZ01p*Z8Rj6rJpqP(4r8Lc$9`JtvEy%1jKx(9eoVtwGJ$tO+p0) zj`6mM8|p0fo19Iq;jVXHs#(}Jhu!!hRH!viipg*&M-pBo)|P+-sDD#_Q^)#}?@*fI z+LK%+;@t5Sw@ zWXS71KqH)`!3aid7MIgQgyoqSu3{Pz z76%qhf|3r;RpRi@a)N*gsT>x4C`(IM3?ZPwc*=2 zj*KC*w8PsL;#R)g001k^#Q#Xh4lt%W4p^3q1wUDNzx&e}4H!Jok^JZ3CvtGlk!5X0m_II$rG3pb?@IY3DAoe zfP^P}vybQ>`wH;B;;WDo0Nx95FkL)+E-M89?UqpK@Za`;lyK5hQ(OoJV1U>1cv$KH4)1^Lqtx)Q(V?#i z{|Tp&%@6llK3e7<`xbE5)1st!`JZsi!ES6!S7!ak{{O=Nzt#W$zxn0v;?oP7pn;f% zPL{BD0wiI5|Mxj_k^msX5pbrPk}XFCg+SLlzio^Iav8WudT2AQdRfP-_x#%M4GqtD z0Hw_0THTos>jJnng3-x;Edh#bJx=LJ1(EfX~bi znYnwq(8}M(+F2U5In(`v)g z=kLdAJj=<#Lj`Kh%^Sa?M@@Q%I>0UD9{+>;N6Qjx$qu!5pNBp$Pm-`tN|yF?8FRZ4 z{O3^vN9Zw7E2|$y@s68Ni)Man+0}YXL9$SQQ7N;4byh7N1DMfg4vnzC6HZKlub_g5 z6@FJF^4aH_4bT7`$mEzDsJK?|~K!qkwPmU(a?8gju(9|MxmSSU(~47QU(i$E&H*+a`qc%Qpe>cB>uVs$c=vlL7?F$xrFiK=j$Nz1XE zmSeD78@23hPeJt+Q?P!j&*{$qF#j0a0(83?US_TdG}Zbli{^;IieUBk5iT>Nu)8@- zs4GDW8uhE!6RVQ^)vaOovBIeF?}HVQEBbLR)-YIN1Zt}R-Jftwe#M^!D20y*@S{l@ zg&-r@N|v@tbQ;E)&KnfS5P{8Z>Aeq<^n%rYUip$L3b?B(E2an-m;zxn4bg z+WF{;>Mtq-WW|PKY8_E`}SJ~ diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/diagrams/jsonpath_error.xml b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/diagrams/jsonpath_error.xml deleted file mode 100644 index e3bac0fc06..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/diagrams/jsonpath_error.xml +++ /dev/null @@ -1 +0,0 @@ -7Vhdb9owFP01PHZK4kDpI6Uf2xDVOibRPlVWfJt4c2zmGAj79bOJ8x1Kq6KxaVRItY/ta/vcc+9N0kPjOL2VeBFNBQHW8xyS9tBVz/Ncx3P1P4NsMgT5XgaEkhI7qQRm9BfkKy26pASS2kQlBFN0UQcDwTkEqoZhKcW6Pu1ZsPquCxxCC5gFmLXROSUqytBh3ynxj0DDKN/ZdexIjPPJFkgiTMS6AqHrHhpLIVTWitMxMENezku27mbHaHEwCVy9ZsH13beUT+5nkTf+OVHzafyYfj6zVlaYLe2FE6Xnj/QP0gAWigpuj682OSdSLDkBY9bpoct1RBXMFjgwo2utAo1FKma65+pm+5j5niAVpBXIHvsWRAxKbvQUOzqwDFoJuTmj69IhBe1RxRm+xbDVQFhYLmnSDcvUG1g7382aXHJFY3gCrT15dOY852+jbtiirkUScDIykat7AcNJQoM6L5BS9WApNO1H0/7Qt72rtDJ0tck7XB/+ITdgOpVVplsu2/bydTt9kIilDGC/RhSWIaj9EQiklofaHq14rN/hsByTwLCiq3r26vKi3eGLoPpmhWD8ul481NBBdm27qJpsGnYuGrobNuxktLTsbCVVXPpVKgtX08ib3D3grxCeze8nsJrAS2kt2SQK4sPFZysYO+SyMz6Rf8T47GTO+xfjsxE8ewP2Jc3sDdjzU3y+Oz5RS2XfE8EX+pHpEIH5TBkbC6bNmLUItn8aT5QUP6AyggboApHDhHK/UWqHx45kv5Pjp4M+1h00+fnHZqz/3+Y+1M59uzV1yn3vyn2Dk8r2qOyoT8Co8bZZvH2+VWa+2zDk/lmdtV9SP/EIJFWYay95DqE4lDhuiU8ndFWXW71ucsGhUWQthBkNudGsVgZo/NKUBxpgNrIDMSXEbNNZXOrl5wD1pfndoHBAtcCgDiV5by8wult+yck8WH4PQ9e/AQ== \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/json_query.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/json_query.md deleted file mode 100644 index 18ea519aaa..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/json_query.md +++ /dev/null @@ -1,356 +0,0 @@ -### jsoncons::jsonpath::json_query - -Returns a `json` array of values or normalized path expressions selected from a root `json` structure. - -#### Header -```c++ -#include - -enum class result_type {value,path}; - -template -Json json_query(const Json& root, - const typename Json::string_view_type& path, - result_type result_t = result_type::value); -``` -#### Parameters - - - - - - - - - - - - - - -
rootJSON value
pathJSONPath expression string
result_tIndicates whether results are matching values (the default) or normalized path expressions
- -#### Return value - -Returns a `json` array containing either values or normalized path expressions matching the input path expression. -Returns an empty array if there is no match. - -#### Exceptions - -Throws [jsonpath_error](jsonpath_error.md) if JSONPath evaluation fails. - -#### Note -Stefan Goessner's javascript implemention returns `false` in case of no match, but in a note he suggests an alternative is to return an empty array. The `jsoncons` implementation takes that alternative and returns an empty array in case of no match. -#### - -### Store examples - -The examples below use the JSON text from [Stefan Goessner's JSONPath](http://goessner.net/articles/JsonPath/) (booklist.json). - -```json -{ "store": { - "book": [ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - }, - { "category": "fiction", - "author": "Herman Melville", - "title": "Moby Dick", - "isbn": "0-553-21311-3", - "price": 8.99 - }, - { "category": "fiction", - "author": "J. R. R. Tolkien", - "title": "The Lord of the Rings", - "isbn": "0-395-19395-8", - "price": 22.99 - } - ], - "bicycle": { - "color": "red", - "price": 19.95 - } - } -} -``` - -#### Return values - -```c++ -#include -#include -#include - -using namespace jsoncons; - -int main() -{ - std::ifstream is("./input/booklist.json"); - json booklist = json::parse(is); - - // The authors of books that are cheaper than $10 - json result1 = jsonpath::json_query(booklist, "$.store.book[?(@.price < 10)].author"); - std::cout << "(1) " << result1 << "\n"; - - // The number of books - json result2 = jsonpath::json_query(booklist, "$..book.length"); - std::cout << "(2) " << result2 << "\n"; - - // The third book - json result3 = jsonpath::json_query(booklist, "$..book[2]"); - std::cout << "(3)\n" << pretty_print(result3) << "\n"; - - // All books whose author's name starts with Evelyn - json result4 = jsonpath::json_query(booklist, "$.store.book[?(@.author =~ /Evelyn.*?/)]"); - std::cout << "(4)\n" << pretty_print(result4) << "\n"; - - // The titles of all books that have isbn number - json result5 = jsonpath::json_query(booklist, "$..book[?(@.isbn)].title"); - std::cout << "(5) " << result5 << "\n"; - - // All authors and titles of books - json result6 = jsonpath::json_query(booklist, "$['store']['book']..['author','title']"); - std::cout << "(6)\n" << pretty_print(result6) << "\n"; - - // Union of two ranges of book titles - json result7 = jsonpath::json_query(booklist, "$..book[1:2,2:4].title"); - std::cout << "(7)\n" << pretty_print(result7) << "\n"; - - // Union of a subset of book titles identified by index - json result8 = jsonpath::json_query(booklist, "$.store[book[0].title,book[1].title,book[3].title]"); - std::cout << "(8)\n" << pretty_print(result8) << "\n"; - - // Union of third book title and all book titles with price > 10 - json result9 = jsonpath::json_query(booklist, "$.store[book[3].title,book[?(@.price > 10)].title]"); - std::cout << "(9)\n" << pretty_print(result9) << "\n"; - - // Intersection of book titles with category fiction and price < 15 - json result10 = jsonpath::json_query(booklist, "$.store.book[?(@.category == 'fiction')][?(@.price < 15)].title"); - std::cout << "(10)\n" << pretty_print(result10) << "\n"; - - // Normalized path expressions - json result11 = jsonpath::json_query(booklist, "$.store.book[?(@.author =~ /Evelyn.*?/)]", jsonpath::result_type::path); - std::cout << "(11)\n" << pretty_print(result11) << "\n"; - - // All titles whose author's second name is 'Waugh' - json result12 = jsonpath::json_query(booklist,"$.store.book[?(tokenize(@.author,'\\\\s+')[1] == 'Waugh')].title"); - std::cout << "(12)\n" << result12 << "\n"; - - // All keys in the second book - json result13 = jsonpath::json_query(booklist,"keys($.store.book[1])[*]"); - std::cout << "(13)\n" << result13 << "\n"; -} -``` -Output: -``` -(1) [[1,2,3,4],[3,4,5,6]] -(2) [[1,2,3,4]] -(3) [[1,2,3,4],[3,4,5,6]] -["John","Nara"] -(1) ["Nigel Rees","Herman Melville"] -(2) [4] -(3) -[ - { - "author": "Herman Melville", - "category": "fiction", - "isbn": "0-553-21311-3", - "price": 8.99, - "title": "Moby Dick" - } -] -(4) -[ - { - "author": "Evelyn Waugh", - "category": "fiction", - "price": 12.99, - "title": "Sword of Honour" - } -] -(5) ["Moby Dick","The Lord of the Rings"] -(6) -[ - "Evelyn Waugh", - "Herman Melville", - "J. R. R. Tolkien", - "Moby Dick", - "Nigel Rees", - "Sayings of the Century", - "Sword of Honour", - "The Lord of the Rings" -] -(7) -[ - "Sword of Honour", - "Moby Dick", - "The Lord of the Rings" -] -(8) -[ - "Sayings of the Century", - "Sword of Honour", - "The Lord of the Rings" -] -(9) -[ - "Sword of Honour", - "The Lord of the Rings" -] -(10) -[ - "Sword of Honour", - "Moby Dick" -] -(11) -[ - "$['store']['book'][1]" -] -(12) -["Sword of Honour"] -(13) -["author","category","price","title"] -``` - -#### Return normalized path expressions - -```c++ -using namespace jsoncons; - -int main() -{ - std::string path = "$.store.book[?(@.price < 10)].title"; - json result = jsonpath::json_query(store,path,result_type::path); - - std::cout << pretty_print(result) << std::endl; -} -``` -Output: -```json -[ - "$['store']['book'][0]['title']", - "$['store']['book'][2]['title']" -] -``` - -### More examples - -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - const json j = json::parse(R"( - [ - { - "root": { - "id" : 10, - "second": [ - { - "names": [ - 2 - ], - "complex": [ - { - "names": [ - 1 - ], - "panels": [ - { - "result": [ - 1 - ] - }, - { - "result": [ - 1, - 2, - 3, - 4 - ] - }, - { - "result": [ - 1 - ] - } - ] - } - ] - } - ] - } - }, - { - "root": { - "id" : 20, - "second": [ - { - "names": [ - 2 - ], - "complex": [ - { - "names": [ - 1 - ], - "panels": [ - { - "result": [ - 1 - ] - }, - { - "result": [ - 3, - 4, - 5, - 6 - ] - }, - { - "result": [ - 1 - ] - } - ] - } - ] - } - ] - } - } - ] - )"); - - // Find all arrays of elements where result.length is 4 - json result1 = jsonpath::json_query(j,"$..[?(@.result.length == 4)].result"); - std::cout << "(1) " << result1 << std::endl; - - // Find array of elements that has id 10 and result.length is 4 - json result2 = jsonpath::json_query(j,"$..[?(@.id == 10)]..[?(@.result.length == 4)].result"); - std::cout << "(2) " << result2 << std::endl; - - // Find all arrays of elements where result.length is 4 and that have value 3 - json result3 = jsonpath::json_query(j,"$..[?(@.result.length == 4 && (@.result[0] == 3 || @.result[1] == 3 || @.result[2] == 3 || @.result[3] == 3))].result"); - std::cout << "(3) " << result3 << std::endl; -} -``` -Output: - -``` -(1) [[1,2,3,4],[3,4,5,6]] -(2) [[1,2,3,4]] -(3) [[1,2,3,4],[3,4,5,6]] -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/json_replace.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/json_replace.md deleted file mode 100644 index 4c4beba4ad..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/json_replace.md +++ /dev/null @@ -1,112 +0,0 @@ -### jsoncons::jsonpath::json_replace - -Searches for all values that match a JSONPath expression and replaces them with the specified value - -#### Header -```c++ -#include -``` - -```c++ -template -void json_replace(Json& root, - const typename Json::string_view_type& path, - T&& new_value) -``` -#### Parameters - - - - - - - - - - - - - - -
rootJSON value
pathJSONPath expression string
new_valueThe value to use as replacement
- -#### Exceptions - -Throws [jsonpath_error](jsonpath_error.md) if JSONPath evaluation fails. - -### Examples - -#### Change the price of a book - -Example file (booklist.json): -```json -{ "store": { - "book": [ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - }, - { "category": "fiction", - "author": "Herman Melville", - "title": "Moby Dick", - "isbn": "0-553-21311-3", - "price": 8.99 - } - ] - } -} -``` -```c++ -#include -#include - -using namespace jsoncons; -using namespace jsoncons::jsonpath; - -int main() -{ - std::ifstream is("input/booklist.json"); - json booklist; - is >> booklist; - - // Change the price of "Moby Dick" - json_replace(booklist,"$.store.book[?(@.isbn == '0-553-21311-3')].price",10.0); - std::cout << pretty_print(booklist) << std::endl; - -} -``` -Output: -```json -{ - "store": { - "book": [ - { - "author": "Nigel Rees", - "category": "reference", - "price": 8.95, - "title": "Sayings of the Century" - }, - { - "author": "Evelyn Waugh", - "category": "fiction", - "price": 12.99, - "title": "Sword of Honour" - }, - { - "author": "Herman Melville", - "category": "fiction", - "isbn": "0-553-21311-3", - "price": 10.0, - "title": "Moby Dick" - } - ] - } -} -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/jsoncons-jsonpath-abnf.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/jsoncons-jsonpath-abnf.md deleted file mode 100644 index af151146ff..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/jsoncons-jsonpath-abnf.md +++ /dev/null @@ -1,48 +0,0 @@ -``` -Path = AbsolutePath / RelativePath - -AbsolutePath = Root S *AdditionalElements - -Root = "$" - -RelativePath = S NameOrWildcard S *AdditionalElements - -NameOrWildcard = Name / Wildcard - -S = *( WSP / CR / LF ) - -AdditionalElements = (("." / "..") NameOrWildcard) / Predicate - -Predicate = "[" S Expr S "]" - -Name = UnquotedName / SingleQuotedName / DoubleQuotedName - -Expr = RelativePath / Slice / Union / Filter - -Wildcard = "*" - -Name = UnquotedName / SingleQuotedName / DoubleQuotedName - -UnquotedName = UnquotedNameCharacter *AdditionalUnquotedNameCharacter - -SingleQuotedName = "'" *SingleQuotedNameCharacter "'" - -DoubleQuotedName = '"' *DoubleQuotedNameCharacter '"' - -UnquotedNameCharacter = ? any unicode character except *, spaces, '.' and '[' ? - -AdditionalUnquotedNameCharacter = ? any unicode character except spaces, '.' and '[' ? - -SingleQuotedNameCharacter = ? any unicode character except an unescaped "'" (single quote) ? - -DoubleQuotedNameCharacter = ? any unicode character except an unescaped '"' (double quote) ? - -Slice = [ SignedInteger ] ":" [ SignedInteger ] [ ":" [ NonZeroSignedInteger ] ] - -Filter = "?(" FilterExpr ")" - -Union = RelativePathOrFilter / "," RelativePathOrFilter *("," RelativePathOrFilter) - -RelativePathOrFilter = RelativePath / Filter -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/jsonpath.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/jsonpath.md deleted file mode 100644 index 995894243d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/jsonpath.md +++ /dev/null @@ -1,225 +0,0 @@ -### jsonpath extension - -The jsonpath extension implements [Stefan Goessner's JSONPath](http://goessner.net/articles/JsonPath/). -It provides functions for search and "search and replace" using JSONPath expressions: - -[json_query](json_query.md) - -[json_replace](json_replace.md) - -### Stefan Goessner's JSONPath - -[JSONPath](http://goessner.net/articles/JsonPath/), a loosely standardized syntax for querying JSON, is a creation of Stefan Goessner. -JSONPath expressions refer to a JSON text in the same way as XPath expressions refer to an XML document. - -The [Jayway JSONPath Evaluator](https://jsonpath.herokuapp.com/) and [JSONPATH Expression Tester](https://jsonpath.curiousconcept.com/) -are good online evaluators for checking JSONPath expressions. - -### jsoncons JSONPath - -The jsoncons implementation follows these [ABNF rules](jsoncons-jsonpath-abnf.md). - -#### Differences with Stefan Goessner's implementation - -- Stefan Goessner's implemention returns `false` in case of no match, but in a note he suggests an alternative is to return an empty array. - The `jsoncons` implementation takes that alternative and returns an empty array in case of no match. -- Names in both the dot notation and the bracket notation may be unquoted (no spaces), single-quoted, or double-quoted. -- Wildcards are allowed in the dot notation -- Unions produce real unions with no duplicates instead of concatenated results -- Union of completely separate paths are allowed, e.g. - - $..[firstName,address.city] - -#### Paths - -JSONPath uses paths to select a set of nodes in a JSON value. Paths can use the dot-notation or the bracket-notation. - -Select the first (indexed 0) book using the dot notation: - - $.store.book.0 - -or - - $. store . book . 0 - -or - - store.book.0 - -or - - $.'store'.'book'.0 - -or - - $."store"."book".0 - -The leading `$` represents the root JSON value. The jsoncons implementation -allows the `$.` to be omitted, and allows single and double quoted as well as -unquoted names. Unquoted names must not contain spaces, and spaces before -and after the name are ignored. - -Select the first (indexed 0) book using the bracket-notation: - - $[store][book][0] - -or - - $['store']['book'][0] - -or - - $["store"]["book"][0] - -Recursively select all book titles under `'$.store'`: - - $.'store'..'title' - -Union of a subset of books identified by index: - - $.store[book[0],book[1],book[3]] - -Union of the fourth book and all books with price > 10: - - $.store[book[3],book[?(@.price > 10)]] - -JSONPath| Description ---------|-------------------------------- -`$`| Represents the root JSON value -`@`| Represents the current node being processed by a filter predicate. -`.` or `[]`| Child operator -`..` |Recursive descent. JSONPath borrows this syntax from [E4X](https://en.wikipedia.org/wiki/ECMAScript_for_XML). -`*` | Wildcard. All objects/elements regardless their names. -`[]` |Subscript operator. -`[,]` |Union. -`[start:end:step]` |Array slice operator borrowed from [ES4](http://wiki.ecmascript.org/doku.php?id=proposals:slice_syntax&s=array+slice). -`()` |Filter expression. -`?()` |Applies a filter expression. - -#### Filter predicates - -JSONPath uses filter predicates to restrict the set of nodes returned by a path. - -[Stefan Goessner's JSONPath](http://goessner.net/articles/JsonPath/) does not provide any specification for the allowable filter expressions, simply stating that expressions can be anything that the underlying script engine can handle. `jsoncons` expressions support the following comparision and arithmetic operators. - -Operator| Description ---------|-------------------------------- -`*` |Left times right -`/` |Left divided by right -`+` |Left plus right -`-` |Left minus right -`&&` |Left and right -|| |Left or right -`==` |Left is equal to right -`!=` |Left is not equal to right -`<` |Left is less than right -`<=` |Left is less or equal to right -`>` |Left is greater than right -`>=` |Left is greater than or equal to right -'=~' |Left matches regular expression [?(@.author =~ /Evelyn.*?/)] - -Unary operators - -Operator| Description ---------|-------------------------------- -`!` |Not right -`-` |Negates right - -Operator precedence - -Precedence|Operator|Associativity -----------|--------|----------- -1 |`!` unary `-` |Right -2 |`=~` |Left -3 |`*` `/` |Left -4 |`+` `-` |Left -5 |`<` `>` `<=` `>=`|Left -6 |`==` `!=` |Left -7 |`&&` |Left -8 ||| |Left - -#### Functions - -Support for functions is a jsoncons extension. - -Functions can be passed JSONPath paths and JSON expressions. -Outside a filter predicate, functions can be passed paths that select from -the root JSON value `$`. Within a filter predicate, functions can be passed either a -path that selects from the root JSON value `$`, or a path that selects from the current node `@`. - -Function|Description|Result|Example -----------|--------|-------|--- -`max(array)`|Returns the maximum value of an array of numbers|`double`|`max($.store.book[*].price)` -`min(array)`|Returns the minimum value of an array of numbers|`double`|`min($.store.book[*].price)` -`count(array)`|Returns the number of items in an array|`uint64_t`|`count($.store.book[*])` -`sum(array)`|Returns the sum value of an array of numbers|`double`|`$.store.book[?(@.price > sum($.store.book[*].price) / count($.store.book[*]))].title` -`avg(array)`|Returns the arithmetic average of each item of an array of numbers. If the input is an empty array, returns `null`.|`double`|`$.store.book[?(@.price > avg($.store.book[*].price))].title` -`prod(array)`|Returns the product of the elements in an array of numbers.|`double`|`$.store.book[?(479373 < prod($..price) && prod($..price) < 479374)].title` -`keys(object)`|Returns an array of keys.|`array of string`|`keys($.store.book[0])[*]` -`tokenize(string,pattern)`|Returns an array of strings formed by splitting the input string into an array of strings, separated by substrings that match the regular expression `pattern`.|`array of string`|`$.store.book[?(tokenize(@.author,'\\s+')[1] == 'Waugh')].title` - -### Examples - -The examples below use the JSON text from [Stefan Goessner's JSONPath](http://goessner.net/articles/JsonPath/) (booklist.json). - -```json -{ "store": { - "book": [ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - }, - { "category": "fiction", - "author": "Herman Melville", - "title": "Moby Dick", - "isbn": "0-553-21311-3", - "price": 8.99 - }, - { "category": "fiction", - "author": "J. R. R. Tolkien", - "title": "The Lord of the Rings", - "isbn": "0-395-19395-8", - "price": 22.99 - } - ], - "bicycle": { - "color": "red", - "price": 19.95 - } - } -} -``` - -JSONPath |Result|Notes ----------|--------------------------------------------------------|------ -`$..book.length` |The number of books|`length` is a property of an array -`$.store.*` |Everything in the store, including books and a bicycle.|`*` is a wildcard symbol -`$..author` |All authors|Recursive descent starting from root ($) to find all authors -`$.store.book[*].author` |All authors of books in the store| -`$.store..price` |The prices of everything in the store.| -`$..book[2]` |The third book|Indices start at `0` -`$..book.2` |The third book|Using the dot notation -`$..book[(@.length-1)]` |The last book in order.|Expressions () can be used as an alternative to explicit names or indices -`$..book[-1:]` |The last book in order.|A negative `start` becomes `start` + `length`. A missing `end` defaults to `length`. -`$..book[0,1]` |The first two books -`$..book[:2]` |All books from index 0 (inclusive) to index 2 (exclusive)|`start` defaults to 0 -`$..book[1:2]` |All books from index 1 (inclusive) to index 2 (exclusive) -`$..book[-2:]` |Last two books|The start index `-2` becomes `-2+4=2`, `end` defaults to `length` (`4`). -`$..book[2:]` |Last two books|`end` defaults to `length` -`$..book[?(@.isbn)]` |All books that have isbn number -`$..book[?(@.price<10)]` |All books that are cheaper than $10 -`$..*` |Everything in the store. -`$.store.book[ ?((@.author == 'Nigel Rees')` || (@.author == 'Evelyn Waugh')) ].title`|The titles of all books by Nigel Rees and Evelyn Waugh -`$.store.book[?(@.author =~ /Evelyn.*?/)]`|All books whose author's name starts with Evelyn -`$.store.book[?((@.author =~ /evelyn.*?/i))]`|All books whose author's name starts with Evelyn, evelyn etc.|`i` indicates case insensitive -`$.store.book[?(!(@.author =~ /Evelyn.*?/))]`|All books whose author's name does not start with Evelyn -`$['store']['book']..['author','title']`|All authors and titles of books in the store -`$.store.book[?(@.price < max($.store.book[*].price))].title`|The titles of all books that are priced less than the most expensive book in the store -`$.store.book[?(@.price > min($.store.book[*].price))].title`|The titles of all books that are priced higher than the cheapest book in the store - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/jsonpath_error.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/jsonpath_error.md deleted file mode 100644 index 916802be57..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpath/jsonpath_error.md +++ /dev/null @@ -1,39 +0,0 @@ -### jsoncons::jsonpath_error - -#### Header - - #include - -`jsoncons::jsonpath::jsonpath_error` defines an exception type for reporting failures in jsonpath queries. - -![jsonpath_error](./diagrams/jsonpath_error.png) - -#### Constructors - - jsonpath_error(std::error_code ec); - - jsonpath_error(std::error_code ec, size_t line, size_t column); - - jsonpath_error(const jsonpath_error& other); - -#### Member functions - - size_t line() const noexcept -Returns the line number to the end of the text where the exception occurred. -Line numbers start at 1. - - size_t column() const noexcept -Returns the column number to the end of the text where the exception occurred. -Column numbers start at 1. - - const char* what() const noexcept -Constructs an error message, including line and column position - -#### Inherited from std::system_error - - const std::error_code code() const noexcept -Returns an error code for this exception - -### Example - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/address.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/address.md deleted file mode 100644 index a5cd00cf36..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/address.md +++ /dev/null @@ -1,206 +0,0 @@ -### jsoncons::jsonpointer::basic_address - -``` -template -class basic_address -``` -#### Header -```c++ -#include -``` - -Two specializations for common character types are defined: - -Type |Definition -----------|------------------------------ -path |`basic_address` -wpath |`basic_address` - -Objects of type `basic_address` represent JSON Pointer strings. - -Addreses are implicitly convertible to string views, -which makes it possible to use them with `jsonpointer::get`, -`jsonpointer::insert_or_assign` etc, which expect string views. - -#### Member types -Type |Definition -------------|------------------------------ -char_type | `CharT` -string_type | `std::basic_string` -string_view_type | `jsoncons::basic_string_view` -const_iterator | A constant [LegacyInputIterator](https://en.cppreference.com/w/cpp/named_req/InputIterator) with a `value_type` of `std::basic_string` -iterator | An alias to `const_iterator` - -#### Header -```c++ -#include -``` - -#### Constructors - - basic_address(); -Constructs an address with an empty string, which points to the root of a json document. - - explicit basic_address(const string_type& s); - explicit basic_address(string_type&& s); - explicit basic_address(const CharT* s); -Constructs an address from a JSON Pointer string - - basic_address(const basic_address&); - - basic_address(basic_address&&); - -#### operator= - - basic_address& operator=(const basic_address&); - - basic_address& operator=(basic_address&&); - -#### Modifiers - - basic_address& operator/=(const string_type& s) -First, appends the JSON Pointer separator `/` to the path. Then appends the string token s, escaping any `/` or `~` characters. - - basic_address& operator+=(const basic_address& p) -Concatenates the current path and the specified path `p`. - -#### Iterators - - iterator begin() const; - iterator end() const; -Iterator access to the tokens in the path. - -#### Accessors - - bool empty() const -Checks if the path is empty - - const string_type& string() const -Returns the string representation of the JSON Pointer by reference. - - operator string_view_type() const; -Returns a string view representation of the JSON Pointer. - -#### Non-member functions - basic_address operator/(const basic_address& lhs, const string_type& rhs); -Concatenates a JSON Pointer path and a token. Effectively returns basic_address(lhs) /= rhs. - - basic_address operator+( const basic_address& lhs, const basic_address& rhs ); -Concatenates two JSON Pointer addresses. Effectively returns basic_address(lhs) += rhs. - - bool operator==(const basic_address& lhs, const basic_address& rhs); - - bool operator!=(const basic_address& lhs, const basic_address& rhs); - - std::basic_ostream& - operator<<(std::basic_ostream& os, const basic_address& p); -Performs stream output - -### Examples - -#### Iterate over the tokens in a JSON Pointer - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - jp::address addr("/store/book/1/author"); - - std::cout << "(1) " << addr << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& token : addr) - { - std::cout << token << "\n"; - } - - std::cout << "\n"; -} -``` -Output: -``` -(1) /store/book/1/author - -(2) -store -book -1 -author -``` - -#### Append tokens to a JSON Pointer - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - jp::address addr; - - addr /= "a/b"; - addr /= ""; - addr /= "m~n"; - - std::cout << "(1) " << addr << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& token : addr) - { - std::cout << token << "\n"; - } - - std::cout << "\n"; -} -``` -Output: -``` -(1) /a~1b//m~0n - -(2) -a/b - -m~n -``` - -#### Concatentate two JSON Pointers - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - jp::address addr("/a~1b"); - - addr += jp::address("//m~0n"); - - std::cout << "(1) " << addr << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& token : addr) - { - std::cout << token << "\n"; - } - - std::cout << "\n"; -} -``` -Output: -``` -(1) /a~1b//m~0n - -(2) -a/b - -m~n -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/address.md.orig b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/address.md.orig deleted file mode 100644 index eb9a6ec364..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/address.md.orig +++ /dev/null @@ -1,275 +0,0 @@ -<<<<<<< HEAD:doc/ref/jsonpointer/address.md -### jsoncons::jsonpointer::basic_address - -``` -template -class basic_address -======= -### jsoncons::jsonpointer::address - -``` -template -class address ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0:doc/ref/jsonpointer/address.md -``` -#### Header -```c++ -#include -``` - -<<<<<<< HEAD:doc/ref/jsonpointer/address.md -Two specializations for common character types are defined: - -Type |Definition -----------|------------------------------ -path |`basic_address` -wpath |`basic_address` - -Objects of type `basic_address` represent JSON Pointer strings. - -Addreses are implicitly convertible to string views, -which makes it possible to use them with `jsonpointer::get`, -`jsonpointer::insert_or_assign` etc, which expect string views. -======= -Objects of type `address` represent JSON Pointer addresses. ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0:doc/ref/jsonpointer/address.md - -#### Member types -Type |Definition -------------|------------------------------ -char_type | `CharT` -string_type | `std::basic_string` -string_view_type | `jsoncons::basic_string_view` -const_iterator | A constant [LegacyInputIterator](https://en.cppreference.com/w/cpp/named_req/InputIterator) with a `value_type` of `std::basic_string` -iterator | An alias to `const_iterator` - -#### Header -```c++ -#include -``` - -#### Constructors - -<<<<<<< HEAD:doc/ref/jsonpointer/address.md - basic_address(); -Constructs an address with an empty string, which points to the root of a json document. - - explicit basic_address(const string_type& s); - explicit basic_address(string_type&& s); - explicit basic_address(const CharT* s); -Constructs an address from a JSON Pointer string - - basic_address(const basic_address&); - - basic_address(basic_address&&); - -#### operator= - - basic_address& operator=(const basic_address&); - - basic_address& operator=(basic_address&&); - -#### Modifiers - - basic_address& operator/=(const string_type& s) -First, appends the JSON Pointer separator `/` to the path. Then appends the string token s, escaping any `/` or `~` characters. - - basic_address& operator+=(const basic_address& p) -Concatenates the current path and the specified path `p`. -======= - address(); - - explicit address(const string_type& s); - - explicit address(string_type&& s); - - explicit address(const CharT* s); - - address(const address&); - - address(address&&); - -#### operator= - - address& operator=(const address&); - - address& operator=(address&&); - -#### Modifiers - - address& operator/=(const string_type& s) -First, appends the JSON Pointer separator `/` to the path. Then appends the string token s, escaping any `/` or `~` characters. - - address& operator+=(const address& p) -Concatenates the current address and the specified address `p`. ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0:doc/ref/jsonpointer/address.md - -#### Iterators - - iterator begin() const; - iterator end() const; -Iterator access to the tokens in the path. - -#### Accessors - - bool empty() const -Checks if the address is empty - - const string_type& string() const -Returns the string representation of the JSON Pointer by reference. - - operator string_view_type() const; -Returns a string view representation of the JSON Pointer. - -#### Non-member functions -<<<<<<< HEAD:doc/ref/jsonpointer/address.md - basic_address operator/(const basic_address& lhs, const string_type& rhs); -Concatenates a JSON Pointer path and a token. Effectively returns basic_address(lhs) /= rhs. - - basic_address operator+( const basic_address& lhs, const basic_address& rhs ); -Concatenates two JSON Pointer addresses. Effectively returns basic_address(lhs) += rhs. - - bool operator==(const basic_address& lhs, const basic_address& rhs); - - bool operator!=(const basic_address& lhs, const basic_address& rhs); - - std::basic_ostream& - operator<<(std::basic_ostream& os, const basic_address& p); -======= - address operator/(const address& lhs, const string_type& rhs); -Concatenates a JSON Pointer address and a token. Effectively returns address(lhs) /= rhs. - - address operator+( const address& lhs, const address& rhs ); -Concatenates two JSON Pointer paths. Effectively returns address(lhs) += rhs. - - bool operator==(const address& lhs, const address& rhs); - - bool operator!=(const address& lhs, const address& rhs); - - std::ostream& - operator<<(std::ostream& os, const address& p); ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0:doc/ref/jsonpointer/address.md -Performs stream output - -### Examples - -#### Iterate over the tokens in a JSON Pointer - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ -<<<<<<< HEAD:doc/ref/jsonpointer/address.md - jp::address addr("/store/book/1/author"); -======= - jp::address p("/store/book/1/author"); ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0:doc/ref/jsonpointer/address.md - - std::cout << "(1) " << addr << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& token : addr) - { - std::cout << token << "\n"; - } - - std::cout << "\n"; -} -``` -Output: -``` -(1) /store/book/1/author - -(2) -store -book -1 -author -``` - -#### Append tokens to a JSON Pointer - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ -<<<<<<< HEAD:doc/ref/jsonpointer/address.md - jp::address addr; -======= - jp::address p; ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0:doc/ref/jsonpointer/address.md - - addr /= "a/b"; - addr /= ""; - addr /= "m~n"; - - std::cout << "(1) " << addr << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& token : addr) - { - std::cout << token << "\n"; - } - - std::cout << "\n"; -} -``` -Output: -``` -(1) /a~1b//m~0n - -(2) -a/b - -m~n -``` - -#### Concatentate two JSON Pointers - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ -<<<<<<< HEAD:doc/ref/jsonpointer/address.md - jp::address addr("/a~1b"); - - addr += jp::address("//m~0n"); -======= - jp::address p("/a~1b"); - - p += jp::address("//m~0n"); ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0:doc/ref/jsonpointer/address.md - - std::cout << "(1) " << addr << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& token : addr) - { - std::cout << token << "\n"; - } - - std::cout << "\n"; -} -``` -Output: -``` -(1) /a~1b//m~0n - -(2) -a/b - -m~n -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/contains.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/contains.md deleted file mode 100644 index 387d3a36af..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/contains.md +++ /dev/null @@ -1,55 +0,0 @@ -### jsoncons::jsonpointer::contains - -Returns `true` if the json doc contains the given JSON Pointer, otherwise `false' - -#### Header -```c++ -#include - -template -bool contains(const Json& doc, const typename Json::string_view_type& path); - -``` - -#### Return value - -Returns `true` if the json doc contains the given JSON Pointer, otherwise `false' - -### Examples - -#### Examples from [RFC6901](https://tools.ietf.org/html/rfc6901) - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - // Example from RFC 6901 - auto j = jsoncons::json::parse(R"( - { - "foo": ["bar", "baz"], - "": 0, - "a/b": 1, - "c%d": 2, - "e^f": 3, - "g|h": 4, - "i\\j": 5, - "k\"l": 6, - " ": 7, - "m~n": 8 - } - )"); - - std::cout << "(1) " << jp::contains(j, "/foo/0") << std::endl; - std::cout << "(2) " << jp::contains(j, "e^g") << std::endl; -} -``` -Output: -```json -(1) true -(2) false -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/diagrams/jsonpointer_error.html b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/diagrams/jsonpointer_error.html deleted file mode 100644 index a3f4cd9821..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/diagrams/jsonpointer_error.html +++ /dev/null @@ -1,12 +0,0 @@ - - - - -jsonpointer_error - - - -
- - - \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/diagrams/jsonpointer_error.png b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/diagrams/jsonpointer_error.png deleted file mode 100644 index 28d4e6602e17a6f867e71a2687fe1dd3010893e9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9017 zcmeHtXEdB`*R~)^qDB%#Mkfi;%fuwQgajdK^d1Bw7`;WJmq8M}w;_526N8zA=-nt0 zhMQ=k_g=nh+|T`f-+G?^-@D$mUOz0ZYoGf(_u2Mw9Q%sUe6CD=<<=DJ zM8qKAZwv()@Y|3d%1=bZPV`hsUdPLHHN&Y}N57^G?1YgsQ@LVqnH_9?D_qi&LaXun zshrO1HS(C67BWq)2tT5y>~u90b2W$%3VFMC1z_Im0x#{LgC#tM3?%tA_Tfyj*B!n! zD$N-ENE{jjdqGFg!7)^xWVm5??5|O@~b(|ye`8AjqF<=f!ErPg(%BcenUlW_SGVR=@ zjC|EWVGj9_Q(#3QM{rB?Kih3~C#UD$iUHCd%Zv0}Hw9ZX1*dYZtP4-4oB6KR%x+iY zDhGB5A5~ua4x=im>b^=$Mp4=BelrJ}7s5(5d7YZ}5_VvN)clA}CK-E(9M~av#G-88Lr#&J=^D|XurDg8i%BFWObIf{fF^yE_vtPU5NtF z=1}Soti$LO9$Sd}@t)j&%Bl_S9!3LlUFgkl*hRW8kBmIHqmc=7Tj+JyFE?hs@%7Af zLBh88Hu6=^EwJ0Xn&-iW1=wXqNw)A$TWo#ERYp`#8n_-;?}_A-{<3h7y?F5j1)Pj- z@f{H&@2Z_K@g<5+T$(oVSJm6T_Q{MJ&I*|}ERIx|t=BqFtyy2S9LRkPokIIVM@kI@ z^1gVi8lF3QN4%iJR$8`P!Fd<-8txDsG;*xA!d!%8`|+scWzKiNM2KAN;9rl zc&UKh-0Bii&akzEWCosjn6gO#AhxCp@7Z3#M$$J&W=3@svdTm+X)#GFG2ZQ z$7AK!w0c--5Ff&-;{#9Lr}#1&W@XFv)b!H2_EefNK1#DH0de2Gl^5hnUCA_`ckCpT zx-~1dsnpw6kyC4vUVo@PSAR?F_XVC|aX!ms(5=~ywPv=n$}|zH(E9VUV_b&kovBPUNie3flLh+i2TQ!!`O#>Zm>9LX5h5cFIO`n7 zs8Bg9CCmXb>-V$T9&2dCW77xIF^T>fDp$W{w35G2qEX zEA0Dv^**EdEI80F$jB#D?hm0~(sY?Ij`NW_^0QMJnM;eWL}-q&WeFSXCDCK}j?{)i z{hbJwMvs$&&43rQNM~66lh&fGooC?fPa-<8`ljA3&d0`RzcCyPs~(W{xI50VY~!8W zGvZ|UCm5cEnEFnC8J-CtL|$@S{{T{8T-2tIInuo|%si$S|Gq>k!uxm?iPH!?()K*v z9R)vfO9|iun(A2N8Hwrco;K2L-81TC4xVbGC&{{hP*uaBk!@{u1Y&d56r1ZR#>Y7& zE|=H$o_p>t>BjzrFU<1TpY*L5o$fL_bA5Jy2d4@63cb=-&7nyi#Fqev%-A0>8A#8T zMmipz%n1cDhdik@nDkx%6|<^qB$IBp!MNW(OQL)G{hZ-D>{n+npO5BlWdW{I={=t5 zCQk}EDb=;2s&;0oz(s|nC$z<^ib-Vt8FYxeiUa1`#Lep2X^)}bB6d7Ia7*&Tc^DKm zb}<=lq4lv&?mJTulu^nG%ar=YXIQbAbsc?N9=H^8$2gQ%;B^KCB2RCEiY(|A{i1uk z!_BgtuQ54oU#!|D{P%Hf41)N%Tb=a6AjNz9AK$&DCKkCV#UjQGziYXaqf2{eO6$Fn z$Vl1=?A1>>N-Pv|+5E&JjCJM2f91X)wOr=7FgeV;fgXKt&94ZY>CkIV>_kE36hBR8 zQCHqY8Io$@<%i#M^Q!{y60ffU?>b4f1Y8qHrinG!3s?n^1S1^}>PJYvWhs80{ zy_FLszQsuZq`{dFjc{qM+mclIY*zH66P~e&Ez>bc7W%B;Cmv{NXmB3 zwyv9(4g-_Qw%vCYUz|6xo<3N18O5CXW@y`#ULzF-@vsk&64SjUBK6_wk1Y;MhTEPW zV|=crm*mlZJC>DM7X}Ty8Kfs=C%QxduJU`0N&+OJbR9rG-KCN)f*4y5?0rdyw0oG7a@8_waM_EXy!}*45&EFucB?b zIZRb{`W-%;NXC*e86GZ-zjE<@kfBHm=r4aXuahDf&R1=B^%7ymcq#0L6=_nQ?Q*S$ z;DtH#`6<)@G!ssC(T0yMP7%Sy#dYQ@{ku^Zbi;h4|2p;siFGJ2qddX%nwVk)mt z7Y6ya^!-Ix8+N)Vx%%QZVO@wY=!zDPY^E0=ll%F3D++2GsKNgY0l`e)4#}Ni-xjv zAo=TUtiIi`o-4mZJQp%NQEhC26Q_GfjC)s-&^~4g6NstsXn%QYe-ynZVOA5L^z)>Z z#ra!dRsiaT8>=C+^H`P5+CJv|wCH7lvxrpS@%Yrl%Gg&NBYNX9(zY*iq)$|@y=>(` z2g1oMWdaPI?(JIv;-kP@`rlp1_(1$mvw;?B`WvIXj4l0$r9k23kMR-K)E*6{E2 zKl#JvEO3BgUp}N0xl={wP@`1BX;kaz^3Zb)=6xE1;A$|XX-6L|m9PnewvHFF0@3cg zkJX1fF~=rIr*X+UE*?I`%JZLxqa8~PT`2#w*Y;#a3}93xl;%^wG6*g$S1k3#2LHvO zSv$y~8!dZ^*C-Pp!|~F6q}RJ-{mKWSpr-7#A!hux1r|)MGYnJFRVcn^sN24*@jGnP zQxt^4Y`iyPb+F-xDW9cX_y4g{YyQwI0 z`{)#8(+dmMs95Bz7$<$4Kq++`F9BNRBQ+3 zfivvL)Q*=K_pf+bSZFH@ivO6tc5$|{&q;slf{Rn($!0XK@TzS`6}{wK=Dsbtd_PGq z_5+9#Rt&uk^cxwctiJ0Dw#==yhEye_zR9%E<(fZb!6CNU{5zXb2N@Rdx_9iLn zKb{ScqAnW$zjpDYKL7E|h1609EDZ9%l3^|MSvY|t!`{zCEK)}w^n~R zJDJcfZ&C-4>7>cye0K6sFJdUlmU-Xei_*M)c)3qqFMdAdr zamNqzEj`iYt*%G{2#4upNZzo!_FXFO!l2y)RK{qwsqKZ+;x$~WNf@gR7^PdtJZb@n z#v8Y(=Oo=Timgbhqn@c5*{x)qVt+lU+I?>~8#NFm+EaHEYoWIM2^_+tUBhY?Ek&cn z_P3Eeh-*a0M-vyG^IYucJB$e{HLT7tB3{WLE5+l>)<;Sw3oo3n2(LCD`aQzy=2Q)>aXbSl%p1{yHZ^2vMf% zHzHgh6tK~{i%pjiA^368R*~WXY|`sn&=)pQsfg&$`qD+ve4RpOe=8ugiDh+;U07*b zlF!-55wbjN_n;J?0?C8|k;NOP%&Tz4xmG4FusrhN z7}St`>w{z(D~P9?c9Gkakv}CLLqfh>4EpM=5oX03Kxadr)!bT}fap=3>N#vVz&x_4 z9Z4Yw(T{4Z9*KD9#gemneV5+*Wwo{*cz#^;rU<&DnDHt)0_gnfZdL3BsTqua~C z;tSQuwUSZneggnR{d6eH1dzKHW~a7tQ5I^I98~1MI_NBe?Uy&1KZueVI{ltS0Aw;U z;lv3b$^;e+U7M(TC6y`dJJN;$DEaG3bQWee5Q=GXO`cLx3FN8IMS zU6O>%Jis{EN8jxpaO2oF0KSLWqqcL=;g5Fu6`=rWhfaKRSz|jt zP+k&tJr**Yy`KgjA}eIerBX!$WU+&_CrQI^hGDHB$@DcfU#k^BoYw%Rx=Y%0H-@MG zBR)=pgar)U*{F3Kn;9uLSyqJs+JL8+Z7(d0md!&Op=N&yJ=~gG8`d}5O@K3ty^1al zx0W?%FP!$?oac|x_3z1Pj{;*s4@UyEWIWf>*EM7_>>dGfx3;o)O+wOgpOqh;xF9@Ru0yy)w3M?V9!#-q*P`s1~FWs&O{`Ig+^8eqr=hZM@E zjd}^(nq?pCCmVn^GO?MyYf|S=9r1$&rlXa)yrWOq0R3U#Psg_7%Oe4^fJ2)sRRHZo zjeoHU8#UZ9{9+jr`(i#P%XQpd@Ng8*b6BvQfUYU#^6xkSvJwWR5wP{~In9dn$wqTo zmPFszx(X@8Bk@>r0W)jX&#q7SN^d*G_1Es;Ld!8?!Z97wLKb zeErzq>#I6tHvj4bu&c(0W6h%a5aW0zZnI49dFDGt{7Hf)w{9e+uX~-J?&BgD?ii}& zPCiV^~pvs_d>@cCfQ|M-#6zOax*;a!rpfKc->UFv8&G4TGPvJW);3`xDqD{Yghx zD@fp(-}veclLz~&;|sG-@}4O6B<_5U3Wa~fS@IDJfI~c|{6~d?UrQ4BVc-~n<}(%s zpsS*c?M{ADl$v2Cod8j0sP@xO{;S`Z$PUdqC?!DN!{jE(I-aLeeX#Jv<&iRn-3qCs z!h-ussZqrROqMCJC+f&RrR@geZY)QAOtL?1a2@@8{%mAwLc?usR>)rVT-47DKHDmz z`sqf_hC4S-P2F^m`de$ttCRa5v@Lbe2Q5NJ$IT~)+Yg}wLn-{PAyCYz$DR4z-!%?v zWyr~`{gsZ7cj_OmeQWlgAT*TJMVw!Yp3;fn3uW;YY-_qu3wxmSskUOTx63hlmS4o za_-XVkj;+5Ng(zT59_|t;M0)MC;YY+w=GsukGJ+6;UNgCxGf$BNE7Lrge|0TJ$M>` z?~5L1x50TGXN5kDc^Y8Ws8Cpwv(daPlT>Wwuk$Ha2@7-94_VQrol(AMbPmEYIJWY8>>SN2c+b zV3ox^fI|5#LZoTxr}EFEaPRnWj|L*tRS?j&qv*q8Jp|Jjf(pt_vxp0em>7dSec zBFv@HpMKj-jq#a1cy;i!2L2KNEgUk#syqGbOGSz-%*|Tm8#y_G!_0*YL9MAYCH5B6 zyf_gH0I3Rje2J`Zafug-`PfiMY1EgL$=lXKB2NT>58h?)<$nhsZ&_PO?ac`wDc1;d z?`3RXa(YMe7XsA!aMjw8Xh)b=g{1ihjEMnvr2fOluXR?96;8lLNyh9}B-vpi^0%01 z5Y5SpBhCQ4663YqdP9I$=viWB79#0r%E9k%06?gO4A8D*%@UFN7-mI4u{@0Jzlats z)Z)lO-{XaOQM+mH{vAch(OSI|7rw0)CNfYqb-lFBR;_n-`sHY|k6*`SQAr{nCWic= zX>&+)1e7pYGFGmO{5I7^fWi z*I=`cc;a>oECnu|6{MI0?2~l;ba%o4RHFC{VhIRK3?V@`@7t4@pqM!NC*N9 z!NwBMA%AA8u>E6y9Vpja=ghryK~Mo35nwY~YUb1aV}FIx9CC$z;P1wOM8pJWdLnlR z$^Wqr0sapOxBIVfr~pBi={la|qBFo=0r+1*N&KI1*vWu!sUmkSUC`414gY^v|NqP8 z*Wr4zIrA9lI`@oFz6ARp?My@gwYcT+6yL+oW)dR^0&CBB;L;=waGbRua@Gx(-i(LHu1SJ|&IALtx4eGbK@WWq78d+b=pi)Y z`w(p9cTHCe=wWSd4l^JU3j;^gLi4)9El1+%8Cjdo>t+$j|EgzZ{Yn%tnh2$4N!NqS zcc=UTDu{nx^`_%#*kmS(;pq_fPM|Q^XZF?a(Dlw$WJ5k?)GQE(^k1u6n26_6CX^o* z&OOi1PRjy#pQkV06f#|bv&l|+OaNTwcy9~hy=7fO%;6$nSlyl><%W|y862&Z(d;YnN1<~M4=qef!&Od z(B1o=&dqI^2NcbPflJ{eHm;0_h!s+7`9gn{ zteeQ}qm+_p>L3&_7fZaoUWyt({QWJG^$$YHpMH(1a1;ryAXJt~$Z5U-puC30+Cvn8 zxl}lGyv)NGmQ)bP4Ed;Pvwgi;vC?~<+c!>!?=A|*?Z31)0PL?*I-s5_E?~0|hG#lL z-e$PX4s5hb*P=AZ-^<;{(g)0priAe-)`GUw!l3HyGXkrfsTt1`;2xXSH|}ZU;~aj% ziYz{B;VF9399a@l*br92lVGJW!sYyGvHQLQH(fJ;=vZba-e>N&>rr*4(inIFQu+X7 zG)^CH2QJV4_`Y~rPt)rR06Xz@CCu#)vGl2Mmp`pFz5dpHRn-r%@wO014=whE)t1X(uP8v7t462^pqx0_2=F@74TecSI z69Z*rYEdG^#4|mngs#4Q(=P-D5L~YBupm+&AmuGyVod@*-@sE8<#axLlnS->0lv@B0?J|eoT?Zg z&Dao+HRt#hKCuZlCc1X02D|Nc)>{Yj527D4j^z`v_rE&%HcmD@I;G{AD diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/diagrams/jsonpointer_error.xml b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/diagrams/jsonpointer_error.xml deleted file mode 100644 index f8b5221762..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/diagrams/jsonpointer_error.xml +++ /dev/null @@ -1 +0,0 @@ -7Vhdb9owFP01PHZK4kDpY0e7bkOt1jEJ+lRZ8W3izbGZYyDs188mzncoRY3GphUhYR/b1/a5595LMkCTOL2ReBndCgJs4DkkHaCrgee5jufqH4NsMwT5XgaEkhI7qQRm9BfkKy26ogSS2kQlBFN0WQcDwTkEqoZhKcWmPu1JsPquSxxCC5gFmLXROSUqytDx0Cnxj0DDKN/ZdexIjPPJFkgiTMSmAqHrAZpIIVTWitMJMENezku27sOe0eJgErh6yYLru28pn97PIm/yc6rmt/FD+vnMWlljtrIXTpSef6m/kAawVFRwe3y1zTmRYsUJGLPOAL3fRFTBbIkDM7rRKtBYpGKme65uto+Z7wlSQVqB7LFvQMSg5FZPsaMjy6CVkJszuikdUtAeVZzhWwxbDYSF5ZIm3bBMHcHa+X7W5IorGsMjaO3JkzPnOX8bdeMWdS2SgJNLE7m6FzCcJDSo8wIpVQtLoWk/mPa7oe1dpZWhq23e4frwi9yA6VRWmW65bNfL1+31QSJWMoDDGlFYhqAORyCQWh5qe7TisWGHw3JMAsOKruvZq8uLdocvguqbFYLx63rxUEMH2bXtomqyadi5aOhu3LCT0dKys5NUcekXqSxc30be9G6Bv0J4Nr+fwnoKz6W1ZJsoiPuLz1Ywdshlb3wi/4Tx2cmc9y/GZyN4Dgbsc5o5GLDnb/H56vhELZV9TwRfmu1A9hGbT5SxiWDajFmLYPfReKKk+AGVETRCF4j0E83DRrUdnzqY/U6aH3v9Z9dr/vNPzdjwv01/qJ3+9mvqLf29Kv2N3lR2QGUn/ROMGg+cxQPosTLz3YYh98/qrP2c+olHIKnCXHvJcwjFocRxS3w6oau63Op1kwsOjSJrIcxoyI1mwVRyDZjyQAPMLu1ATAkx23QWl3r56aG+NF8dFA6oFhjUoSTv+AKju+XLnMyD5SsxdP0b \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/get.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/get.md deleted file mode 100644 index da291af819..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/get.md +++ /dev/null @@ -1,284 +0,0 @@ -### jsoncons::jsonpointer::get - -Selects a `json` value. - -#### Header -```c++ -#include - -template -typename std::enable_if::value,J&>::type -get(J& root, const typename J::string_view_type& path); // (1) - -template -typename std::enable_if::value,const J&>::type -get(const J& root, const typename J::string_view_type& path); // (2) - -template -typename std::enable_if::value,J>::type -get(const J& root, const typename J::string_view_type& path); // (3) - -template -typename std::enable_if::value,J&>::type -get(J& root, const typename J::string_view_type& path, std::error_code& ec); // (4) - -template -typename std::enable_if::value,const J&>::type -get(const J& root, const typename J::string_view_type& path, std::error_code& ec); // (5) - -template -typename std::enable_if::value,J>::type -get(const J& root, const typename J::string_view_type& path, std::error_code& ec); // (6) -``` - -#### Return value - -(1) On success, returns the selected item by reference. - - json j = json::array{"baz","foo"}; - json& item = jsonpointer::get(j,"/0"); // "baz" - -(2) On success, returns the selected item by const reference. - - const json j = json::array{"baz","foo"}; - const json& item = jsonpointer::get(j,"/1"); // "foo" - -(3) On success, returns the selected item by reference, otherwise an undefined item by reference. - - json j = json::array{"baz","foo"}; - std::error_code ec; - json& item = jsonpointer::get(j,"/1",ec); // "foo" - -(4) On success, returns the selected item by const reference, otherwise an undefined item by const reference. - - const json j = json::array{"baz","foo"}; - std::error_code ec; - const json& item = jsonpointer::get(j,"/0",ec); // "baz" - -### Exceptions - -(1) Throws a [jsonpointer_error](jsonpointer_error.md) if get fails. - -(2) Throws a [jsonpointer_error](jsonpointer_error.md) if get fails. - -(3) Throws a [jsonpointer_error](jsonpointer_error.md) if get fails. - -(4) Sets the `std::error_code&` to the [jsonpointer_error_category](jsonpointer_errc.md) if get fails. - -(5) Sets the `std::error_code&` to the [jsonpointer_error_category](jsonpointer_errc.md) if get fails. - -(6) Sets the `std::error_code&` to the [jsonpointer_error_category](jsonpointer_errc.md) if get fails. - -#### Requirements - -The type J satisfies the requirements for `jsonpointer::get` if it defines the following types - -name |type |notes -------------------|----------------------|--------------- -`value_type` |`J::value_type` | -`reference` |`J::reference` | -`const_reference` |`J::const_reference` | -`pointer` |`J::pointer` | -`const_pointer` |`J::const_pointer` | -`string_type` |`J::string_type` | -`string_view_type`|`J::string_view_type` | - -and given - -- a value `index` of type `size_t` -- a value `key` of type `string_view_type` -- an rvalue expression `j` of type `J` - -the following expressions are valid - -expression |return type |effects ----------------|---------------------------|--------------- -is_array() |`bool` | -is_object() |`bool` | -size() |`size_t` | -contains(key) |`bool` | -at(index) |`reference` or `value_type`| -at(key) |`reference` or `value_type`| - -### See also - -[basic_address](address.md) - -### Examples - -#### Select author from second book - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - auto j = jsoncons::json::parse(R"( - [ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - } - ] - )"); - - // Using exceptions to report errors - try - { - jsoncons::json result = jp::get(j, "/1/author"); - std::cout << "(1) " << result << std::endl; - } - catch (const jp::jsonpointer_error& e) - { - std::cout << e.what() << std::endl; - } - - // Using error codes to report errors - std::error_code ec; - const jsoncons::json& result = jp::get(j, "/0/title", ec); - - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << "(2) " << result << std::endl; - } -} -``` -Output: -```json -(1) "Evelyn Waugh" -(2) "Sayings of the Century" -``` - -#### Using addresses with jsonpointer::get - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - auto j = jsoncons::json::parse(R"( - { - "a/b": ["bar", "baz"], - "m~n": ["foo", "qux"] - } - )"); - - jp::address addr; - addr /= "m~n"; - addr /= "1"; - - std::cout << "(1) " << addr << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& item : addr) - { - std::cout << item << "\n"; - } - std::cout << "\n"; - - jsoncons::json item = jp::get(j, addr); - std::cout << "(3) " << item << "\n"; -} -``` -Output: -``` -(1) /m~0n/1 - -(2) -m~n -1 - -(3) "qux" -``` - -#### Examples from [RFC6901](https://tools.ietf.org/html/rfc6901) - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - auto j = jsoncons::json::parse(R"( - { - "foo": ["bar", "baz"], - "": 0, - "a/b": 1, - "c%d": 2, - "e^f": 3, - "g|h": 4, - "i\\j": 5, - "k\"l": 6, - " ": 7, - "m~n": 8 - } - )"); - - try - { - const jsoncons::json& result1 = jp::get(j, ""); - std::cout << "(1) " << result1 << std::endl; - const jsoncons::json& result2 = jp::get(j, "/foo"); - std::cout << "(2) " << result2 << std::endl; - const jsoncons::json& result3 = jp::get(j, "/foo/0"); - std::cout << "(3) " << result3 << std::endl; - const jsoncons::json& result4 = jp::get(j, "/"); - std::cout << "(4) " << result4 << std::endl; - const jsoncons::json& result5 = jp::get(j, "/a~1b"); - std::cout << "(5) " << result5 << std::endl; - const jsoncons::json& result6 = jp::get(j, "/c%d"); - std::cout << "(6) " << result6 << std::endl; - const jsoncons::json& result7 = jp::get(j, "/e^f"); - std::cout << "(7) " << result7 << std::endl; - const jsoncons::json& result8 = jp::get(j, "/g|h"); - std::cout << "(8) " << result8 << std::endl; - const jsoncons::json& result9 = jp::get(j, "/i\\j"); - std::cout << "(9) " << result9 << std::endl; - const jsoncons::json& result10 = jp::get(j, "/k\"l"); - std::cout << "(10) " << result10 << std::endl; - const jsoncons::json& result11 = jp::get(j, "/ "); - std::cout << "(11) " << result11 << std::endl; - const jsoncons::json& result12 = jp::get(j, "/m~0n"); - std::cout << "(12) " << result12 << std::endl; - } - catch (const jp::jsonpointer_error& e) - { - std::cerr << e.what() << std::endl; - } -} -``` -Output: -```json -(1) {"":0," ":7,"a/b":1,"c%d":2,"e^f":3,"foo":["bar","baz"],"g|h":4,"i\\j":5,"k\"l":6,"m~n":8} -(2) ["bar","baz"] -(3) "bar" -(4) 0 -(5) 1 -(6) 2 -(7) 3 -(8) 4 -(9) 5 -(10) 6 -(11) 7 -(12) 8 -``` - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/get.md.orig b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/get.md.orig deleted file mode 100644 index 63db4f2bec..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/get.md.orig +++ /dev/null @@ -1,288 +0,0 @@ -### jsoncons::jsonpointer::get - -Selects a `json` value. - -#### Header -```c++ -#include - -template -typename std::enable_if::value,J&>::type -get(J& root, const typename J::string_view_type& path); // (1) - -template -typename std::enable_if::value,const J&>::type -get(const J& root, const typename J::string_view_type& path); // (2) - -template -typename std::enable_if::value,J>::type -get(const J& root, const typename J::string_view_type& path); // (3) - -template -typename std::enable_if::value,J&>::type -get(J& root, const typename J::string_view_type& path, std::error_code& ec); // (4) - -template -typename std::enable_if::value,const J&>::type -get(const J& root, const typename J::string_view_type& path, std::error_code& ec); // (5) - -template -typename std::enable_if::value,J>::type -get(const J& root, const typename J::string_view_type& path, std::error_code& ec); // (6) -``` - -#### Return value - -(1) On success, returns the selected item by reference. - - json j = json::array{"baz","foo"}; - json& item = jsonpointer::get(j,"/0"); // "baz" - -(2) On success, returns the selected item by const reference. - - const json j = json::array{"baz","foo"}; - const json& item = jsonpointer::get(j,"/1"); // "foo" - -(3) On success, returns the selected item by reference, otherwise an undefined item by reference. - - json j = json::array{"baz","foo"}; - std::error_code ec; - json& item = jsonpointer::get(j,"/1",ec); // "foo" - -(4) On success, returns the selected item by const reference, otherwise an undefined item by const reference. - - const json j = json::array{"baz","foo"}; - std::error_code ec; - const json& item = jsonpointer::get(j,"/0",ec); // "baz" - -### Exceptions - -(1) Throws a [jsonpointer_error](jsonpointer_error.md) if get fails. - -(2) Throws a [jsonpointer_error](jsonpointer_error.md) if get fails. - -(3) Throws a [jsonpointer_error](jsonpointer_error.md) if get fails. - -(4) Sets the `std::error_code&` to the [jsonpointer_error_category](jsonpointer_errc.md) if get fails. - -(5) Sets the `std::error_code&` to the [jsonpointer_error_category](jsonpointer_errc.md) if get fails. - -(6) Sets the `std::error_code&` to the [jsonpointer_error_category](jsonpointer_errc.md) if get fails. - -#### Requirements - -The type J satisfies the requirements for `jsonpointer::get` if it defines the following types - -name |type |notes -------------------|----------------------|--------------- -`value_type` |`J::value_type` | -`reference` |`J::reference` | -`const_reference` |`J::const_reference` | -`pointer` |`J::pointer` | -`const_pointer` |`J::const_pointer` | -`string_type` |`J::string_type` | -`string_view_type`|`J::string_view_type` | - -and given - -- a value `index` of type `size_t` -- a value `key` of type `string_view_type` -- an rvalue expression `j` of type `J` - -the following expressions are valid - -expression |return type |effects ----------------|---------------------------|--------------- -is_array() |`bool` | -is_object() |`bool` | -size() |`size_t` | -contains(key) |`bool` | -at(index) |`reference` or `value_type`| -at(key) |`reference` or `value_type`| - -### See also - -<<<<<<< HEAD -[basic_address](address.md) -======= -[path](address.md) ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0 - -### Examples - -#### Select author from second book - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - auto j = jsoncons::json::parse(R"( - [ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - } - ] - )"); - - // Using exceptions to report errors - try - { - jsoncons::json result = jp::get(j, "/1/author"); - std::cout << "(1) " << result << std::endl; - } - catch (const jp::jsonpointer_error& e) - { - std::cout << e.what() << std::endl; - } - - // Using error codes to report errors - std::error_code ec; - const jsoncons::json& result = jp::get(j, "/0/title", ec); - - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << "(2) " << result << std::endl; - } -} -``` -Output: -```json -(1) "Evelyn Waugh" -(2) "Sayings of the Century" -``` - -#### Using addresses with jsonpointer::get - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - auto j = jsoncons::json::parse(R"( - { - "a/b": ["bar", "baz"], - "m~n": ["foo", "qux"] - } - )"); - - jp::address addr; - addr /= "m~n"; - addr /= "1"; - - std::cout << "(1) " << addr << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& item : addr) - { - std::cout << item << "\n"; - } - std::cout << "\n"; - - jsoncons::json item = jp::get(j, addr); - std::cout << "(3) " << item << "\n"; -} -``` -Output: -``` -(1) /m~0n/1 - -(2) -m~n -1 - -(3) "qux" -``` - -#### Examples from [RFC6901](https://tools.ietf.org/html/rfc6901) - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - auto j = jsoncons::json::parse(R"( - { - "foo": ["bar", "baz"], - "": 0, - "a/b": 1, - "c%d": 2, - "e^f": 3, - "g|h": 4, - "i\\j": 5, - "k\"l": 6, - " ": 7, - "m~n": 8 - } - )"); - - try - { - const jsoncons::json& result1 = jp::get(j, ""); - std::cout << "(1) " << result1 << std::endl; - const jsoncons::json& result2 = jp::get(j, "/foo"); - std::cout << "(2) " << result2 << std::endl; - const jsoncons::json& result3 = jp::get(j, "/foo/0"); - std::cout << "(3) " << result3 << std::endl; - const jsoncons::json& result4 = jp::get(j, "/"); - std::cout << "(4) " << result4 << std::endl; - const jsoncons::json& result5 = jp::get(j, "/a~1b"); - std::cout << "(5) " << result5 << std::endl; - const jsoncons::json& result6 = jp::get(j, "/c%d"); - std::cout << "(6) " << result6 << std::endl; - const jsoncons::json& result7 = jp::get(j, "/e^f"); - std::cout << "(7) " << result7 << std::endl; - const jsoncons::json& result8 = jp::get(j, "/g|h"); - std::cout << "(8) " << result8 << std::endl; - const jsoncons::json& result9 = jp::get(j, "/i\\j"); - std::cout << "(9) " << result9 << std::endl; - const jsoncons::json& result10 = jp::get(j, "/k\"l"); - std::cout << "(10) " << result10 << std::endl; - const jsoncons::json& result11 = jp::get(j, "/ "); - std::cout << "(11) " << result11 << std::endl; - const jsoncons::json& result12 = jp::get(j, "/m~0n"); - std::cout << "(12) " << result12 << std::endl; - } - catch (const jp::jsonpointer_error& e) - { - std::cerr << e.what() << std::endl; - } -} -``` -Output: -```json -(1) {"":0," ":7,"a/b":1,"c%d":2,"e^f":3,"foo":["bar","baz"],"g|h":4,"i\\j":5,"k\"l":6,"m~n":8} -(2) ["bar","baz"] -(3) "bar" -(4) 0 -(5) 1 -(6) 2 -(7) 3 -(8) 4 -(9) 5 -(10) 6 -(11) 7 -(12) 8 -``` - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/insert.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/insert.md deleted file mode 100644 index 410c3e6f08..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/insert.md +++ /dev/null @@ -1,188 +0,0 @@ -### jsoncons::jsonpointer::insert - -Adds a `json` value. - -#### Header -```c++ -#include - -template -void insert(J& target, const typename J::string_view_type& path, const J& value); // (1) - -template -void insert(J& target, const typename J::string_view_type& path, const J& value, std::error_code& ec); // (2) -``` - -Inserts a value into the target at the specified path, if the path doesn't specify an object member that already has the same key. - -- If `path` specifies an array index, a new value is inserted into the array at the specified index. - -- If `path` specifies an object member that does not already exist, a new member is added to the object. - -#### Return value - -None - -### Exceptions - -(1) Throws a [jsonpointer_error](jsonpointer_error.md) if `insert` fails. - -(2) Sets the `std::error_code&` to the [jsonpointer_error_category](jsonpointer_errc.md) if `insert` fails. - -### Examples - -#### Add a member to a target location that does not already exist - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - auto target = jsoncons::json::parse(R"( - { "foo": "bar"} - )"); - - std::error_code ec; - jp::insert(target, "/baz", jsoncons::json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} -``` -Output: -```json -{"baz":"qux","foo":"bar"} -``` - -#### Insert an element to the second position in an array - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - auto target = jsoncons::json::parse(R"( - { "foo": [ "bar", "baz" ] } - )"); - - std::error_code ec; - jp::insert(target, "/foo/1", jsoncons::json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} -``` -Output: -```json -{"foo":["bar","qux","baz"]} -``` - -#### Insert a value at the end of an array - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - auto target = jsoncons::json::parse(R"( - { "foo": [ "bar", "baz" ] } - )"); - - std::error_code ec; - jp::insert(target, "/foo/-", jsoncons::json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} -``` -Output: -```json -{"foo":["bar","baz","qux"]} -``` - -#### Attempt to insert object member at a location that already exists - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - auto target = jsoncons::json::parse(R"( - { "foo": "bar", "baz" : "abc"} - )"); - - std::error_code ec; - jp::insert(target, "/baz", jsoncons::json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} -``` -Output: -``` -Key already exists -``` - -#### Attempt to insert value to a location in an array that exceeds the size of the array - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - auto target = jsoncons::json::parse(R"( - { "foo": [ "bar", "baz" ] } - )"); - - std::error_code ec; - jp::insert_or_assign(target, "/foo/3", jsoncons::json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} -``` -Output: -``` -Index exceeds array size -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/insert_or_assign.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/insert_or_assign.md deleted file mode 100644 index 1cb949ac54..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/insert_or_assign.md +++ /dev/null @@ -1,67 +0,0 @@ -### jsoncons::jsonpointer::insert_or_assign - -Adds a `json` value. - -#### Header -```c++ -#include - -template -void insert_or_assign(J& target, const typename J::string_view_type& path, const J& value); // (1) - -template -void insert_or_assign(J& target, const typename J::string_view_type& path, const J& value, std::error_code& ec); // (2) -``` - -Inserts a value into the target at the specified path, or if the path specifies an object member that already has the same key, assigns the new value to that member - -- If `path` specifies an array index, a new value is inserted into the array at the specified index. - -- If `path` specifies an object member that does not already exist, a new member is added to the object. - -- If `path` specifies an object member that does exist, that member's value is replaced. - -#### Return value - -None - -### Exceptions - -(1) Throws a [jsonpointer_error](jsonpointer_error.md) if `insert_or_assign` fails. - -(2) Sets the `std::error_code&` to the [jsonpointer_error_category](jsonpointer_errc.md) if `insert_or_assign` fails. - -### Examples - -#### Insert or assign an object member at a location that already exists - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - auto target = jsoncons::json::parse(R"( - { "foo": "bar", "baz" : "abc"} - )"); - - std::error_code ec; - jp::insert_or_assign(target, "/baz", jsoncons::json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} -``` -Output: -```json -{"baz":"qux","foo":"bar"} -``` - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/jsonpointer.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/jsonpointer.md deleted file mode 100644 index 562a6c343e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/jsonpointer.md +++ /dev/null @@ -1,153 +0,0 @@ -### jsonpointer extension - -[JSON Pointer](https://tools.ietf.org/html/rfc6901) defines a string syntax to locate a specific value in a JSON document. -A JSON Pointer is a string of zero or more tokens, each token prefixed by `/` characters. -These tokens denote keys in JSON objects or indexes in JSON arrays. An empty string with zero tokens points -to the root of a json document. - -The characters `~` and `/` have special meanings in JSON Pointer, -so if a key in a JSON object has these characters, `~` needs to be escaped as `~0`, -and `/` needs to be escaped as `~1`. - -When applied to a JSON array, the character `-` indicates one past the last element in the array. - -### Classes - - - - - -
basic_addressObjects of type basic_address represent JSON Pointer addresses.
- -### Non-member functions - - - - - - - - - - - - - - - - - - - - - - - - - - -
containsReturns true if the json document contains the given JSON Pointer
getGet a value from a JSON document using JSON Pointer path notation.
insertInserts a value in a JSON document using JSON Pointer path notation, if the path doesn't specify an object member that already has the same key.
insert_or_assignInserts a value in a JSON document using JSON Pointer path notation, or if the path specifies an object member that already has the same key, assigns the new value to that member.
removeRemoves a value from a JSON document using JSON Pointer path notation.
replaceReplaces a value in a JSON document using JSON Pointer path notation.
- -### Examples - -#### Select author from second book - -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - auto j = json::parse(R"( - [ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - } - ] - )"); - - // Using exceptions to report errors - try - { - json result = jsonpointer::get(j, "/1/author"); - std::cout << "(1) " << result << std::endl; - } - catch (const jsonpointer::jsonpointer_error& e) - { - std::cout << e.what() << std::endl; - } - - // Using error codes to report errors - std::error_code ec; - const json& result = jsonpointer::get(j, "/0/title", ec); - - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << "(2) " << result << std::endl; - } -} -``` -Output: -```json -(1) "Evelyn Waugh" -(2) "Sayings of the Century" -``` - -#### Using addresses with jsonpointer::get - -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - auto j = json::parse(R"( - { - "a/b": ["bar", "baz"], - "m~n": ["foo", "qux"] - } - )"); - - jsonpointer::address addr; - addr /= "m~n"; - addr /= "1"; - - std::cout << "(1) " << addr << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& item : addr) - { - std::cout << item << "\n"; - } - std::cout << "\n"; - - json item = jsonpointer::get(j, addr); - std::cout << "(3) " << item << "\n"; -} -``` -Output: -``` -(1) /m~0n/1 - -(2) -m~n -1 - -(3) "qux" -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/jsonpointer_errc.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/jsonpointer_errc.md deleted file mode 100644 index cb1971a878..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/jsonpointer_errc.md +++ /dev/null @@ -1,26 +0,0 @@ -### jsoncons::jsonpointer::jsonpointer_errc - -The constant integer values scoped by `jsonpointer_errc` define the values for jsonpointer error codes. - -#### Header - -```c++ -#include -``` - -### Member constants - -constant |Description -------------------------------------|------------------------------ -`expected_slash` |Expected / -`index_exceeds_array_size` |Index exceeds array size -`expected_0_or_1` |Expected '0' or '1' after escape character '~' -`invalid_index` |Invalid array index -`name_not_found` |Name not found -`key_already_exists` |Key already exists -`expected_object_or_array` |Expected object or array -`end_of_input` |Unexpected end of input - - - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/jsonpointer_error.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/jsonpointer_error.md deleted file mode 100644 index 95f063228a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/jsonpointer_error.md +++ /dev/null @@ -1,55 +0,0 @@ -### jsoncons::jsonpointer::jsonpointer_error - -#### Header - - #include - -`jsoncons::jsonpointer::jsonpointer_error` defines an exception type for reporting failures in jsonpointer operations. - -![jsonpointer_error](./diagrams/jsonpointer_error.png) - -#### Constructors - - jsonpointer_error(std::error_code ec) - - jsonpointer_error(const jsonpointer_error& other) - -#### Member functions - - const char* what() const noexcept -Returns an error message - -#### Inherited from std::system_error - - const std::error_code code() const noexcept -Returns an error code for this exception - -### Example - -```c++ -#include -#include - -using jsoncons::json; - -int main() -{ - string s = "[1,2,3,4,]"; - try - { - jsoncons::json val = jsoncons::json::parse(s); - } - catch(const jsoncons::jsonpointer_error& e) - { - std::cout << "Caught jsonpointer_error with category " - << e.code().category().name() - << ", code " << e.code().value() - << " and message " << e.what() << std::endl; - } -} -``` - -Output: -``` -Caught jsonpointer_error with category jsoncons.jsonpointer, code 0x6 and message "Name not found" -``` diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/pointer.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/pointer.md deleted file mode 100644 index c079c1c78f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/pointer.md +++ /dev/null @@ -1,200 +0,0 @@ -### jsoncons::jsonpointer::basic_pointer - -``` -template -class basic_pointer -``` -#### Header -```c++ -#include -``` - -Two specializations for common character types are defined: - -Type |Definition -----------|------------------------------ -pointer |`basic_pointer` -wpointer |`basic_pointer` - -Objects of type `basic_pointer` represent a JSON Pointer. - -#### Member types -Type |Definition -------------|------------------------------ -char_type | `CharT` -string_type | `std::basic_string` -string_view_type | `jsoncons::basic_string_view` -const_iterator | A constant [LegacyInputIterator](https://en.cppreference.com/w/cpp/named_req/InputIterator) with a `value_type` of `std::basic_string` -iterator | An alias to `const_iterator` - -#### Header -```c++ -#include -``` - -#### Constructors - - basic_pointer(); - - explicit basic_pointer(const string_type& pointer); - - explicit basic_pointer(string_type&& pointer); - - explicit basic_pointer(const CharT* s); - - basic_pointer(const basic_pointer&); - - basic_pointer(basic_pointer&&); - -#### operator= - - basic_pointer& operator=(const basic_pointer&); - - basic_pointer& operator=(basic_pointer&&); - -#### Modifiers - - basic_pointer& operator/=(const string_type& s) -First, appends the JSON Pointer separator `/`. Then appends the token s, escaping any `/` or `~` characters. - - basic_pointer& operator+=(const basic_pointer& p) -Concatenates the current pointer and the specified pointer `p`. - -#### Iterators - - iterator begin() const; - iterator end() const; -Iterator access to the tokens in the pointer. - -#### Accessors - - bool empty() const -Checks if the pointer is empty - - const string_view_type& string_view() const - operator string_view_type() const; -Access the JSON Pointer pointer as a string view. - -#### Non-member functions - basic_pointer operator/(const basic_pointer& lhs, const string_type& rhs); -Concatenates a JSON Pointer pointer and a token. Effectively returns basic_pointer(lhs) /= rhs. - - basic_pointer operator+( const basic_pointer& lhs, const basic_pointer& rhs ); -Concatenates two JSON Pointers. Effectively returns basic_pointer(lhs) += rhs. - - bool operator==(const basic_pointer& lhs, const basic_pointer& rhs); - - bool operator!=(const basic_pointer& lhs, const basic_pointer& rhs); - - std::basic_ostream& - operator<<(std::basic_ostream& os, const basic_pointer& p); -Performs stream output - -### Examples - -#### Iterate over the tokens in a JSON Pointer - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - jp::pointer p("/store/book/1/author"); - - std::cout << "(1) " << p << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& token : p) - { - std::cout << token << "\n"; - } - - std::cout << "\n"; -} -``` -Output: -``` -(1) /store/book/1/author - -(2) -store -book -1 -author -``` - -#### Append tokens to a JSON Pointer - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - jp::pointer p; - - p /= "a/b"; - p /= ""; - p /= "m~n"; - - std::cout << "(1) " << p << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& token : p) - { - std::cout << token << "\n"; - } - - std::cout << "\n"; -} -``` -Output: -``` -(1) /a~1b//m~0n - -(2) -a/b - -m~n -``` - -#### Concatentate two JSON Pointers - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - jp::pointer p("/a~1b"); - - p += jp::pointer("//m~0n"); - - std::cout << "(1) " << p << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& token : p) - { - std::cout << token << "\n"; - } - - std::cout << "\n"; -} -``` -Output: -``` -(1) /a~1b//m~0n - -(2) -a/b - -m~n -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/remove.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/remove.md deleted file mode 100644 index cab575697a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/remove.md +++ /dev/null @@ -1,92 +0,0 @@ -### jsoncons::jsonpointer::remove - -Removes a `json` element. - -#### Header -```c++ -#include - -template -void remove(J& target, const typename J::string_view_type& path); // (1) - -template -void remove(J& target, const typename J::string_view_type& path, std::error_code& ec); // (2) -``` - -Removes the value at the location specifed by `path`. - -#### Return value - -None - -### Exceptions - -(1) Throws a [jsonpointer_error](jsonpointer_error.md) if `remove` fails. - -(2) Sets the `std::error_code&` to the [jsonpointer_error_category](jsonpointer_errc.md) if `remove` fails. - -### Examples - -#### Remove an object member - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - auto target = jsoncons::json::parse(R"( - { "foo": "bar", "baz" : "qux"} - )"); - - std::error_code ec; - jp::remove(target, "/baz", ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} -``` -Output: -```json -{"foo":"bar"} -``` - -#### Remove an array element - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - auto target = jsoncons::json::parse(R"( - { "foo": [ "bar", "qux", "baz" ] } - )"); - - std::error_code ec; - jp::remove(target, "/foo/1", ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} -``` -Output: -```json -{"foo":["bar","baz"]} -``` - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/replace.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/replace.md deleted file mode 100644 index 08a64d280c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/jsonpointer/replace.md +++ /dev/null @@ -1,100 +0,0 @@ -### jsoncons::jsonpointer::replace - -Replace a `json` element or member. - -#### Header -```c++ -#include - -template -void replace(J& target, const typename J::string_view_type& path, const J& value); - -template -void replace(J& target, const typename J::string_view_type& path, const J& value, std::error_code& ec); -``` - -Replaces the value at the location specified by `path` with a new value. - -#### Return value - -None - -#### Exceptions - -(1) Throws a [jsonpointer_error](jsonpointer_error.md) if `replace` fails. - -(2) Sets the `std::error_code&` to the [jsonpointer_error_category](jsonpointer_errc.md) if `replace` fails. - -### Examples - -#### Replace an object value - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - auto target = jsoncons::json::parse(R"( - { - "baz": "qux", - "foo": "bar" - } - )"); - - std::error_code ec; - jp::replace(target, "/baz", jsoncons::json("boo"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} -``` -Output: -```json -{ - "baz": "boo", - "foo": "bar" -} -``` - -#### Replace an array value - -```c++ -#include -#include - -namespace jp = jsoncons::jsonpointer; - -int main() -{ - auto target = jsoncons::json::parse(R"( - { "foo": [ "bar", "baz" ] } - )"); - - std::error_code ec; - jp::replace(target, "/foo/1", jsoncons::json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << pretty_print(target) << std::endl; - } -} -``` -Output: -```json -{ - "foo": ["bar","qux"] -} -``` - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/line_split_kind.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/line_split_kind.md deleted file mode 100644 index bf7fd6fe93..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/line_split_kind.md +++ /dev/null @@ -1,6 +0,0 @@ -### jsoncons::line_split_kind - -```c++ -enum class line_split_kind{same_line,new_line,multi_line}; -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/decode_msgpack.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/decode_msgpack.md deleted file mode 100644 index 77f2cc4567..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/decode_msgpack.md +++ /dev/null @@ -1,30 +0,0 @@ -### jsoncons::msgpack::decode_msgpack - -Decodes a [MessagePack](http://msgpack.org/index.html) data format into a C++ data structure. - -#### Header -```c++ -#include - -template -T decode_msgpack(const std::vector& v); // (1) - -template -T decode_msgpack(std::istream& is); // (2) -``` - -(1) Reads a MessagePack bytes buffer into a type T if T is an instantiation of [basic_json](../json.md) -or if T supports [json_type_traits](../json_type_traits.md). - -(2) Reads a MessagePack binary stream into a type T if T is an instantiation of [basic_json](../json.md) -or if T supports [json_type_traits](../json_type_traits.md). - -#### Exceptions - -Throws [ser_error](../ser_error.md) if parsing fails. - -#### See also - -- [encode_msgpack](encode_msgpack.md) encodes a json value to the [MessagePack](http://msgpack.org/index.html) data format. - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/diagrams/msgpack_encoder.png b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/diagrams/msgpack_encoder.png deleted file mode 100644 index b2171a8ea6ab6896c14d41dfd0954c66f952566e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5466 zcmeI0`9IWOyvN5fh>(PmB}sNt*}~Y8EnD_=6jB&tU&kOO%OvD0lr^CYS;v~8ktI8Y zX(q-_7>2Qr<<9rs`^$aY|KL9EFQ3onanAemIq&m2=e*y~_b18RRG*!dpA`TAup1iO ze*^%~foabkEKIcPs_4uq0Km`}VPzKt3DXaF?DxzB>h9qK0GL?)s70xqKY3@>*4QN3 z5u4_5P2~bT9rmlJR(8*=%NUHwa)>q5-CqBm*pIg%F;ydK(({6PRwnS>1b5y;^nE^zC#3rlY!kYm&dqlDGRk4~trdzJ1z`Hp01<=<;-%lj}sFZ9BoH z@2K+CCDACu)xsmOQNLk$ulx^pYWLc&ffpgy=EG{z7C>CvZLzT&y5d+uOA0jhiEwkf z%MEVv@9bzl6ryafac^|Y$mj9%TRV=TuMw~Z!G=V#p6i24tQ8X_ug8!<+F-N$|@eCemHJGscUc~c~P`{CX1Sm}N>xL%l8seAbrN=_snjbVN(e z|K5UxlIQq`;Za>iBxOS9X^J8veh291(eOYR=cuQ8dv~;P&f*CwWU#MJkePQr`y|3P z%y}!F{As`~@=@!LRkUe=;UH6nx#;j6p36pYYAtU!zLa-KaB*XoSa|R{^4PC)!dMOV z!cos~!{ZN3NfKg@!|A6^76d_;{-z>|=e8l2Odm(sen|-wXnK{5G(zQSgAm&! zxf7a~=H3d4{K!ESwPjw7Dhu8@7dBmaqq9wbC9R|~``g`f*4X;n=`eM_z-Fe$Mb~b& z@7yTPUvzL*zuv_0mtk88DcrWwQoEbPsUkCgb)l#4eXb}sdK_SU{9fgy<#G2+`R*Ie z>8OuLymD7&`a} zDTu2LBS_0pcP8Flx+ivQ)!+H?Hq}B%$}hkFTpy zE4r1AP8S~OAS4V7xGyPEq}y+sXy0O&5z_|BqUD1A8vRBZvHF`=aT;E~p2Py@PTupA zJ3Hq#RpaSh*|rCY+*BOSKLd`cMph~ITs~8nguYgsP~YQD7n;=u8d(*ViT{yQS%OB( zTU>eP>j^xf!-$xjhVT~DB<-wCa&q6YlLQdYj;Y{pNBTWSrSutRTV^6K>eCH;w>P#C z^5}<_mw61iQd=e;F92jAF0J%gblSv_+0`^H!t@Uwq(5c)E2(u5VD!%yz^r2g>x$Mn z)(aT1MAM8++C7&`K}zeOJA1d|$xRr8G0W8RB>H%$v3qH{I*RR3yz&jlt%x3_Z6Qa?*gG8Ejp<7Xa~ za`uwCQ?4#jhBcvfAO|L5VJ4(DFdmBOh=*o~yB2nGW3EKI7iDgKvM64t8wO?kEHmrR z*NA-6$HOymur;4iFQgVSVBPy^r25V5ak{8}ixFTvAAC5s&m(Qqw6=?{P}7xIX;ouXxrZC+I+nSI%h#{vaQ8dN7|neA36)P7{+5u(xg>P8n1VM0q3lx6y&i zrlIUhKf928d4g+vQY~3v`0w9C+kAUzlzn0}F)txmz{#$`Cr5qvm*r4*h8gHDhI+JC z1>9%kI8(d2OtXw{%TVw+lCJD-uZlWCyYG7Mkd>e0c6F98Vsa}p~JlZ>}l>`2q z$Cq>sOg57SquwtuOcuXskD@lfXs+GKe@|;CdRxNZ7~8Al-czkbnkW{Ls}GP^R7FB!q1lQt`1spcQsN&7BQc@?L6Box@D z(KA21^uw|*NkBE=hP_|f9n$W+A>h3ER3tl?iXEgM0?lH(k@BkpJ zTs3i%H28{mqEC)xV7^yKKEky(S;ojhlNaT9j9mpE zhW9Nm297cy=#WMG<%EcSZl5^p$!J-LS7mPVNuTj-9U(8O5Fa_2qP`}8_R{}=%Du;P zL+fl?J&#?=+gWgZs+Do5H?@I02yEK5;_hU!nhzIfK^uueFYmzwm-tZKMb0Y_uBDp8t;V50nP`0B6E%^E~6@_1J2h;igJ z-V*rnz4hS{8Q}r|N}<2@*%r9ZphyMFyD*5{NXQCgkvgBM*@u+J*j3JW3=J#$ zc1lwA)KB{Q_sr>J+S2_gr8MhQi%1NM4H$PA%U21D>i9KAiP~)2cas0L2WzKpv@&AI zbmMY`Oq!50+t`!Y1mQkf+{^J%L{0QR$c3X>so9-jTubtgScx*!mF8$FsqwQ0xN;7S z01(F_Fz+~bQqUyxpfxA!1N;<6i?lY%H7c&%Yh3DgBeZ~!fP8NvtW9!+5^Ad_bw?aR z)Z9~2{kKEZezAgIWi_C*QO+r&L{i#Q{ZQpx6@RSBrRc`EjRBh0JTJ(oUSDPiH?DyJ zK9Qx`D_HJu@SZtJ)5R(D^f=u9YnXo*h*;q)*Y+7k7d9X(Lso`P`|vxV*qzv5>EJ1n zYp57*f<7Q_N~_6mH>Dj^UWOJlKUdo3JdE#gzJy^li;8bJ#ZO&XaeMLwH^qT7VJN5z zF_@t=PDdD5`^}XIb9~LNZDl>4;;=Vkgu{JM-y5K{Qs53{Z(H5bDUJ~UMJQ&MQpA5v z7`^Ve!5uwo00c$a4)w4LQYQBP(Gw!EU4JfhNBb((V%V2Qj6J$HioQr(7Ot&81mw)L<>5sa(4wY@15DJb~-Dj z#+R@p%jWnwgxlTFWgV0Re6VIFJrFS;d;@whJj z58nGj>QL4Cw=4F4bydJrnQ5QwS2b^;uYehw(Fsau)0}1;ef}m>ei=X$OqJQlHYOAL zSF(^nOhW76?X`O*wcCpVh4klOl0Y5_%h>$<{M&=e_-;+fA91XAbz|WgBE!c|?+++^ z2JjTp)6w2xN@Hx%0)Enh(VX2{%;mDAle!Q?qfLNeA)8eU>-RF+oWMg@6#n1*|Fkmk zBTh~5la_&;JNh_iS>Z$4 zRLlAC@KIXpB8}&2eW&i(*SaTB0{=Am2fyy_|I~5z?+(3s`aw&2);om{O`!1%j20XJ ztu&sMJjZ(l($1KYnV-h2JxfS%aC2XMU7e*(h28AS|7Lin`t7K_-<}?)$A`BA77Tv3 zM@K)B71o41b*Bn%e#uevehY9_xiw95@K3Y;Yk~AC%#l&4Y2NB7jbZoPb47Gt_U`=s zxk#Sv{2ao}&c86!ul#ryKqLQfwZ;%i`3 zz#feiodlXN#l^b3GIr)%%Ex*o1$+hL!Gs1#?bAX2GQ=v_%s1Rbg?4Oa7p`A68=GNU z1`Ka(_~vLtHZX{DX!*`!5ggZJIv4x1royx2oZmDDF895ab^K26B*K`@Fk=s4ZhQKl zL<_L)>93=eT~o;uKp4N0PxdPzjeFz52dprO1{qO zBc;8`tOeHT0 z&k|671=hRbq1B%yhie{mOzwe3`NrYZU~OK6JfPlxzWd{5j`d;bZ^WChB2UK~4Vn-F zN1}64_BaJi7F6+b9pl@JoO-T7qp+ne{e!om@^^pUo+UgCrmo;pVyfC(0kX_m_aB#B zs3lwD@@QP3y+?x*|5heHS6ha12PfuF9^aIqwsBDdw$WpI;slE8{_!EDHf(Foy`bvo zv)Ub-N(V9O9?5u5%^S&}L?#MHAE)oYAcSuXI@@!iFzQN*RTl)?s6pxbL~pLe_qHO- zhpx$h&xn?8WG!Itd^Lf`SraLD8bJc`QFI^LSrCmOY}A(gO0A0A>ji vVyYapAZ=C1MB51b-{(I6r`-cdvgT>UyB}$|_w%%WrT{}d)B9g^TwnYLB%7KR diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/diagrams/msgpack_encoder.xml b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/diagrams/msgpack_encoder.xml deleted file mode 100644 index c486dc72df..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/diagrams/msgpack_encoder.xml +++ /dev/null @@ -1 +0,0 @@ -vVZRb9owEP41eeyUxIXSxxZYt6FW65gEfUJufE28OjZzDIT9+tnEJnESVCpV7UPl+853tr/77kKAxnl5J/E6uxcEWBCHpAzQJIjjq2ik/xtgXwFoGFZAKimpoKgG5vQfWNBt21AChbdRCcEUXftgIjiHRHkYllLs/G0vgvmnrnEKHWCeYNZFF5SorEJHg7DGvwFNM3dyFFpPjt1mCxQZJmLXgNA0QGMphKpWeTkGZrhzvFRxX094jxeTwNU5AdOH3yWfPc6zePx3phb3+VP548Jm2WK2sQ9+xgVNVn8KwVeaU6WTrzLMCQMZxEOmT7p9NqvUrPqQw1PV3vEnxYYTMFcItXuXUQXzNU6Md6cFo7FM5Uxb0TG6+SR3P5AKygZkn3gHIgcl93qL9Y4s21ZukWN/VxcvchLMGoW7tBi2ekmPmWtK9cKy+g6Gr04wnBep5uF1BTzRPfMJ5L5QxsaCCXmIRXD403ihpHiFhgcN0TUiH1OOs+ox+Mx6jDr16NAKnNyY0aGthOFCV8tnEkqqlpZ0s34y6y8Da03KhmuydwbXl1+6BMZoRBmzDjtYLu5kDQqxkQm8LTyFZQrq7REAxBuE3Yo2KtZXMIdJYFjRrT8++6poT/gpqH7ZUTCXvl5i1NJB9Wwb1Jx2rTzXLd2NWnkqWjp5DpI6PvoslaXb+yyePSzxL0gvFo8z2M6gp+u/8wwkVZjrounPIcWpxHlHerqtlC82vz254NDqZQthRlNuFKuFoscJujVNSvWH7MY6ckqIOaZ3TviT5CMaP2wVMuxpfNSjo/j9ja/N+jNaVbD+LYKm/wE= \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/encode_msgpack.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/encode_msgpack.md deleted file mode 100644 index 7344c0fbdb..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/encode_msgpack.md +++ /dev/null @@ -1,103 +0,0 @@ -### jsoncons::msgpack::encode_msgpack - -Encodes a C++ data structure into the [MessagePack](http://msgpack.org/index.html) data format. - -#### Header -```c++ -#include - -template -void encode_msgpack(const T& jval, std::vector& v); // (1) - -template -void encode_msgpack(const T& jval, std::ostream& os); // (2) -``` - -(1) Writes a value of type T into a bytes buffer in the MessagePack data format. Type T must be an instantiation of [basic_json](../json.md) -or support [json_type_traits](../json_type_traits.md). - -(2) Writes a value of type T into a binary stream in the MessagePack data format. Type T must be an instantiation of [basic_json](../json.md) -or support [json_type_traits](../json_type_traits.md). - -#### See also - -- [decode_msgpack](decode_msgpack) decodes a [MessagePack](http://msgpack.org/index.html) data format to a json value. - -### Examples - -#### MessagePack example - -```c++ -#include -#include - -using namespace jsoncons; -using namespace jsoncons::msgpack; - -int main() -{ - ojson j1; - j1["zero"] = 0; - j1["one"] = 1; - j1["two"] = 2; - j1["null"] = null_type(); - j1["true"] = true; - j1["false"] = false; - j1["max int64_t"] = (std::numeric_limits::max)(); - j1["max uint64_t"] = (std::numeric_limits::max)(); - j1["min int64_t"] = (std::numeric_limits::lowest)(); - j1["max int32_t"] = (std::numeric_limits::max)(); - j1["max uint32_t"] = (std::numeric_limits::max)(); - j1["min int32_t"] = (std::numeric_limits::lowest)(); - j1["max int16_t"] = (std::numeric_limits::max)(); - j1["max uint16_t"] = (std::numeric_limits::max)(); - j1["min int16_t"] = (std::numeric_limits::lowest)(); - j1["max int8_t"] = (std::numeric_limits::max)(); - j1["max uint8_t"] = (std::numeric_limits::max)(); - j1["min int8_t"] = (std::numeric_limits::lowest)(); - j1["max double"] = (std::numeric_limits::max)(); - j1["min double"] = (std::numeric_limits::lowest)(); - j1["max float"] = (std::numeric_limits::max)(); - j1["zero float"] = 0.0; - j1["min float"] = (std::numeric_limits::lowest)(); - j1["Key too long for small string optimization"] = "String too long for small string optimization"; - - std::vector v; - encode_msgpack(j1, v); - - ojson j2 = decode_msgpack(v); - - std::cout << pretty_print(j2) << std::endl; -} -``` -Output: -```json -{ - "zero": 0, - "one": 1, - "two": 2, - "null": null, - "true": true, - "false": false, - "max int64_t": 9223372036854775807, - "max uint64_t": 18446744073709551615, - "min int64_t": -9223372036854775808, - "max int32_t": 2147483647, - "max uint32_t": 4294967295, - "min int32_t": -2147483648, - "max int16_t": 32767, - "max uint16_t": 65535, - "min int16_t": -32768, - "max int8_t": 127, - "max uint8_t": 255, - "min int8_t": -128, - "max double": 1.79769313486232e+308, - "min double": -1.79769313486232e+308, - "max float": 3.40282346638529e+038, - "zero float": 0.0, - "min float": -3.40282346638529e+038, - "Key too long for small string optimization": "String too long for small string optimization" -} -``` - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/msgpack.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/msgpack.md deleted file mode 100644 index 7bad78cdc2..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/msgpack.md +++ /dev/null @@ -1,94 +0,0 @@ -### msgpack extension - -The msgpack extension implements encode to and decode from the [MessagePack](http://msgpack.org/index.html) data format. -You can either parse into or serialize from a variant-like structure, [basic_json](../json.md), or your own -data structures, using [json_type_traits](../json_type_traits.md). - - -[decode_msgpack](decode_msgpack.md) - -[encode_msgpack](encode_msgpack.md) - -[msgpack_encoder](msgpack_encoder.md) - -#### jsoncons-MessagePack mappings - -jsoncons data item|jsoncons tag|BSON data item ---------------|------------------|--------------- -null | | nil -bool | | true or false -int64 | | negative fixnum, int 8, int 16, int 32, int 64 -uint64 | | positive fixnum, uint 8, uint 16, uint 32, uint 64 -double | | float32 or float64 -string | | fixstr, str 8, str 16 or str 32 -byte_string | | bin 8, bin 16 or bin 32 -array | | array -object | | map - -### Examples - -Example file (book.json): -```json -[ - { - "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { - "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - } -] -``` -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - std::ifstream is("input/book.json"); - ojson j1; - is >> j1; - - // Encode ojson to MessagePack - std::vector v; - msgpack::encode_msgpack(j1, v); - - // Decode MessagePack to ojson - ojson j2 = msgpack::decode_msgpack(v); - - std::cout << pretty_print(j2) << std::endl; - - // or to json (now alphabetically sorted) - json j3 = msgpack::decode_msgpack(v); - - // or to wjson (converts from utf8 to wide characters) - wjson j4 = msgpack::decode_msgpack(v); -} -``` -Output: -```json -[ - { - "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { - "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - } -] -``` - - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/msgpack_encoder.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/msgpack_encoder.md deleted file mode 100644 index 9036b77f2b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/msgpack/msgpack_encoder.md +++ /dev/null @@ -1,100 +0,0 @@ -### jsoncons::bson::basic_msgpack_encoder - -```c++ -template< - class Result> -> class basic_msgpack_encoder : public jsoncons::json_content_handler -``` - -`basic_msgpack_encoder` is noncopyable. - -#### Header - - #include - -![msgpack_encoder](./diagrams/msgpack_encoder.png) - -Four specializations for common character types and result types are defined: - -Type |Definition ----------------------------|------------------------------ -msgpack_encoder |basic_msgpack_encoder -bson_bytes_encoder |basic_msgpack_encoder - -#### Member types - -Type |Definition ----------------------------|------------------------------ -char_type |char -result_type |Result -string_view_type | - -#### Constructors - - explicit basic_msgpack_encoder(result_type result) -Constructs a new encoder that writes to the specified result. - -#### Destructor - - virtual ~basic_msgpack_encoder() - -### Inherited from [basic_json_content_handler](../json_content_handler.md) - -#### Member functions - - bool begin_object(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool begin_object(size_t length, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool end_object(const ser_context& context = null_ser_context()) - - bool begin_array(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool begin_array(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool end_array(const ser_context& context=null_ser_context()); - - bool name(const string_view_type& name, - const ser_context& context=null_ser_context()); - - bool string_value(const string_view_type& value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool byte_string_value(const byte_string_view& b, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool byte_string_value(const uint8_t* p, size_t size, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool int64_value(int64_t value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool uint64_value(uint64_t value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool double_value(double value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool bool_value(bool value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool null_value(semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - void flush() - -### Examples - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ojson.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ojson.md deleted file mode 100644 index 5fd96cf317..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ojson.md +++ /dev/null @@ -1,87 +0,0 @@ -### jsoncons::ojson - -```c++ -typedef basic_json> ojson -``` -The `ojson` class is an instantiation of the `basic_json` class template that uses `char` as the character type. The original insertion order of an object's name/value pairs is preserved. - -The `jsoncons` library will always rebind the supplied allocator from the template parameter to internal data structures. - -#### Header - - #include - -#### Interface - -The interface is the same as [json](json.md), with these provisos: - -- `ojson`, like `json`, supports object member `insert_or_assign` methods that take an `object_iterator` as the first parameter. But while with `json` that parameter is just a hint that allows optimization, with `ojson` it is the actual location where to insert the member. - -- In `ojson`, the `insert_or_assign` members that just take a name and a value always insert the member at the end. - -#### See also - -- [json](json.md) constructs a json value that sorts name-value members alphabetically - -- [wjson](wjson.md) constructs a wide character json value that sorts name-value members alphabetically - -- [wojson](wojson.md) constructs a wide character json value that preserves the original insertion order of an object's name/value pairs - -### Examples -```c++ -ojson o = ojson::parse(R"( -{ - "street_number" : "100", - "street_name" : "Queen St W", - "city" : "Toronto", - "country" : "Canada" -} -)"); - -std::cout << pretty_print(o) << std::endl; -``` -Output: -```json -{ - "street_number": "100", - "street_name": "Queen St W", - "city": "Toronto", - "country": "Canada" -} -``` -Insert "postal_code" at end -```c++ -o.insert_or_assign("postal_code", "M5H 2N2"); - -std::cout << pretty_print(o) << std::endl; -``` -Output: -```json -{ - "street_number": "100", - "street_name": "Queen St W", - "city": "Toronto", - "country": "Canada", - "postal_code": "M5H 2N2" -} -``` -Insert "province" before "country" -```c++ -auto it = o.find("country"); -o.insert_or_assign(it,"province","Ontario"); - -std::cout << pretty_print(o) << std::endl; -``` -Output: -```json -{ - "street_number": "100", - "street_name": "Queen St W", - "city": "Toronto", - "province": "Ontario", - "country": "Canada", - "postal_code": "M5H 2N2" -} -``` diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/parse_error_handler.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/parse_error_handler.md deleted file mode 100644 index fc58ed91e2..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/parse_error_handler.md +++ /dev/null @@ -1,42 +0,0 @@ -### jsoncons::parse_error_handler - -```c++ -class parse_error_handler; -``` - -When parsing JSON text with [json_reader](json_reader.md), if you need to implement -customized error handling, you must implement this abstract class -and pass it in the constructor of [json_reader](json_reader.md). The `read` method -will then report all warnings and errors through this interface. - -#### Header - - #include - -#### Destructor - - virtual ~json_error_handler() - -#### Public interface methods - - void error(std::error_code ec, - const ser_context& context) throw (ser_error) = 0 -Called for recoverable errors. Calls `do_error`, if `do_error` returns `false`, throws a [ser_error](ser_error.md), otherwise an attempt is made to recover. - - void fatal_error(std::error_code ec, - const ser_context& context) throw (ser_error) = 0 -Called for unrecoverable errors. Calls `do_fatal_error` and throws a [ser_error](ser_error.md). - -#### Private virtual implementation methods - - virtual bool do_error(std::error_code ec, - const ser_context& context) = 0 -Receive an error event, possibly recoverable. An [error_code](json_error_category.md) indicates the type of error. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. Returns `false` to fail, `true` to attempt recovery. - - virtual void do_fatal_error(std::error_code ec, - const ser_context& context) = 0 -Receives a non recoverable error. An [error_code](json_error_category.md) indicates the type of error. Contextual information including -line and column number is provided in the [context](ser_context.md) parameter. - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/rename_object_member_filter.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/rename_object_member_filter.md deleted file mode 100644 index b48c6fbe2f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/rename_object_member_filter.md +++ /dev/null @@ -1,55 +0,0 @@ -### jsoncons::rename_object_member_filter - -```c++ -typedef basic_rename_object_member_filter rename_object_member_filter; -``` -The `rename_object_member_filter` class is an instantiation of the `basic_rename_object_member_filter` class template that uses `char` as the character type. - -Renames object member names. - -#### Header -```c++ -#include -``` - -![rename_object_member_filter](./diagrams/rename_object_member_filter.png) - -#### Constructors - - rename_object_member_filter(const std::string& name, - const std::string& new_name, - json_content_handler& handler) - - rename_object_member_filter(const std::string& name, - const std::string& new_name, - json_content_handler& handler) - -### See also - -- [json_filter](json_filter.md) - -### Examples - -#### Rename object member names - -```c++ -#include -#include - -using namespace jsoncons; - -int main() -{ - ojson j = ojson::parse(R"({"first":1,"second":2,"fourth":3})"); - - json_encoder encoder(std::cout); - - rename_object_member_filter filter("fourth","third",encoder); - j.dump(filter); -} -``` -Output: -```json -{"first":1,"second":2,"third":3} -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/semantic_tag.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/semantic_tag.md deleted file mode 100644 index a07a465b2c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/semantic_tag.md +++ /dev/null @@ -1,19 +0,0 @@ -### jsoncons::semantic_tag - -```c++ -enum class semantic_tag : uint8_t -{ - none = 0, - undefined, - datetime, - timestamp, - bigint, - bigdec, - base16, - base64, - base64url, - uri, - bigfloat -}; -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ser_context.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ser_context.md deleted file mode 100644 index 57ee058882..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ser_context.md +++ /dev/null @@ -1,22 +0,0 @@ -### jsoncons::ser_context - -```c++ -class ser_context; -``` - -Provides contextual information for serializing and deserializing JSON and JSON-like data formats. This information may be used for error reporting. - -#### Header - - #include - - virtual size_t line() const = 0; -Returns the line number for the text being parsed. -Line numbers start at 1. - - virtual size_t column() const = 0; -Returns the column number to the end of the text being parsed. -Column numbers start at 1. - - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ser_error.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ser_error.md deleted file mode 100644 index db0185c748..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ser_error.md +++ /dev/null @@ -1,68 +0,0 @@ -### jsoncons::ser_error - -#### Header - - #include - -`jsoncons::ser_error` defines an exception type for reporting serialization and deserialization failures. - -![ser_error](./diagrams/ser_error.png) - -std::exception - -#### Constructors - - ser_error(std::error_code ec); - - ser_error(std::error_code ec, size_t position); - - ser_error(std::error_code ec, - size_t line, - size_t column); - - ser_error(const ser_error& other); - -#### Member functions - - size_t line() const noexcept -Returns the line number to the end of the text where the exception occurred. -Line numbers start at 1. - - size_t column() const noexcept -Returns the column number to the end of the text where the exception occurred. -Column numbers start at 1. - - const char* what() const -Constructs an error message, including line and column position - -#### Inherited from std::system_error - - const std::error_code code() const noexcept -Returns an error code for this exception - -### Example - - #include - - using jsoncons::json; - - int main() - { - string s = "[1,2,3,4,]"; - try - { - jsoncons::json val = jsoncons::json::parse(s); - } - catch(const jsoncons::ser_error& e) - { - std::cout << "Caught ser_error with category " - << e.code().category().name() - << ", code " << e.code().value() - << " and message " << e.what() << std::endl; - } - } - - -Output: - - Caught ser_error with category json_input, code 1 and message Unexpected value separator ',' at line 1 and column 10 diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/spaces_option.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/spaces_option.md deleted file mode 100644 index 9e6fc32a1b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/spaces_option.md +++ /dev/null @@ -1,12 +0,0 @@ -### jsoncons::spaces_option - -```c++ -enum class spaces_option : uint8_t -{ - no_spaces=0, - space_after, - space_before, - space_before_and_after -}; -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_array_iterator.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_array_iterator.md deleted file mode 100644 index c81ea586d4..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_array_iterator.md +++ /dev/null @@ -1,192 +0,0 @@ -### jsoncons::staj_array_iterator - -```c++ -template -using staj_array_iterator = basic_staj_array_iterator>; -``` - -#### Header -```c++ -#include -``` - -A `staj_array_iterator` is an [InputIterator](https://en.cppreference.com/w/cpp/named_req/InputIterator) that -accesses the individual stream events from a [staj_reader](staj_reader.md) and, provided that when it is constructed -the current stream event has type `staj_event_type::begin_array`, it retrieves the elements of the JSON array -as items of type `T`. If when it is constructed the current stream event does not have type `staj_event_type::begin_array`, -it becomes equal to the default-constructed iterator. - -#### Member types - -Member type |Definition -------------------------------------|------------------------------ -`char_type`|char -`value_type`|`T` -`difference_type`|`std::ptrdiff_t` -`pointer`|`value_type*` -`reference`|`value_type&` -`iterator_category`|[std::input_iterator_tag](https://en.cppreference.com/w/cpp/iterator/iterator_tags) - -#### Constructors - - staj_array_iterator() noexcept; // (1) - - staj_array_iterator(basic_staj_reader& reader); // (2) - - staj_array_iterator(basic_staj_reader& reader, - std::error_code& ec); // (3) - -(1) Constructs the end iterator - -(2) Constructs a `staj_array_iterator` that refers to the first element of the array - following the current stream event `begin_array`. If there is no such element, - returns the end iterator. If a parsing error is encountered, throws a - [ser_error](ser_error.md). - -(3) Constructs a `staj_array_iterator` that refers to the first member of the array - following the current stream event `begin_array`. If there is no such element, - returns the end iterator. If a parsing error is encountered, returns the end iterator - and sets `ec`. - -#### Member functions - - const T& operator*() const - - const T* operator->() const - - staj_array_iterator& operator++() - staj_array_iterator& increment(std::error_code& ec) - staj_array_iterator operator++(int) -Advances the iterator to the next array element. - -#### Non-member functions - - template - bool operator==(const staj_array_iterator& a, const staj_array_iterator& b) - - template - bool operator!=(const staj_array_iterator& a, const staj_array_iterator& b) - - template - staj_array_iterator begin(staj_array_iterator iter) noexcept; // (1) - - template - staj_array_iterator end(const staj_array_iterator&) noexcept; // (2) - -(1) Returns iter unchanged - -(2) Returns a default-constructed `stax_array_iterator`, which serves as an end iterator. The argument is ignored. - -The `begin` and `end` non-member functions enable the use of `stax_array_iterators` with range-based for loops. - -### Examples - -#### Iterate over a JSON array, returning json values - -```c++ -const std::string example = R"( -[ - { - "employeeNo" : "101", - "name" : "Tommy Cochrane", - "title" : "Supervisor" - }, - { - "employeeNo" : "102", - "name" : "Bill Skeleton", - "title" : "Line manager" - } -] -)"; - -int main() -{ - std::istringstream is(example); - - json_cursor reader(is); - - staj_array_iterator it(reader); - - for (const auto& j : it) - { - std::cout << pretty_print(j) << "\n"; - } - std::cout << "\n\n"; -} -``` -Output: -``` -{ - "employeeNo": "101", - "name": "Tommy Cochrane", - "title": "Supervisor" -} -{ - "employeeNo": "102", - "name": "Bill Skeleton", - "title": "Line manager" -} -``` - -#### Iterate over the JSON array, returning employee values - -```c++ -struct employee -{ - std::string employeeNo; - std::string name; - std::string title; -}; - -namespace jsoncons -{ - template - struct json_type_traits - { - template - struct json_type_traits - { - static bool is(const Json& j) noexcept - { - return j.is_object() && j.contains("employeeNo") && j.contains("name") && j.contains("title"); - } - static employee as(const Json& j) - { - employee val; - val.employeeNo = j["employeeNo"].template as(); - val.name = j["name"].template as(); - val.title = j["title"].template as(); - return val; - } - static Json to_json(const employee& val) - { - Json j; - j["employeeNo"] = val.employeeNo; - j["name"] = val.name; - j["title"] = val.title; - return j; - } - }; -} - -int main() -{ - std::istringstream is(example); - - json_cursor reader(is); - - staj_array_iterator it(reader); - - for (const auto& val : it) - { - std::cout << val.employeeNo << ", " << val.name << ", " << val.title << "\n"; - } - std::cout << "\n\n"; -} -``` -Output: -``` -101, Tommy Cochrane, Supervisor -102, Bill Skeleton, Line manager -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_event.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_event.md deleted file mode 100644 index ed0bb2c873..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_event.md +++ /dev/null @@ -1,40 +0,0 @@ -### jsoncons::staj_event - -```c++ -typedef basic_staj_event staj_event; -``` - -#### Header -```c++ -#include - -A JSON-like data event. -``` - -| Event type | Sample data | Valid accessors | -|-------------------|------------------------|-----------------| -| begin_object | | | -| end_object | | | -| begin_array | | | -| end_array | | | -| name | "foo" | `as()`, `as`, `as()` | -| string_value | "1000" | `as()`, `as`, `as()`, `as()`, `as()` | -| byte_string_value | 0x660x6F0x6F | `as()`, `as()` | -| int64_value | -1000 | `as()`, `as()`, `as`, `as()` | -| uint64_value | 1000 | `as()`, `as()`, `as()`, `as()`, `as()` | -| double_value | 125.72 | `as()`, `as()` | -| bool_value | true | `as()`, `as()` | -| null_value | | `as()` | - -#### Member functions - - staj_event_type event_type() const noexcept; -Returns a [staj_event_type](staj_event_type.md) for this event. - - semantic_tag get_semantic_tag() const noexcept; -Returns a [semantic_tag](semantic_tag.md) for this event. - - template - T as(Args&&... args) const; -Attempts to convert the json value to the template value type. - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_event_type.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_event_type.md deleted file mode 100644 index fc028609ea..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_event_type.md +++ /dev/null @@ -1,27 +0,0 @@ -### jsoncons::staj_event_type - -#### Header -```c++ -#include -``` - -```c++ -enum class staj_event_type -{ - begin_object, - end_object, - begin_array, - end_array, - name, - string_value, - byte_string_value, - bigint_value, - bigdec_value, - int64_value, - uint64_value, - double_value, - bool_value, - null_value -}; -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_filter.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_filter.md deleted file mode 100644 index b9d623944d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_filter.md +++ /dev/null @@ -1,20 +0,0 @@ -### jsoncons::staj_filter - -```c++ -typedef basic_staj_filter staj_filter; -``` - -#### Header -```c++ -#include -``` - -#### Destructor - - virtual ~basic_staj_filter() = default; - -#### Member functions - - virtual bool accept(const staj_event& event, const ser_context& context) = 0; -Tests whether the [current event](staj_event.md) is part of the stream. Returns `true` if the filter accepts the event, `false` otherwise. - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_object_iterator.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_object_iterator.md deleted file mode 100644 index c9910b1d22..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_object_iterator.md +++ /dev/null @@ -1,131 +0,0 @@ -### jsoncons::staj_object_iterator - -```c++ -template -using staj_object_iterator = basic_staj_object_iterator>; -``` - -#### Header -```c++ -#include -``` - -A `staj_object_iterator` is an [InputIterator](https://en.cppreference.com/w/cpp/named_req/InputIterator) that -accesses the individual stream events from a [staj_reader](staj_reader.md) and, provided that when it is constructed -the current stream event has type `begin_object`, it returns the elements -of the JSON object as items of type `std::pair`. If when it is constructed the current stream event -does not have type `begin_object`, it becomes equal to the default-constructed iterator. - -#### Member types - -Member type |Definition -------------------------------------|------------------------------ -`char_type`|`char` -`key_type`|`std::basic_string` -`value_type`|`std::pair` -`difference_type`|`std::ptrdiff_t` -`pointer`|`value_type*` -`reference`|`value_type&` -`iterator_category`|[std::input_iterator_tag](https://en.cppreference.com/w/cpp/iterator/iterator_tags) - -#### Constructors - - staj_object_iterator() noexcept; // (1) - - staj_object_iterator(basic_staj_reader& reader); // (2) - - staj_object_iterator(basic_staj_reader& reader, - std::error_code& ec); // (3) - -(1) Constructs the end iterator - -(2) Constructs a `staj_object_iterator` that refers to the first member of the object - following the current stream event `begin_object`. If there is no such member, - returns the end iterator. If a parsing error is encountered, throws a - [ser_error](ser_error.md). - -(3) Constructs a `staj_object_iterator` that refers to the first member of the object - following the current stream event `begin_object`. If there is no such member, - returns the end iterator. If a parsing error is encountered, returns the end iterator - and sets `ec`. - -#### Member functions - - const key_value_type& operator*() const - - const key_value_type* operator->() const - - staj_object_iterator& operator++(); - staj_object_iterator operator++(int); - staj_object_iterator& increment(std::error_code& ec); -Advances the iterator to the next object member. - -#### Non-member functions - - template - bool operator==(const staj_object_iterator& a, const staj_object_iterator& b) - - template - bool operator!=(const staj_object_iterator& a, const staj_object_iterator& b) - - template - staj_object_iterator begin(staj_object_iterator iter) noexcept; // (1) - - template - staj_object_iterator end(const staj_object_iterator&) noexcept; // (2) - -(1) Returns iter unchanged - -(2) Returns a default-constructed `stax_array_iterator`, which serves as an end iterator. The argument is ignored. - -The `begin` and `end` non-member functions enable the use of `stax_array_iterators` with range-based for loops. - -### Examples - -#### Iterate over a JSON object, returning key-json value pairs - -```c++ -const std::string example = R"( -{ - "application": "hiking", - "reputons": [ - { - "rater": "HikingAsylum.array_example.com", - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rating": 0.90 - } - ] -} -)"; - -int main() -{ - std::istringstream is(object_example); - - json_cursor reader(is); - - staj_object_iterator it(reader); - - for (const auto& kv : it) - { - std::cout << kv.first << ":\n" << pretty_print(kv.second) << "\n"; - } - std::cout << "\n\n"; -} -``` -Output: -``` -application: -"hiking" -reputons: -[ - { - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rater": "HikingAsylum.array_example.com", - "rating": 0.90 - } -] -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_reader.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_reader.md deleted file mode 100644 index 02a25a2891..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/staj_reader.md +++ /dev/null @@ -1,59 +0,0 @@ -### jsoncons::staj_reader - -```c++ -typedef basic_staj_reader staj_reader -``` - -#### Header -```c++ -#include -``` - -The `staj_reader` interface supports forward, read-only, access to JSON and JSON-like data formats. - -The `staj_reader` is designed to iterate over stream events until `done()` returns `true`. -The `next()` function causes the reader to advance to the next stream event. The `current()` function -returns the current stream event. The data can be accessed using the [staj_event](staj_event.md) -interface. When `next()` is called, copies of data previously accessed may be invalidated. - -#### Destructor - - virtual ~basic_staj_reader() = default; - -#### Member functions - - virtual bool done() const = 0; -Check if there are no more events. - - virtual const staj_event& current() const = 0; -Returns the current [staj_event](staj_event.md). - - virtual void accept(json_content_handler& handler) = 0; -Sends the parse events from the current event to the -matching completion event to the supplied [handler](json_content_handler.md) -E.g., if the current event is `begin_object`, sends the `begin_object` -event and all inbetween events until the matching `end_object` event. -If a parsing error is encountered, throws a [ser_error](ser_error.md). - - virtual void accept(json_content_handler& handler, - std::error_code& ec) = 0; -Sends the parse events from the current event to the -matching completion event to the supplied [handler](json_content_handler.md) -E.g., if the current event is `begin_object`, sends the `begin_object` -event and all inbetween events until the matching `end_object` event. -If a parsing error is encountered, sets `ec`. - - virtual void next() = 0; -Get the next event. If a parsing error is encountered, throws a [ser_error](ser_error.md). - - virtual void next(std::error_code& ec) = 0; -Get the next event. If a parsing error is encountered, sets `ec`. - - virtual const ser_context& context() const = 0; -Returns the current [context](ser_context.md) - -#### See also - -- [staj_array_iterator](staj_array_iterator.md) -- [staj_object_iterator](staj_object_iterator.md) - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/decode_ubjson.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/decode_ubjson.md deleted file mode 100644 index f212cbfcc6..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/decode_ubjson.md +++ /dev/null @@ -1,30 +0,0 @@ -### jsoncons::ubjson::decode_ubjson - -Decodes a [Universal Binary JSON Specification (JSON)](http://ubjson.org/) data format into a C++ data structure. - -#### Header -```c++ -#include - -template -T decode_ubjson(const std::vector& v); // (1) - -template -T decode_ubjson(std::istream>& is); // (2) -``` - -(1) Reads a UBJSON bytes buffer into a type T if T is an instantiation of [basic_json](../json.md) -or if T supports [json_type_traits](../json_type_traits.md). - -(2) Reads a UBJSON binary stream into a type T if T is an instantiation of [basic_json](../json.md) -or if T supports [json_type_traits](../json_type_traits.md). - -#### Exceptions - -Throws [ser_error](../ser_error.md) if parsing fails. - -#### See also - -- [encode_ubjson](encode_ubjson.md) encodes a json value to the [Universal Binary JSON Specification](http://ubjson.org/) data format. - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/diagrams/ubjson_encoder.png b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/diagrams/ubjson_encoder.png deleted file mode 100644 index c2dbc8fe89f67e29311f1b6eafdbcb9c8462923d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5346 zcmeI0`8!nc-~VR_McKj>p@ovQNQ|8<70JHFkbRlPmUZlF4Jl;_*;4j>8H^vC?Iv1FEnE?R6g+~uH zjQ{{3g!b%omXUVdxUp~o0MK^@n^^ck19g1ty1mVdJmT(neE?npkW`k0U7Y*d6u>qXSnd@ z`mie7vs=H%E{^*2)joa^@lc%3Yilwaybsw0*F#?kh)CnBd!mn7y~<+p!GD9gAAXng zuK8P8(BwM7zCJC>O%MsG{(SZmlVVs>bG+Ns;%P~_Tfwz~cFU8?-xQHBymCnwcF%>+ zyz&T}SiI?N9PTD$*>th}8{X~DWL6$WuPcczh`#Vlyd}siCMw+HRgy%iCt1BY*4eV? zZ`Ptd{v$u@ zF%Ol+P&4D2(FgN+8wZ^d&;3%I^6pRSoVZ_~Zfi^)qNkg1EQ7R4>4`8H;1VX^iOE_F+deOw0(e1v*CqXWOofr)kXSdk5c=6=HuMCZA^)RMJ!`x5WtA(&9_FK!BoqSyfpLWZLJ$CzP4>muG z@JeklKC2b^WB<{gDy8_&?)lAQ2Z+eQL%}vz@GO6c>9ETAxT7zM)mvWMkB%s{E+eo4 ze#qD2`mf90&AGwTd~h7=HK!ax&vSm$B)MJ zyyrtMS}i$D6f(2eUYqRs;*NnKFBsZfBJ(BnWIexkV?mWn;f(rePq5fO>dFwx@~l`3h8>Zt0#$=1 zTgwPss(&pdWXIYVpVk}-C%rDOM(uX@xqnlW%NC&goXGxRq`g$sCgUGa64-TL$G@OQ zVLceD9V1&Qb}!H#e9}n2KRy?+XWV z*AHa@0PN^Tn(8KjwzzC)ANM#-#Ig`-&UjeJRYm0WrZ-Wsa;@ZZ?38&XRJGY)d7X#CA{Q6Fb;Vfh@`4V?u%+4pp0LLG%xoZOn&p+OW&g3;r z*E%5*`4!E0B%fIXmm@$@{XI#1PD44$(mpGFBa@S%outi4Ct8+d9AUW^xuS1)K|fbT zf{9JwgAL06=WzZ5>)Ohch(xB+Z2JA`Ai-7a( z(2-KHIAJ((RFg8HnuKN`9o`UIRX4zYQVObVJw2iNoE!y#{niYOnhUxzRYHR~r%Q~B zp2a3IUwG=2t?D)+1P>$St?It2A z;Nnd^uNFbouxEb*weebKSxd!bC`Ur> z#cWg2ZiAJjptr=<(e5v)-R~El7MiWpd;^h3(6A1Pi?{SYu&GegXf;^%N#qkCo{c}A zlf%Go3@H&4sbLf4mbKxhVdhEG{p3kVc&a%h)U^F@qiiL+b%WPoDQB#owK4pNpo5g| zZtA-Wr*5NmX0N~(V?p1|+TBlSsBkd_)|YbEkK>#;E)@-OTLX z6ETPoqnu(M5`N$_-4=qZD^aB{x70P94b0#Ie2SGC?0_HCQrr7{tcHMkjW6ah#b4qm zmoEN-lDOF-+Cs?8H2(QpE<^nzUv%HcYfPNSmzk2n`#;=fJ@NMhm|x3sb{Q)+ zyZ~iNjHM^sU?eeN>^=4L)Hqjd+JeUqLTUBUr~okt3%K&4NU)y7%P%}#-H9JvQS%}2gGbHbIvN2wU{Vj*_d)p%`pA))2 zeWpEG<15TQ9tV>>jQG+a&n3!E|1fPEdpbA2zMfrRZoxVh^s}pVJ*Ed5;s); z3Lex_g2@z2l*yQQYE0Sq%^YnRYF*K|dl(UnYYiekCa2Ubc)U+Dvs@-KYbAYlWw}}w zuo0d-R2{nT4JgMkx@AFGa{2aD-6>>~@+q{~(%NZ(CA?mtkxij2eY`U~!wMBqgu7OR zwOWmvsb?G8={TKVe{WvV3xF_OR%3Rxte$DLuq*uJfTk~Q?B7qx>LmZXmFpML1#C*Z zETb(DvQ_i_O>7kjEeOi>R({<;(~j3~vOuV+Xk6O{1)#iLS5NQ?%@hb5nu+2+nM#eA z0m^wv(KP|et5CmP%z%?y)UWsksaf%bElxpgzXRGC!i@>n+A0QY&oWZ(`LcH)RP)N{qjMX?57iom$=?K zrM3ND@PMp*=>&c2C0lW98qMycisFZT=-@ABrSaotiSxHyK_UaAJ>66O-1(F1(P}Si zRk9TWDi;~D*dXPzxXAm6&`S*?FNO)3UFMNrxp`HV-CWB)UN^nY88OEiT>?LzW1Xmc zQ-!+i=Y|o5BfiI+q(ctAs?W#6CI*>07>tGCkPuu$Tzn4Xz_}CYip^q}x{i{19#K{C z!4r$c99dS-cgV3&Obid)vmT@qtqk42#1C0k*RboBU4>%N%IJpp&D6W@U^semp&SST|&T#+acHZF+EskF{+ASFiM!!Cwl~L!hThRjVGp zk&WGUVi>88Bcq()jgsq#!p3ZwsZ%2(u_>Ag%}K07Qz_Ffxx%TSZ&C9?Y0X2br|@o0 zg^E=K598qE7w>m+q{e_(>5c9~J6{SwnG-~3p^-gPG~W1NCXhF| zD!!3@B)Gj4T=~aV_9n++XX9!aI7YBSX>a7FvoU36j?_b5T%bEvBctnDBnIu_>3VxMcs?b-FF zmN?Hq7(yIBp6ZVf<$xXH1eQe|Yg02-bLK{^V=F{^*UUPI_W_j?NeD75}0(p#o96gyI(nT!^fk#~D06pK~RN zMlknESmlPDPy3b-Qtfa5sb?dgWj)p!di{@=8x4|1QV3C{+^&q6Qd}y3GG3zW(vVaE z4UaU1b4JZ*@y7mNo%E(Oh`M+8ZnGAp=L58-^xte)8pPBn6FGDW2GO)ZE)+w>ZfW<%z%cZnnK4#layN*;SE&1CuDoCx1pW)L{*RC&_7_;~ADuzkPc6NFC#O6i<RYE!>f2B#h zPvVp7rx2E9=1_rz0!12s{)g$)x|)1f-;`U_Xz)nciqjBQ#1Sd{K*H9Kg0}|;?G}1s z+HEE(Z8ndx??bH+p9#{U`0#%_G%kc|O$c5=DDYSOC>^^~X@hcH8_xf9?}@cC&5fX! z&%F>ka#cdDzCO*} zuB7qyD}bH+op~CcY!51XFU7$3{^&@DpTNoF_E2hyw{{Y*FbhcX!*XwO(G36H=P`dE^9t=`d--rA-g3jo`bMly*SR>Yk9=XxmIpMOmYIPQ z0IcR!wl7z0ntim=q#%;0fE^wduUK|F)(Dpd{=Ll0GR{pkKofqcp=}%*$!7(^yKPO? zNIUg^FbQ9XR_2$(F{ymOLbl5%lj^dwr~Z;p7S-bqtpP+kn2Kl^k)zA z>IXo)UMz0>}m7+88R+!uE=|-{iF2OrvB)vj{3<)#-4#XkNm3r2JQuF^bYP< zm-s`f)U1J_2Opz@yS3Bm;HN=N9=Jp+qDP0~Pr`jzS6W181@W7hn08K@MQpr{l}KB* z#E%OrUX3HZ)=;h5eM0O7l|IBGJZKae@UeeV<@>PyZBD^O#p3lVTR&o8)g>iEHKGe1 zA-QV^ch#w?tB5WTV5Y`FXM{XZ3Lo9lc+@&vC&7u%ZGTGV2r;LnXG1%UFffCQ0pE)AY`W|WM}pnP8#6^DE=31HoF zS2IOy0vg0Ae{t^Bzu=@>(C}JiHh(Fb79o#`@<;mzrMWFK4x?Y6&>ZnQ$Bw#2c{}>> z7xS$j_ux=;XQRe0Z9LSTw)@ike@k~V0gB|YrIn|90-GL!+)ZfT07H1@wlc#}y6fzF zbO^Xo^m-RkR2a-kQ)QF97@Az4s|ghSPeYWp2TMP!$kPcZcbO8(O*<)R2a YaANxTfP)I7{ksG_($d$g)Ub>CFVEt0ivR!s diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/diagrams/ubjson_encoder.xml b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/diagrams/ubjson_encoder.xml deleted file mode 100644 index 4698371f69..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/diagrams/ubjson_encoder.xml +++ /dev/null @@ -1 +0,0 @@ -vVbfb9owEP5r8tgpiQuljy3QbkNU65gEfUImvibeHJs5BsL++tmJTX6iUqlqHyrfd76z/d13Fzw0TvNHibfJXBBgXuiT3EMTLwxvgpH+b4BjCaChXwKxpKSEggpY0H9gQbdtRwlkjY1KCKbotglGgnOIVAPDUopDc9urYM1TtziGDrCIMOuiS0pUUqKjgV/hX4HGiTs58K0nxW6zBbIEE3GoQWjqobEUQpWrNB8DM9w5Xsq4hzPe08UkcHVJwPTpV85nz4skHP+dqeU8fcm/X9kse8x29sEbnNFo/TsTfK05VTr5OsGcMJBeOGT6pPuNWcVm1YcUT1VHx58UO07AXMHX7kNCFSy2ODLegxaMxhKVMm0Fp+j6k9z9QCrIa5B94iOIFJQ86i3WO7JsW7kFjv1DVbzASTCpFe7aYtjqJT5lrijVC8vqOxi+OcPwblNwDDzSLfMJ3L5SxsaCCVnEIij+NJ4pKf5AzYOG6BaRj6nGReUYfGY5Rp1ydGgFTu7M5NBWxHCmi9VkEnKqVpZ0s34x6y8Da03ymmtydAbXl1+5BMaoRRmzCissF3e2BpnYyQje1p3CMgb19gQA0piD3YrWKtZXMIdJYFjRfXN69lXRnvBDUP2yk2Cum3oJUUsH5bNtUH3YtfLctnQ3auUpaenkKSR1evRFKov38yScPa3wT4ivls8z2M+gp+m/8QQkVZjroumvIcWxxGlHerqtVFNszfbkgkOrly2EGY25UawWih4n6N40KdXfsTvrSCkh5pjeOdGcJB/R+H6rkH5P46MeHYXvb3xtVl/RsoLVTxE0/Q8= \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/encode_ubjson.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/encode_ubjson.md deleted file mode 100644 index d49a52b7a8..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/encode_ubjson.md +++ /dev/null @@ -1,25 +0,0 @@ -### jsoncons::ubjson::encode_ubjson - -Encodes a C++ data structure to the [Universal Binary JSON Specification (UBJSON)](http://ubjsonspec.org/) data format. - -#### Header -```c++ -#include - -template -void encode_ubjson(const T& jval, std::vector& v); // (1) - -template -void encode_ubjson(const T& jval, std::ostream& os); // (2) -``` - -(1) Writes a value of type T into a bytes buffer in the UBJSON data format. Type T must be an instantiation of [basic_json](../json.md) -or support [json_type_traits](../json_type_traits.md). - -(2) Writes a value of type T into a binary stream in the UBJSON data format. Type T must be an instantiation of [basic_json](../json.md) -or support [json_type_traits](../json_type_traits.md). - -#### See also - -- [decode_ubjson](decode_ubjson) decodes a [Binary JSON](http://ubjsonspec.org/) data format to a json value. - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/ubjson.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/ubjson.md deleted file mode 100644 index 82484b5d51..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/ubjson.md +++ /dev/null @@ -1,179 +0,0 @@ -### ubjson extension - -The ubjson extension implements encode to and decode from the [Universal Binary JSON Specification](http://ubjson.org/) data format. -You can either parse into or serialize from a variant-like structure, [basic_json](../json.md), or your own -data structures, using [json_type_traits](../json_type_traits.md). - -[decode_ubjson](decode_ubjson.md) - -[encode_ubjson](encode_ubjson.md) - -[ubjson_encoder](ubjson_encoder.md) - -#### jsoncons-ubjson mappings - -jsoncons data item|jsoncons tag|UBJSON data item ---------------|------------------|--------------- -null | | null -bool | | true or false -int64 | | uint8_t or integer -uint64 | | uint8_t or integer -double | | float 32 or float 64 -string | | string -string | bigint | high precision number type -string | bigdec | high precision number type -byte_string | | array of uint8_t -array | | array -object | | object - -### Examples - -#### encode/decode UBJSON from/to basic_json - -```c++ -#include -#include -#include - -using namespace jsoncons; - -int main() -{ - ojson j1 = ojson::parse(R"( - { - "application": "hiking", - "reputons": [ - { - "rater": "HikingAsylum.example.com", - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rating": 0.90 - } - ] - } - )"); - - // Encode a basic_json value to UBJSON - std::vector data; - ubjson::encode_ubjson(j1, data); - - // Decode UBJSON to a basic_json value - ojson j2 = ubjson::decode_ubjson(data); - std::cout << "(1)\n" << pretty_print(j2) << "\n\n"; - - // Accessing the data items - - const ojson& reputons = j2["reputons"]; - - std::cout << "(2)\n"; - for (auto element : reputons.array_range()) - { - std::cout << element.at("rated").as() << ", "; - std::cout << element.at("rating").as() << "\n"; - } - std::cout << std::endl; - - // Get a UBJSON value for a nested data item with jsonpointer - std::error_code ec; - const auto& rated = jsonpointer::get(j2, "/reputons/0/rated", ec); - if (!ec) - { - std::cout << "(3) " << rated.as_string() << "\n"; - } - - std::cout << std::endl; -} -``` -Output: -``` -(1) -{ - "application": "hiking", - "reputons": [ - { - "rater": "HikingAsylum.example.com", - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rating": 0.9 - } - ] -} - -(2) -Marilyn C, 0.9 - -(3) Marilyn C -``` - -#### encode/decode UBJSON from/to your own data structures - -```c++ -#include -#include -namespace ns { - struct reputon - { - std::string rater; - std::string assertion; - std::string rated; - double rating; - - friend bool operator==(const reputon& lhs, const reputon& rhs) - { - return lhs.rater == rhs.rater && lhs.assertion == rhs.assertion && - lhs.rated == rhs.rated && lhs.rating == rhs.rating; - } - - friend bool operator!=(const reputon& lhs, const reputon& rhs) - { - return !(lhs == rhs); - }; - }; - - class reputation_object - { - std::string application; - std::vector reputons; - - // Make json_type_traits specializations friends to give accesses to private members - JSONCONS_TYPE_TRAITS_FRIEND; - public: - reputation_object() - { - } - reputation_object(const std::string& application, const std::vector& reputons) - : application(application), reputons(reputons) - {} - - friend bool operator==(const reputation_object& lhs, const reputation_object& rhs) - { - return (lhs.application == rhs.application) && (lhs.reputons == rhs.reputons); - } - - friend bool operator!=(const reputation_object& lhs, const reputation_object& rhs) - { - return !(lhs == rhs); - }; - }; - -} // ns - -// Declare the traits. Specify which data members need to be serialized. -JSONCONS_MEMBER_TRAITS_DECL(ns::reputon, rater, assertion, rated, rating) -JSONCONS_MEMBER_TRAITS_DECL(ns::reputation_object, application, reputons) - -int main() -{ - ns::reputation_object val("hiking", { ns::reputon{"HikingAsylum.example.com","strong-hiker","Marilyn C",0.90} }); - - // Encode a ns::reputation_object value to UBJSON - std::vector data; - ubjson::encode_ubjson(val, data); - - // Decode UBJSON to a ns::reputation_object value - ns::reputation_object val2 = ubjson::decode_ubjson(data); - - assert(val2 == val); -} -``` - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/ubjson_encoder.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/ubjson_encoder.md deleted file mode 100644 index 5cc6f98c9f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/ubjson/ubjson_encoder.md +++ /dev/null @@ -1,100 +0,0 @@ -### jsoncons::ubjson::basic_ubjson_encoder - -```c++ -template< - class Result> -> class basic_ubjson_encoder : public jsoncons::json_content_handler -``` - -`basic_ubjson_encoder` is noncopyable and nonmoveable. - -#### Header - - #include - -![ubjson_encoder](./diagrams/ubjson_encoder.png) - -Four specializations for common character types and result types are defined: - -Type |Definition ----------------------------|------------------------------ -ubjson_encoder |basic_ubjson_encoder -ubjson_bytes_encoder |basic_ubjson_encoder - -#### Member types - -Type |Definition ----------------------------|------------------------------ -char_type |char -result_type |Result -string_view_type | - -#### Constructors - - explicit basic_ubjson_encoder(result_type result) -Constructs a new encoder that writes to the specified result. - -#### Destructor - - virtual ~basic_ubjson_encoder() - -### Inherited from [basic_json_content_handler](../json_content_handler.md) - -#### Member functions - - bool begin_object(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool begin_object(size_t length, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool end_object(const ser_context& context = null_ser_context()) - - bool begin_array(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool begin_array(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool end_array(const ser_context& context=null_ser_context()); - - bool name(const string_view_type& name, - const ser_context& context=null_ser_context()); - - bool string_value(const string_view_type& value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool byte_string_value(const byte_string_view& b, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool byte_string_value(const uint8_t* p, size_t size, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool int64_value(int64_t value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool uint64_value(uint64_t value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool double_value(double value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool bool_value(bool value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - bool null_value(semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()); - - void flush() - -### Examples - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wjson.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wjson.md deleted file mode 100644 index cadd491d54..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wjson.md +++ /dev/null @@ -1,27 +0,0 @@ -### jsoncons::wjson - -```c++ -typedef basic_json> wjson -``` -The `wjson` class is an instantiation of the `basic_json` class template that uses `wchar_t` as the character type. The order of an object's name/value pairs is not preserved, they are sorted alphabetically by name. If you want to preserve the original insertion order, use [wojson](wojson.md) instead. - -The `jsoncons` library will always rebind the supplied allocator from the template parameter to internal data structures. - -#### Header -```c++ -#include -``` -#### Interface - -The interface is the same as [json](json.md), substituting wide character instantiations of classes - `std::wstring`, `std::wistream`, etc. - for utf8 character ones. - -#### See also - -- [wojson](wojson.md) constructs a wide character json value that preserves the original insertion order of an object's name/value pairs - -- [json](json.md) constructs a utf8 character json value that sorts name-value members alphabetically - -- [ojson](ojson.md) constructs a utf8 character json value that preserves the original insertion order of an object's name/value pairs - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wjson_encoder.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wjson_encoder.md deleted file mode 100644 index a21102c873..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wjson_encoder.md +++ /dev/null @@ -1,15 +0,0 @@ -### jsoncons::wjson_encoder - -```c++ -typedef basic_json_encoder wjson_encoder -``` - -The `wjson_encoder` class is an instantiation of the `basic_json_encoder` class template that uses `wchar_t` as the character type. It implements [wjson_content_handler](basic_json_content_handler.md) and supports pretty print serialization. - -#### Header - - #include - -#### Interface - -The interface is the same as [json_encoder](json_encoder.md), substituting wide character instantiations of classes - `std::wstring`, etc. - for utf8 character ones. diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wjson_options.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wjson_options.md deleted file mode 100644 index d61ec79c81..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wjson_options.md +++ /dev/null @@ -1,16 +0,0 @@ -### jsoncons::wjson_options - -```c++ -typedef basic_json_options wjson_options -``` -The wjson_options class is an instantiation of the basic_json_options class template that uses `wchar_t` as the character type. - -#### Header - -```c++ -#include -``` - -#### Interface - -The interface is the same as [json_options](json_options.md), substituting wide character instantiations of classes - `std::wstring`, etc. - for utf8 character ones. diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wjson_reader.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wjson_reader.md deleted file mode 100644 index 78dc37358f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wjson_reader.md +++ /dev/null @@ -1,16 +0,0 @@ -### jsoncons::wjson_reader - -```c++ - typedef basic_json_reader> wjson_reader -``` -The `wjson_reader` class is an instantiation of the `basic_json_reader` class template that uses `wchar_t` as the character type -and `stream_source` as the input source. - -#### Header - - #include - -#### Interface - -The interface is the same as [json_reader](json_reader.md), substituting wide character instantiations of classes - `std::wstring`, `std::wistream`, etc. - for utf8 character ones. - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wojson.md b/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wojson.md deleted file mode 100644 index bb7bdd997e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/doc/ref/wojson.md +++ /dev/null @@ -1,31 +0,0 @@ -### jsoncons::wojson - -```c++ -typedef basic_json> wojson -``` -The `wojson` class is an instantiation of the `basic_json` class template that uses `wchar_t` as the character type. The original insertion order of an object's name/value pairs is preserved. - -The `jsoncons` library will always rebind the supplied allocator from the template parameter to internal data structures. - -#### Header -```c++ -#include -``` -#### Interface - -The interface is the same as [wjson](wjson.md), substituting wide character instantiations of classes - `std::wstring`, `std::wistream`, etc. - for utf8 character ones. - -- `wojson`, like `wjson`, supports object member `insert_or_assign` methods that take an `object_iterator` as the first parameter. But while with `wjson` that parameter is just a hint that allows optimization, with `wojson` it is the actual location where to insert the member. - -- In `wojson`, the `insert_or_assign` members that just take a name and a value always insert the member at the end. - -#### See also - -- [wjson](wjson.md) constructs a wide character json value that sorts name-value members alphabetically - -- [json](json.md) constructs a utf8 character json value that sorts name-value members alphabetically - -- [ojson](ojson.md) constructs a utf8 character json value that preserves the original insertion order of an object's name/value pairs - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/address-book.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/address-book.json deleted file mode 100644 index c3d5d98369..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/address-book.json +++ /dev/null @@ -1,13 +0,0 @@ - { - "address-book" : - [ - { - "name":"Jane Roe", - "email":"jane.roe@example.com" - }, - { - "name":"John", - "email" : "john.doe@example.com" - } - ] - } diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/booklist.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/booklist.json deleted file mode 100644 index ec78a4216b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/booklist.json +++ /dev/null @@ -1,28 +0,0 @@ -{ "store": { - "book": [ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - }, - { "category": "fiction", - "author": "Herman Melville", - "title": "Moby Dick", - "isbn": "0-553-21311-3", - "price": 8.99 - }, - { "category": "fiction", - "author": "J. R. R. Tolkien", - "title": "The Lord of the Rings", - "isbn": "0-395-19395-8", - "price": 22.99 - } - ] - } -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/books.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/books.json deleted file mode 100644 index ae9677c2e3..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/books.json +++ /dev/null @@ -1,30 +0,0 @@ -[ - { - "title" : "Kafka on the Shore", - "author" : "Haruki Murakami", - "price" : 25.17, - "reviews" : - [ - {"rating" : "*****"}, - {"rating" : "*****"} - ] - }, - { - "title" : "Women: A Novel", - "author" : "Charles Bukowski", - "price" : 12.00, - "reviews" : - [ - {"rating" : "*****"}, - {"rating" : "*"} - ] - }, - { - "title" : "Cutter's Way", - "author" : "Ivan Passer", - "reviews" : - [ - {"rating" : "****"} - ] - } -] diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/countries.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/countries.json deleted file mode 100644 index 29629e4253..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/countries.json +++ /dev/null @@ -1,7 +0,0 @@ -[ - ["country_code","name"], - ["ABW","ARUBA"], - ["ATF","FRENCH SOUTHERN TERRITORIES, D.R. OF"], - ["VUT","VANUATU"], - ["WLF","WALLIS & FUTUNA ISLANDS"] -] diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/employees.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/employees.json deleted file mode 100644 index a4837466b4..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/employees.json +++ /dev/null @@ -1,30 +0,0 @@ -[ - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - } -] diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/multiple-json-objects.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/multiple-json-objects.json deleted file mode 100644 index c5516582ed..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/multiple-json-objects.json +++ /dev/null @@ -1,3 +0,0 @@ -{"a":1,"b":2,"c":3} -{"a":4,"b":5,"c":6} -{"a":7,"b":8,"c":9} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/sales.csv b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/sales.csv deleted file mode 100644 index b5177e6983..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/sales.csv +++ /dev/null @@ -1,6 +0,0 @@ -customer_name,has_coupon,phone_number,zip_code,sales_tax_rate,total_amount -"John Roe",true,0272561313,01001,0.05,431.65 -"Jane Doe",false,416-272-2561,55416,0.15,480.70 -"Joe Bloggs",false,"4162722561","55416",0.15,300.70 -"John Smith",FALSE,NULL,22313-1450,0.15,300.70 - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/tasks.csv b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/tasks.csv deleted file mode 100644 index facd635164..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/input/tasks.csv +++ /dev/null @@ -1,6 +0,0 @@ -project_id, task_name, task_start, task_finish -4001,task1,01/01/2003,01/31/2003 -4001,task2,02/01/2003,02/28/2003 -4001,task3,03/01/2003,03/31/2003 -4002,task1,04/01/2003,04/30/2003 -4002,task2,05/01/2003, diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/booklist.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/booklist.json deleted file mode 100644 index 29293a53c7..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/booklist.json +++ /dev/null @@ -1,37 +0,0 @@ -[ - { - "author": "Haruki Murakami", - "category": "Fiction", - "date": "2006-01-03", - "isbn": "1400079276", - "price": 13.45, - "title": "Kafka on the Shore" - }, - { - "author": "Charles Bukowski", - "category": "Fiction", - "date": "2004-07-08", - "isbn": "1852272007", - "price": 22.48, - "ratings": { - "*****": 4 - }, - "title": "Pulp" - }, - { - "author": "Haruki Murakami", - "category": "Fiction", - "date": "2002-04-09", - "isbn": "037571894X", - "price": 9.01, - "title": "A Wild Sheep Chase: A Novel" - }, - { - "author": "George Crile", - "category": "History", - "date": "2007-11-06", - "isbn": "0802143415", - "price": 10.5, - "title": "Charlie Wilson's War" - } -] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/booklist2.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/booklist2.json deleted file mode 100644 index 29293a53c7..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/booklist2.json +++ /dev/null @@ -1,37 +0,0 @@ -[ - { - "author": "Haruki Murakami", - "category": "Fiction", - "date": "2006-01-03", - "isbn": "1400079276", - "price": 13.45, - "title": "Kafka on the Shore" - }, - { - "author": "Charles Bukowski", - "category": "Fiction", - "date": "2004-07-08", - "isbn": "1852272007", - "price": 22.48, - "ratings": { - "*****": 4 - }, - "title": "Pulp" - }, - { - "author": "Haruki Murakami", - "category": "Fiction", - "date": "2002-04-09", - "isbn": "037571894X", - "price": 9.01, - "title": "A Wild Sheep Chase: A Novel" - }, - { - "author": "George Crile", - "category": "History", - "date": "2007-11-06", - "isbn": "0802143415", - "price": 10.5, - "title": "Charlie Wilson's War" - } -] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/export_settings.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/export_settings.json deleted file mode 100644 index 8f1ad880ad..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/export_settings.json +++ /dev/null @@ -1 +0,0 @@ -{"File Format Options":{"Color Spaces":["sRGB","AdobeRGB","ProPhoto RGB"],"Image Formats":["JPEG","PSD","TIFF","DNG"]},"File Settings":{"Color Space":"sRGB","Image Format":"JPEG","Limit File Size":true,"Limit File Size To":10000},"Image Sizing":{"Dimension 1":9.84,"Resize To Fit":true,"Resize Unit":"pixels","Resize What":"long_edge"}} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/new-address-book1.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/new-address-book1.json deleted file mode 100644 index d390928d82..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/new-address-book1.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "address-book": [ - { - "first-name": "Jane", - "last-name": "Roe", - "email": "jane.roe@example.com" - }, - { - "first-name": "John", - "email": "john.doe@example.com" - } - ] -} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/new-address-book2.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/new-address-book2.json deleted file mode 100644 index 628c37a1c8..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/new-address-book2.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "address-book": [ - { - "email": "jane.roe@example.com", - "first-name": "Jane", - "last-name": "Roe" - }, - { - "email": "john.doe@example.com", - "first-name": "John" - } - ] -} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/tasks.csv b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/tasks.csv deleted file mode 100644 index de05a3ee4c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/tasks.csv +++ /dev/null @@ -1,6 +0,0 @@ -(1) -project_id,task_name,task_start,task_finish -4001,task2,02/01/2003,02/28/2003 -4001,task3,03/01/2003,03/31/2003 -4002,task1,04/01/2003,04/30/2003 -4002,task2,05/01/2003, diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/tasks.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/tasks.json deleted file mode 100644 index d61c59d14d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/tasks.json +++ /dev/null @@ -1,33 +0,0 @@ -(1) -[ - { - "project_id": 4001, - "task_name": "task1", - "task_start": "01/01/2003", - "task_finish": "01/31/2003" - }, - { - "project_id": 4001, - "task_name": "task2", - "task_start": "02/01/2003", - "task_finish": "02/28/2003" - }, - { - "project_id": 4001, - "task_name": "task3", - "task_start": "03/01/2003", - "task_finish": "03/31/2003" - }, - { - "project_id": 4002, - "task_name": "task1", - "task_start": "04/01/2003", - "task_finish": "04/30/2003" - }, - { - "project_id": 4002, - "task_name": "task2", - "task_start": "05/01/2003" - } -] - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/xxx.txt b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/xxx.txt deleted file mode 100644 index bed26dd937..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/output/xxx.txt +++ /dev/null @@ -1,3 +0,0 @@ -[ - "\u007F\u07FF\u0800" -] diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/array_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/array_examples.cpp deleted file mode 100644 index 85be172281..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/array_examples.cpp +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include - -using namespace jsoncons; - -void array_example1() -{ - { - std::vector v = {1,2,3,4}; - json a(v); - std::cout << a << std::endl; - } - { - json j = json::array{1,true,"last"}; - auto d = j.as>(); - for (auto x : d) - { - std::cout << x << std::endl; - } - } - { - std::map m = {{"one",1},{"two",2},{"three",3}}; - json j(m); - std::cout << j << std::endl; - } - { - json j; - j["one"] = 1; - j["two"] = 2; - j["three"] = 3; - - auto um = j.as>(); - for (const auto& x : um) - { - std::cout << x.first << "=" << x.second << std::endl; - } - } -} - -void accessing_a_json_value_as_a_vector() -{ - std::string s = "{\"my-array\" : [1,2,3,4]}"; - json val = json::parse(s); - std::vector v = val["my-array"].as>(); - for (size_t i = 0; i < v.size(); ++i) - { - if (i > 0) - { - std::cout << ","; - } - std::cout << v[i]; - } - std::cout << std::endl; -} - -void construct_json_from_vector() -{ - json root; - - std::vector addresses; - json address1; - address1["city"] = "San Francisco"; - address1["state"] = "CA"; - address1["zip"] = "94107"; - address1["country"] = "USA"; - addresses.push_back(address1); - - json address2; - address2["city"] = "Sunnyvale"; - address2["state"] = "CA"; - address2["zip"] = "94085"; - address2["country"] = "USA"; - addresses.push_back(address2); - - root["addresses"] = addresses; - - std::cout << pretty_print(root) << std::endl; - - std::cout << "size=" << root["addresses"].size() << std::endl; - for (size_t i = 0; i < root["addresses"].size(); ++i) - { - std::cout << root["addresses"][i] << std::endl; - } -} - -void add_element_to_array() -{ - json cities = json::array(); // an empty array - std::cout << cities << std::endl; // output is "[]" - cities.push_back("Toronto"); - cities.push_back("Vancouver"); - cities.insert(cities.array_range().begin(),"Montreal"); // inserts "Montreal" at beginning of array - - std::cout << cities << std::endl; -} - -void reserve_array_capacity() -{ - json cities = json::array(); - cities.reserve(10); // storage is allocated - std::cout << "capacity=" << cities.capacity() << ", size=" << cities.size() << std::endl; - - cities.push_back("Toronto"); - cities.push_back("Vancouver"); - cities.insert(cities.array_range().begin(),"Montreal"); - std::cout << "capacity=" << cities.capacity() << ", size=" << cities.size() << std::endl; - - std::cout << cities << std::endl; -} - -void make_empty_array() -{ - std::cout << "empty array" <() << std::endl; - } -} - -void make_1_dimensional_array_1() -{ - std::cout << "1 dimensional array 1" <(10); - a[1] = 1; - a[2] = 2; - std::cout << pretty_print(a) << std::endl; -} - -void make_1_dimensional_array_2() -{ - std::cout << "1 dimensional array 2" <(10,0); - a[1] = 1; - a[2] = 2; - a[3] = json::array(); - std::cout << pretty_print(a) << std::endl; -} - -void make_2_dimensional_array() -{ - std::cout << "2 dimensional array" <(3,4,0); - a[0][0] = "Tenor"; - a[0][1] = "ATM vol"; - a[0][2] = "25-d-MS"; - a[0][3] = "25-d-RR"; - a[1][0] = "1Y"; - a[1][1] = 0.20; - a[1][2] = 0.009; - a[1][3] = -0.006; - a[2][0] = "2Y"; - a[2][1] = 0.18; - a[2][2] = 0.009; - a[2][3] = -0.005; - - std::cout << pretty_print(a) << std::endl; -} - -void make_3_dimensional_array() -{ - std::cout << "3 dimensional array" <(4,3,2,0); - - double val = 1.0; - for (size_t i = 0; i < a.size(); ++i) - { - for (size_t j = 0; j < a[i].size(); ++j) - { - for (size_t k = 0; k < a[i][j].size(); ++k) - { - a[i][j][k] = val; - val += 1.0; - } - } - } - std::cout << pretty_print(a) << std::endl; -} - -void array_examples() -{ - std::cout << "\nArray examples\n\n"; - array_example1(); - - construct_json_from_vector(); - add_element_to_array(); - reserve_array_capacity(); - accessing_a_json_value_as_a_vector(); - make_empty_array(); - array_range_based_for_loop(); - make_1_dimensional_array_1(); - make_1_dimensional_array_2(); - make_2_dimensional_array(); - make_3_dimensional_array(); - - std::cout << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/basics_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/basics_examples.cpp deleted file mode 100644 index 814861b32a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/basics_examples.cpp +++ /dev/null @@ -1,171 +0,0 @@ -// jsoncons_test.cpp : Defines the entry point for the console application. -// - -#include -#include -#include - -using namespace jsoncons; -using namespace jsoncons::jsonpath; - -void basics_json_example1() -{ - // Construct a book object - json book1; - - book1["category"] = "Fiction"; - book1["title"] = "A Wild Sheep Chase: A Novel"; - book1["author"] = "Haruki Murakami"; - book1["date"] = "2002-04-09"; - book1["price"] = 9.01; - book1["isbn"] = "037571894X"; - - // Construct another using the member function insert_or_assign - json book2; - - book2.insert_or_assign("category", "History"); - book2.insert_or_assign("title", "Charlie Wilson's War"); - book2.insert_or_assign("author", "George Crile"); - book2.insert_or_assign("date", "2007-11-06"); - book2.insert_or_assign("price", 10.50); - book2.insert_or_assign("isbn", "0802143415"); - - // Use insert_or_assign again, but more efficiently - json book3; - - // Reserve memory, to avoid reallocations - book3.reserve(6); - - // Insert in name alphabetical order - // Give insert_or_assign a hint where to insert the next member - auto hint = book3.insert_or_assign(book3.object_range().begin(),"author", "Haruki Murakami"); - hint = book3.insert_or_assign(hint, "category", "Fiction"); - hint = book3.insert_or_assign(hint, "date", "2006-01-03"); - hint = book3.insert_or_assign(hint, "isbn", "1400079276"); - hint = book3.insert_or_assign(hint, "price", 13.45); - hint = book3.insert_or_assign(hint, "title", "Kafka on the Shore"); - - // Construct a fourth from a string - json book4 = json::parse(R"( - { - "category" : "Fiction", - "title" : "Pulp", - "author" : "Charles Bukowski", - "date" : "2004-07-08", - "price" : 22.48, - "isbn" : "1852272007" - } - )"); - - // Construct a booklist array - json booklist = json::array(); - - // For efficiency, reserve memory, to avoid reallocations - booklist.reserve(4); - - // For efficency, tell jsoncons to move the contents - // of the four book objects into the array - booklist.push_back(std::move(book1)); - booklist.push_back(std::move(book2)); - - // Add the third one to the front - auto where = booklist.insert(booklist.array_range().begin(),std::move(book3)); - - // Add the last one immediately after - booklist.insert(where+1,std::move(book4)); - - // See what's left of book1, 2, 3 and 4 (expect nulls) - std::cout << book1 << "," << book2 << "," << book3 << "," << book4 << std::endl; - - //Loop through the booklist elements using a range-based for loop - for(auto book : booklist.array_range()) - { - std::cout << book["title"].as() - << "," - << book["price"].as() << std::endl; - } - - // The second book - json& book = booklist[1]; - - //Loop through the book members using a range-based for loop - for(auto member : book.object_range()) - { - std::cout << member.key() - << "," - << member.value() << std::endl; - } - - auto it = book.find("author"); - if (it != book.object_range().end()) - { - // member "author" found - } - - if (book.contains("author")) - { - // book has member "author" - } - - std::string s = book.get_with_default("author", "author unknown"); - // Returns author if found, otherwise "author unknown" - - try - { - book["ratings"].as(); - } - catch (const std::out_of_range&) - { - // member "ratings" not found - } - - // Add ratings - book["ratings"]["*****"] = 4; - book["ratings"]["*"] = 1; - - // Delete one-star ratings - book["ratings"].erase("*"); - - // Serialize the booklist to a file - std::ofstream os("./output/booklist.json"); - os << pretty_print(booklist); -} - -void basics_json_example2() -{ - // Deserialize the booklist - std::ifstream is("./output/booklist.json"); - json booklist; - is >> booklist; - - // Use a JSONPath expression to find - - // (1) The authors of books that cost less than $12 - json result = json_query(booklist, "$[*][?(@.price < 12)].author"); - std::cout << "(1) " << result << std::endl; - - // (2) The number of books - result = json_query(booklist, "$.length"); - std::cout << "(2) " << result << std::endl; - - // (3) The third book - result = json_query(booklist, "$[2]"); - std::cout << "(3) " << std::endl << pretty_print(result) << std::endl; - - // (4) The authors of books that were published in 2004 - result = json_query(booklist, "$[*][?(@.date =~ /2004.*?/)].author"); - std::cout << "(4) " << result << std::endl; - - // (5) The titles of all books that have ratings - result = json_query(booklist, "$[*][?(@.ratings)].title"); - std::cout << "(5) " << result << std::endl; -} - -void basics_examples() -{ - std::cout << "\nBasics\n\n"; - basics_json_example1(); - basics_json_example2(); - std::cout << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/basics_wexamples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/basics_wexamples.cpp deleted file mode 100644 index 4510198f9e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/basics_wexamples.cpp +++ /dev/null @@ -1,170 +0,0 @@ -// jsoncons_test.cpp : Defines the entry point for the console application. -// - -#include -#include -#include - -using namespace jsoncons; - -void basics_wjson_example1() -{ - // Construct a book object - wjson book1; - - book1[L"category"] = L"Fiction"; - book1[L"title"] = L"A Wild Sheep Chase: A Novel"; - book1[L"author"] = L"Haruki Murakami"; - book1[L"date"] = L"2002-04-09"; - book1[L"price"] = 9.01; - book1[L"isbn"] = L"037571894X"; - - // Construct another using the member function insert_or_assign - wjson book2; - - book2.insert_or_assign(L"category", L"History"); - book2.insert_or_assign(L"title", L"Charlie Wilson's War"); - book2.insert_or_assign(L"author", L"George Crile"); - book2.insert_or_assign(L"date", L"2007-11-06"); - book2.insert_or_assign(L"price", 10.50); - book2.insert_or_assign(L"isbn", L"0802143415"); - - // Use insert_or_assign again, but more efficiently - wjson book3; - - // Reserve memory, to avoid reallocations - book3.reserve(6); - - // Insert in name alphabetical order - // Give insert_or_assign a hint where to insert the next member - auto hint = book3.insert_or_assign(book3.object_range().begin(), L"author", L"Haruki Murakami"); - hint = book3.insert_or_assign(hint, L"category", L"Fiction"); - hint = book3.insert_or_assign(hint, L"date", L"2006-01-03"); - hint = book3.insert_or_assign(hint, L"isbn", L"1400079276"); - hint = book3.insert_or_assign(hint, L"price", 13.45); - hint = book3.insert_or_assign(hint, L"title", L"Kafka on the Shore"); - - // Construct a fourth from a string - - wjson book4 = wjson::parse(LR"( - { - "category" : "Fiction", - "title" : "Pulp", - "author" : "Charles Bukowski", - "date" : "2004-07-08", - "price" : 22.48, - "isbn" : "1852272007" - } - )"); - // Construct a booklist array - wjson booklist = wjson::array(); - - // For efficiency, reserve memory, to avoid reallocations - booklist.reserve(4); - - // For efficency, tell jsoncons to move the contents - // of the four book objects into the array - booklist.push_back(std::move(book1)); - booklist.push_back(std::move(book2)); - - // Add the third one to the front - auto where = booklist.insert(booklist.array_range().begin(),std::move(book3)); - - // Add the last one immediately after - booklist.insert(where+1,std::move(book4)); - - // See what's left of book1, 2, 3 and 4 (expect nulls) - std::wcout << book1 << L"," << book2 << L"," << book3 << L"," << book4 << std::endl; - - //Loop through the booklist elements using a range-based for loop - for (const auto& book : booklist.array_range()) - { - std::wcout << book[L"title"].as() - << L"," - << book[L"price"].as() << std::endl; - } - - // The second book - wjson& book = booklist[1]; - - //Loop through the book members using a range-based for loop - for(const auto& member : book.object_range()) - { - std::wcout << member.key() - << L"," - << member.value() << std::endl; - } - - auto it = book.find(L"author"); - if (it != book.object_range().end()) - { - // book has member "author" - } - - if (book.contains(L"author")) - { - // book has member "author" - } - - std::wstring s = book.get_with_default(L"author", L"author unknown"); - // Returns author if found, otherwise "author unknown" - - try - { - book[L"ratings"].as(); - } - catch (const std::out_of_range&) - { - // member "ratings" not found - } - - // Add ratings - book[L"ratings"][L"*****"] = 4; - book[L"ratings"][L"*"] = 2; - - // Delete one-star ratings - book[L"ratings"].erase(L"*"); - - // Serialize the booklist to a file - std::wofstream os("./output/booklist2.json"); - os << pretty_print(booklist); -} - -void basics_wjson_example2() -{ - // Deserialize the booklist - std::wifstream is("./output/booklist2.json"); - wjson booklist; - is >> booklist; - - // Use a JSONPath expression to find - // - // (1) The authors of books that cost less than $12 - wjson result = jsonpath::json_query(booklist, L"$[*][?(@.price < 12)].author"); - std::wcout << L"(1) " << result << std::endl; - - // (2) The number of books - result = jsonpath::json_query(booklist, L"$.length"); - std::wcout << L"(2) " << result << std::endl; - - // (3) The third book - result = jsonpath::json_query(booklist, L"$[2]"); - std::wcout << L"(3) " << std::endl << pretty_print(result) << std::endl; - - // (4) The authors of books that were published in 2004 - result = jsonpath::json_query(booklist, L"$[*][?(@.date =~ /2004.*?/)].author"); - std::wcout << L"(4) " << result << std::endl; - - // (5) The titles of all books that have ratings - result = jsonpath::json_query(booklist, L"$[*][?(@.ratings)].title"); - std::wcout << L"(5) " << result << std::endl; -} - -void basics_wexamples() -{ - std::cout << "\nBasics\n\n"; - basics_wjson_example1(); - basics_wjson_example2(); - std::cout << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/bson_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/bson_examples.cpp deleted file mode 100644 index 57f532aa1a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/bson_examples.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include - -using namespace jsoncons; - -void serialize_to_bson() -{ - std::vector buffer; - bson::bson_bytes_encoder encoder(buffer); - encoder.begin_array(); // The total number of bytes comprising - // the bson document will be calculated - encoder.string_value("cat"); - encoder.byte_string_value(byte_string({'p','u','r','r'})); - encoder.int64_value(1431027667, semantic_tag::timestamp); - encoder.end_array(); - encoder.flush(); - - for (auto c : buffer) - { - std::cout << std::hex << std::setprecision(2) << std::setw(2) - << std::noshowbase << std::setfill('0') << static_cast(c); - } - std::cout << "\n\n"; - -/* - 22000000 -- Total number of bytes comprising the document (34 bytes) - 02 -- UTF-8 string - 3000 -- "0" - 04000000 -- number bytes in the string (including trailing byte) - 636174 -- "cat" - 00 -- trailing byte - 05 -- binary - 3100 -- "1" - 04000000 -- number of bytes - 70757272 -- 'P''u''r''r' - 09 -- datetime - 3200 -- "2" - d3bf4b55 -- 1431027667 - 00 -*/ -} - -void bson_examples() -{ - std::cout << "\nbson examples\n\n"; - serialize_to_bson(); - std::cout << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/byte_string_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/byte_string_examples.cpp deleted file mode 100644 index 36b3877e9f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/byte_string_examples.cpp +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under Boost license - -#include - -using namespace jsoncons; // For brevity - -void construct_json_byte_string() -{ - byte_string bs = {'H','e','l','l','o'}; - - // default suggested encoding (base64url) - json j1(bs); - std::cout << "(1) "<< j1 << "\n\n"; - - // base64 suggested encoding - json j2(bs, semantic_tag::base64); - std::cout << "(2) "<< j2 << "\n\n"; - - // base16 suggested encoding - json j3(bs, semantic_tag::base16); - std::cout << "(3) "<< j3 << "\n\n"; -} - -void retrieve_json_value_as_byte_string() -{ - json j; - j["ByteString"] = byte_string({'H','e','l','l','o'}); - j["EncodedByteString"] = json("SGVsbG8=", semantic_tag::base64); - - std::cout << "(1)\n"; - std::cout << pretty_print(j) << "\n\n"; - - // Retrieve a byte string as a jsoncons::byte_string - byte_string bs1 = j["ByteString"].as(); - std::cout << "(2) " << bs1 << "\n\n"; - - // or alternatively as a std::vector - std::vector v = j["ByteString"].as>(); - - // Retrieve a byte string from a text string containing base64 character values - byte_string bs2 = j["EncodedByteString"].as(); - std::cout << "(3) " << bs2 << "\n\n"; - - // Retrieve a byte string view to access the memory that's holding the byte string - byte_string_view bsv3 = j["ByteString"].as(); - std::cout << "(4) " << bsv3 << "\n\n"; - - // Can't retrieve a byte string view of a text string - try - { - byte_string_view bsv4 = j["EncodedByteString"].as(); - } - catch (const std::exception& e) - { - std::cout << "(5) "<< e.what() << "\n\n"; - } -} - -void serialize_json_byte_string() -{ - byte_string bs = {'H','e','l','l','o'}; - - json j(bs); - - // default - std::cout << "(1) "<< j << "\n\n"; - - // base16 - json_options options2; - options2.byte_string_format(byte_string_chars_format::base16); - std::cout << "(2) "<< print(j, options2) << "\n\n"; - - // base64 - json_options options3; - options3.byte_string_format(byte_string_chars_format::base64); - std::cout << "(3) "<< print(j, options3) << "\n\n"; - - // base64url - json_options options4; - options4.byte_string_format(byte_string_chars_format::base64url); - std::cout << "(4) "<< print(j, options4) << "\n\n"; -} - -void byte_string_examples() -{ - std::cout << "byte_string examples" << "\n\n"; - construct_json_byte_string(); - serialize_json_byte_string(); - retrieve_json_value_as_byte_string(); -} - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/cbor_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/cbor_examples.cpp deleted file mode 100644 index a45eabf339..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/cbor_examples.cpp +++ /dev/null @@ -1,400 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -void serialize_to_cbor_buffer() -{ - std::vector buffer; - cbor::cbor_bytes_encoder encoder(buffer); - - encoder.begin_array(); // Indefinite length array - encoder.string_value("cat"); - encoder.byte_string_value(byte_string({'p','u','r','r'})); - encoder.byte_string_value(byte_string({'h','i','s','s'}), - semantic_tag::base64); // suggested conversion to base64 - encoder.int64_value(1431027667, semantic_tag::timestamp); - encoder.end_array(); - encoder.flush(); - - for (auto c : buffer) - { - std::cout << std::hex << std::setprecision(2) << std::setw(2) - << std::noshowbase << std::setfill('0') << static_cast(c); - } - std::cout << "\n\n"; - -/* - 9f -- Start indefinte length array - 63 -- String value of length 3 - 636174 -- "cat" - 44 -- Byte string value of length 4 - 70757272 -- 'p''u''r''r' - d6 - Expected conversion to base64 - 44 - 68697373 -- 'h''i''s''s' - c1 -- Tag value 1 (seconds relative to 1970-01-01T00:00Z in UTC time) - 1a -- 32 bit unsigned integer - 554bbfd3 -- 1431027667 - ff -- "break" -*/ -} - -void serialize_to_cbor_stream() -{ - std::ostringstream os; - cbor::cbor_encoder encoder(os); - - encoder.begin_array(3); // array of length 3 - encoder.string_value("-18446744073709551617", semantic_tag::bigint); - encoder.string_value("184467440737095516.16", semantic_tag::bigdec); - encoder.int64_value(1431027667, semantic_tag::timestamp); - encoder.end_array(); - encoder.flush(); - - for (auto c : os.str()) - { - std::cout << std::hex << std::setprecision(2) << std::setw(2) - << std::noshowbase << std::setfill('0') << (int)unsigned char(c); - } - std::cout << "\n\n"; - -/* - 83 -- array of length 3 - c3 -- Tag 3 (negative bignum) - 49 -- Byte string value of length 9 - 010000000000000000 -- Bytes content - c4 -- Tag 4 (decimal fraction) - 82 -- Array of length 2 - 21 -- -2 (exponent) - c2 Tag 2 (positive bignum) - 49 -- Byte string value of length 9 - 010000000000000000 - c1 -- Tag 1 (seconds relative to 1970-01-01T00:00Z in UTC time) - 1a -- 32 bit unsigned integer - 554bbfd3 -- 1431027667 -*/ -} - -void cbor_reputon_example() -{ - ojson j1 = ojson::parse(R"( - { - "application": "hiking", - "reputons": [ - { - "rater": "HikingAsylum.example.com", - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rating": 0.90 - } - ] - } - )"); - - // Encode a basic_json value to a CBOR value - std::vector data; - cbor::encode_cbor(j1, data); - - // Decode a CBOR value to a basic_json value - ojson j2 = cbor::decode_cbor(data); - std::cout << "(1)\n" << pretty_print(j2) << "\n\n"; - - // Accessing the data items - - const ojson& reputons = j2["reputons"]; - - std::cout << "(2)\n"; - for (auto element : reputons.array_range()) - { - std::cout << element.at("rated").as() << ", "; - std::cout << element.at("rating").as() << "\n"; - } - std::cout << std::endl; - - // Get a CBOR value for a nested data item with jsonpointer - std::error_code ec; - auto const& rated = jsonpointer::get(j2, "/reputons/0/rated", ec); - if (!ec) - { - std::cout << "(3) " << rated.as_string() << "\n"; - } - - std::cout << std::endl; -} - -void decode_cbor_byte_string() -{ - // byte string of length 5 - std::vector buf = {0x45,'H','e','l','l','o'}; - json j = cbor::decode_cbor(buf); - - auto bs = j.as(); - - // byte_string to ostream displays as hex - std::cout << "(1) "<< bs << "\n\n"; - - // byte string value to JSON text becomes base64url - std::cout << "(2) " << j << std::endl; -} - -void decode_byte_string_with_encoding_hint() -{ - // semantic tag indicating expected conversion to base64 - // followed by byte string of length 5 - std::vector buf = {0xd6,0x45,'H','e','l','l','o'}; - json j = cbor::decode_cbor(buf); - - auto bs = j.as(); - - // byte_string to ostream displays as hex - std::cout << "(1) "<< bs << "\n\n"; - - // byte string value to JSON text becomes base64 - std::cout << "(2) " << j << std::endl; -} - -void encode_cbor_byte_string() -{ - // construct byte string value - json j(byte_string("Hello")); - - std::vector buf; - cbor::encode_cbor(j, buf); - - std::cout << std::hex << std::showbase << "(1) "; - for (auto c : buf) - { - std::cout << (int)c; - } - std::cout << std::dec << "\n\n"; - - json j2 = cbor::decode_cbor(buf); - std::cout << "(2) " << j2 << std::endl; -} - -void encode_byte_string_with_encoding_hint() -{ - // construct byte string value - json j1(byte_string("Hello"), semantic_tag::base64); - - std::vector buf; - cbor::encode_cbor(j1, buf); - - std::cout << std::hex << std::showbase << "(1) "; - for (auto c : buf) - { - std::cout << (int)c; - } - std::cout << std::dec << "\n\n"; - - json j2 = cbor::decode_cbor(buf); - std::cout << "(2) " << j2 << std::endl; -} - -void query_cbor() -{ - // Construct a json array of numbers - json j = json::array(); - - j.emplace_back(5.0); - - j.emplace_back(0.000071); - - j.emplace_back("-18446744073709551617",semantic_tag::bigint); - - j.emplace_back("1.23456789012345678901234567890", semantic_tag::bigdec); - - j.emplace_back("0x3p-1", semantic_tag::bigfloat); - - // Encode to JSON - std::cout << "(1)\n"; - std::cout << pretty_print(j); - std::cout << "\n\n"; - - // as() and as() - std::cout << "(2)\n"; - std::cout << std::dec << std::setprecision(15); - for (const auto& item : j.array_range()) - { - std::cout << item.as() << ", " << item.as() << "\n"; - } - std::cout << "\n"; - - // Encode to CBOR - std::vector v; - cbor::encode_cbor(j,v); - - std::cout << "(3)\n"; - for (auto c : v) - { - std::cout << std::hex << std::setprecision(2) << std::setw(2) - << std::setfill('0') << static_cast(c); - } - std::cout << "\n\n"; -/* - 85 -- Array of length 5 - fa -- float - 40a00000 -- 5.0 - fb -- double - 3f129cbab649d389 -- 0.000071 - c3 -- Tag 3 (negative bignum) - 49 -- Byte string value of length 9 - 010000000000000000 - c4 -- Tag 4 (decimal fraction) - 82 -- Array of length 2 - 38 -- Negative integer of length 1 - 1c -- -29 - c2 -- Tag 2 (positive bignum) - 4d -- Byte string value of length 13 - 018ee90ff6c373e0ee4e3f0ad2 - c5 -- Tag 5 (bigfloat) - 82 -- Array of length 2 - 20 -- -1 - 03 -- 3 -*/ - - // Decode back to json - json other = cbor::decode_cbor(v); - assert(other == j); - - // Query with JSONPath - std::cout << "(4)\n"; - json result = jsonpath::json_query(other,"$[?(@ < 1.5)]"); - std::cout << pretty_print(result) << "\n\n"; -} - -void encode_cbor_with_packed_strings() -{ - ojson j = ojson::parse(R"( -[ - { - "name" : "Cocktail", - "count" : 417, - "rank" : 4 - }, - { - "rank" : 4, - "count" : 312, - "name" : "Bath" - }, - { - "count" : 691, - "name" : "Food", - "rank" : 4 - } - ] -)"); - - cbor::cbor_options options; - options.pack_strings(true); - std::vector buf; - - cbor::encode_cbor(j, buf, options); - - for (auto c : buf) - { - std::cout << std::hex << std::setprecision(2) << std::setw(2) - << std::noshowbase << std::setfill('0') << static_cast(c); - } - std::cout << "\n"; - -/* - d90100 -- tag (256) - 83 -- array(3) - a3 -- map(3) - 64 -- text string (4) - 6e616d65 -- "name" - 68 -- text string (8) - 436f636b7461696c -- "Cocktail" - 65 -- text string (5) - 636f756e74 -- "count" - 1901a1 -- unsigned(417) - 64 -- text string (4) - 72616e6b -- "rank" - 04 -- unsigned(4) - a3 -- map(3) - d819 -- tag(25) - 03 -- unsigned(3) - 04 -- unsigned(4) - d819 -- tag(25) - 02 -- unsigned(2) - 190138 -- unsigned(312) - d819 -- tag(25) - 00 -- unsigned(0) - 64 -- text string(4) - 42617468 -- "Bath" - a3 -- map(3) - d819 -- tag(25) - 02 -- unsigned(2) - 1902b3 -- unsigned(691) - d819 -- tag(25) - 00 -- unsigned(0) - 64 - text string(4) - 466f6f64 -- "Food" - d819 -- tag(25) - 03 -- unsigned(3) - 04 -- unsigned(4) -*/ - - ojson j2 = cbor::decode_cbor(buf); - assert(j2 == j); -} - -void decode_cbor_with_packed_strings() -{ - std::vector v = {0xd9,0x01,0x00, // tag(256) - 0x85, // array(5) - 0x63, // text(3) - 0x61,0x61,0x61, // "aaa" - 0xd8, 0x19, // tag(25) - 0x00, // unsigned(0) - 0xd9, 0x01,0x00, // tag(256) - 0x83, // array(3) - 0x63, // text(3) - 0x62,0x62,0x62, // "bbb" - 0x63, // text(3) - 0x61,0x61,0x61, // "aaa" - 0xd8, 0x19, // tag(25) - 0x01, // unsigned(1) - 0xd9, 0x01,0x00, // tag(256) - 0x82, // array(2) - 0x63, // text(3) - 0x63,0x63,0x63, // "ccc" - 0xd8, 0x19, // tag(25) - 0x00, // unsigned(0) - 0xd8, 0x19, // tag(25) - 0x00 // unsigned(0) - }; - - ojson j = cbor::decode_cbor(v); - - std::cout << pretty_print(j) << "\n"; -} - -void cbor_examples() -{ - std::cout << "\ncbor examples\n\n"; - decode_byte_string_with_encoding_hint(); - encode_byte_string_with_encoding_hint(); - decode_cbor_byte_string(); - encode_cbor_byte_string(); - serialize_to_cbor_buffer(); - serialize_to_cbor_stream(); - cbor_reputon_example(); - query_cbor(); - encode_cbor_with_packed_strings(); - - decode_cbor_with_packed_strings(); - - std::cout << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/container_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/container_examples.cpp deleted file mode 100644 index f3176c0fc6..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/container_examples.cpp +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -void container_example1() -{ - std::cout << "Convert from and to standard library containers\n" << std::endl; - - { - std::vector v{1, 2, 3, 4}; - json j(v); - std::cout << "(1) "<< j << std::endl; - std::deque d = j.as>(); - } - - std::vector vec2{1ul, 2ul, 3ul, 4ul}; - json j_vec2(vec2); - std::cout << j_vec2 << std::endl; - - std::deque deque1{1.123, 2.234, 3.456, 4.567}; - json j_deque1(deque1); - std::cout << j_deque1 << std::endl; - - std::list list1{true, true, false, true}; - json j_list1(list1); - std::cout << j_list1 << std::endl; - - std::forward_listflist1 {12345678909876, 23456789098765, 34567890987654, 45678909876543}; - json j_flist1(flist1); - std::cout << j_flist1 << std::endl; - - std::array array1 {{1, 2, 3, 4}}; - json j_array1(array1); - - std::set set1{"one", "two", "three", "four", "one"}; - json j_set1(set1); // only one entry for "one" is used - std::cout << j_set1 << std::endl; - // ["four", "one", "three", "two"] - - std::unordered_set uset1{"one", "two", "three", "four", "one"}; - json j_uset1(uset1); // only one entry for "one" is used - std::cout << j_uset1 << std::endl; - // maybe ["two", "three", "four", "one"] - - std::multiset mset1{"one", "two", "one", "four"}; - json j_mset1(mset1); // only one entry for "one" is used - std::cout << j_mset1 << std::endl; - // maybe ["one", "two", "four"] - - std::unordered_multiset umset1 {"one", "two", "one", "four"}; - json j_umset1(umset1); // both entries for "one" are used - // maybe ["one", "two", "one", "four"] - - { - std::map m{{"one",1},{"two",2},{"three",3}}; - json j(m); - std::cout << "(1) " << j << std::endl; - std::unordered_map um = j.as>(); - } - - std::unordered_map umap1{ {"one", 1.2}, {"two", 2.3}, {"three", 3.4} }; - json j_umap1(umap1); - std::cout << j_umap1 << std::endl; - - std::cout << std::endl; -} - -void container_examples() -{ - std::cout << "\nContainer examples\n\n"; - container_example1(); - std::cout << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/csv_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/csv_examples.cpp deleted file mode 100644 index a986d8bd65..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/csv_examples.cpp +++ /dev/null @@ -1,409 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include - -using namespace jsoncons; - -void csv_source_to_json_value() -{ - const std::string s = R"(Date,1Y,2Y,3Y,5Y -2017-01-09,0.0062,0.0075,0.0083,0.011 -2017-01-08,0.0063,0.0076,0.0084,0.0112 -2017-01-08,0.0063,0.0076,0.0084,0.0112 -)"; - - csv::csv_options options; - options.assume_header(true) - .column_types("string,float,float,float,float"); - - // mapping_type::n_objects - options.mapping(csv::mapping_type::n_objects); - ojson j1 = csv::decode_csv(s,options); - std::cout << "\n(1)\n"<< pretty_print(j1) << "\n"; - - // mapping_type::n_rows - options.mapping(csv::mapping_type::n_rows); - ojson j2 = csv::decode_csv(s,options); - std::cout << "\n(2)\n"<< pretty_print(j2) << "\n"; - - // mapping_type::m_columns - options.mapping(csv::mapping_type::m_columns); - ojson j3 = csv::decode_csv(s,options); - std::cout << "\n(3)\n" << pretty_print(j3) << "\n"; -} - -void csv_source_to_cpp_object() -{ - const std::string input = R"(Date,1Y,2Y,3Y,5Y -2017-01-09,0.0062,0.0075,0.0083,0.011 -2017-01-08,0.0063,0.0076,0.0084,0.0112 -2017-01-08,0.0063,0.0076,0.0084,0.0112 -)"; - - csv::csv_options ioptions; - ioptions.header_lines(1) - .mapping(csv::mapping_type::n_rows); - - typedef std::vector> table_type; - - table_type table = csv::decode_csv(input,ioptions); - - std::cout << "(1)\n"; - for (const auto& row : table) - { - std::cout << std::get<0>(row) << "," - << std::get<1>(row) << "," - << std::get<2>(row) << "," - << std::get<3>(row) << "," - << std::get<4>(row) << "\n"; - } - std::cout << "\n"; - - std::string output; - - csv::csv_options ooptions; - ooptions.column_names("Date,1Y,2Y,3Y,5Y"); - csv::encode_csv(table, output, ooptions); - - std::cout << "(2)\n"; - std::cout << output << "\n"; -} - -void csv_decode_without_type_inference() -{ - std::string s = R"(employee-no,employee-name,dept,salary -00000001,"Smith,Matthew",sales,150000.00 -00000002,"Brown,Sarah",sales,89000.00 -)"; - - csv::csv_options options; - options.assume_header(true) - .infer_types(false); - ojson j = csv::decode_csv(s,options); - - std::cout << pretty_print(j) << std::endl; -} - -void read_write_csv_tasks() -{ - std::ifstream is("./input/tasks.csv"); - - json_decoder decoder; - csv::csv_options options; - options.assume_header(true) - .trim(true) - .ignore_empty_values(true) - .column_types("integer,string,string,string"); - csv::csv_reader reader(is,decoder,options); - reader.read(); - ojson tasks = decoder.get_result(); - - std::cout << "(1)\n"; - std::cout << pretty_print(tasks) << "\n\n"; - - std::cout << "(2)\n"; - csv::csv_encoder encoder(std::cout); - tasks.dump(encoder); -} - -void serialize_array_of_arrays_to_comma_delimited() -{ - std::string in_file = "./input/countries.json"; - std::ifstream is(in_file); - - json countries; - is >> countries; - - csv::csv_encoder encoder(std::cout); - countries.dump(encoder); -} - -void serialize_to_tab_delimited_file() -{ - std::string in_file = "./input/employees.json"; - std::ifstream is(in_file); - - json employees; - is >> employees; - - csv::csv_options options; - options.field_delimiter('\t'); - csv::csv_encoder encoder(std::cout,options); - - employees.dump(encoder); -} - -void serialize_books_to_csv_file() -{ - const json books = json::parse(R"( - [ - { - "title" : "Kafka on the Shore", - "author" : "Haruki Murakami", - "price" : 25.17 - }, - { - "title" : "Women: A Novel", - "author" : "Charles Bukowski", - "price" : 12.00 - }, - { - "title" : "Cutter's Way", - "author" : "Ivan Passer" - } - ] - )"); - - csv::csv_encoder encoder(std::cout); - - books.dump(encoder); -} - -void serialize_books_to_csv_file_with_reorder() -{ - const json books = json::parse(R"( - [ - { - "title" : "Kafka on the Shore", - "author" : "Haruki Murakami", - "price" : 25.17 - }, - { - "title" : "Women: A Novel", - "author" : "Charles Bukowski", - "price" : 12.00 - }, - { - "title" : "Cutter's Way", - "author" : "Ivan Passer" - } - ] - )"); - - csv::csv_options options; - options.column_names("author,title,price"); - - csv::csv_encoder encoder(std::cout, options); - - books.dump(encoder); -} - -void last_column_repeats() -{ - const std::string bond_yields = R"(Date,Yield -2017-01-09,0.0062,0.0075,0.0083,0.011,0.012 -2017-01-08,0.0063,0.0076,0.0084,0.0112,0.013 -2017-01-08,0.0063,0.0076,0.0084,0.0112,0.014 -)"; - - json_decoder decoder1; - csv::csv_options options1; - options1.header_lines(1); - options1.column_types("string,float*"); - std::istringstream is1(bond_yields); - csv::csv_reader reader1(is1, decoder1, options1); - reader1.read(); - ojson val1 = decoder1.get_result(); - std::cout << "\n(1)\n" << pretty_print(val1) << "\n"; - - json_decoder decoder2; - csv::csv_options options2; - options2.assume_header(true); - options2.column_types("string,[float*]"); - std::istringstream is2(bond_yields); - csv::csv_reader reader2(is2, decoder2, options2); - reader2.read(); - ojson val2 = decoder2.get_result(); - std::cout << "\n(2)\n" << pretty_print(val2) << "\n"; -} - -void last_two_columns_repeat() -{ - const std::string holidays = R"(1,CAD,2,UK,3,EUR,4,US -38719,2-Jan-2006,40179,1-Jan-2010,38719,2-Jan-2006,39448,1-Jan-2008 -38733,16-Jan-2006,40270,2-Apr-2010,38733,16-Jan-2006,39468,21-Jan-2008 -)"; - - // array of arrays - json_decoder decoder1; - csv::csv_options options1; - options1.column_types("[integer,string]*"); - std::istringstream is1(holidays); - csv::csv_reader reader1(is1, decoder1, options1); - reader1.read(); - ojson val1 = decoder1.get_result(); - std::cout << "(1)\n" << pretty_print(val1) << "\n"; - - // array of objects - json_decoder decoder2; - csv::csv_options options2; - options2.header_lines(1); - options2.column_names("CAD,UK,EUR,US"); - options2.column_types("[integer,string]*"); - std::istringstream is2(holidays); - csv::csv_reader reader2(is2, decoder2, options2); - reader2.read(); - ojson val2 = decoder2.get_result(); - std::cout << "(2)\n" << pretty_print(val2) << "\n"; -} - -void decode_csv_string() -{ - std::string s = R"(employee-no,employee-name,dept,salary -00000001,\"Smith,Matthew\",sales,150000.00 -00000002,\"Brown,Sarah\",sales,89000.00 -)"; - - csv::csv_options options; - options.assume_header(true) - .column_types("string,string,string,float"); - json j = csv::decode_csv(s,options); - - std::cout << pretty_print(j) << std::endl; -} - -void decode_csv_stream() -{ - const std::string bond_yields = R"(Date,1Y,2Y,3Y,5Y -2017-01-09,0.0062,0.0075,0.0083,0.011 -2017-01-08,0.0063,0.0076,0.0084,0.0112 -2017-01-08,0.0063,0.0076,0.0084,0.0112 -)"; - - csv::csv_options options; - options.assume_header(true) - .column_types("string,float,float,float,float"); - - std::istringstream is(bond_yields); - - ojson j = csv::decode_csv(is,options); - - std::cout << pretty_print(j) << std::endl; -} - -void encode_csv_file_from_books() -{ - const json books = json::parse(R"( - [ - { - "title" : "Kafka on the Shore", - "author" : "Haruki Murakami", - "price" : 25.17 - }, - { - "title" : "Women: A Novel", - "author" : "Charles Bukowski", - "price" : 12.00 - }, - { - "title" : "Cutter's Way", - "author" : "Ivan Passer" - } - ] - )"); - - csv::encode_csv(books, std::cout); -} - -void decode_encode_csv_tasks() -{ - std::ifstream is("./input/tasks.csv"); - - csv::csv_options options; - options.assume_header(true) - .trim(true) - .ignore_empty_values(true) - .column_types("integer,string,string,string"); - ojson tasks = csv::decode_csv(is, options); - - std::cout << "(1)\n" << pretty_print(tasks) << "\n\n"; - - std::cout << "(2)\n"; - csv::encode_csv(tasks, std::cout); -} - -void csv_parser_type_inference() -{ - csv::csv_options options; - options.assume_header(true) - .mapping(csv::mapping_type::n_objects); - - std::ifstream is1("input/sales.csv"); - ojson j1 = csv::decode_csv(is1,options); - std::cout << "\n(1)\n"<< pretty_print(j1) << "\n"; - - options.mapping(csv::mapping_type::n_rows); - std::ifstream is2("input/sales.csv"); - ojson j2 = csv::decode_csv(is2,options); - std::cout << "\n(2)\n"<< pretty_print(j2) << "\n"; - - options.mapping(csv::mapping_type::m_columns); - std::ifstream is3("input/sales.csv"); - ojson j3 = csv::decode_csv(is3,options); - std::cout << "\n(3)\n"<< pretty_print(j3) << "\n"; -} - -// Examples with subfields - - -void decode_csv_with_subfields() -{ - const std::string s = R"(calculationPeriodCenters,paymentCenters,resetCenters -NY;LON,TOR,LON -NY,LON,TOR;LON -"NY";"LON","TOR","LON" -"NY","LON","TOR";"LON" -)"; - json_options print_options; - print_options.array_array_line_splits(line_split_kind::same_line); - - csv::csv_options options1; - options1.assume_header(true) - .subfield_delimiter(';'); - - json j1 = csv::decode_csv(s,options1); - std::cout << "(1)\n" << pretty_print(j1,print_options) << "\n\n"; - - csv::csv_options options2; - options2.mapping(csv::mapping_type::n_rows) - .subfield_delimiter(';'); - - json j2 = csv::decode_csv(s,options2); - std::cout << "(2)\n" << pretty_print(j2,print_options) << "\n\n"; - - csv::csv_options options3; - options3.assume_header(true) - .mapping(csv::mapping_type::m_columns) - .subfield_delimiter(';'); - - json j3 = csv::decode_csv(s,options3); - std::cout << "(3)\n" << pretty_print(j3,print_options) << "\n\n"; -} - -void csv_examples() -{ - std::cout << "\nCSV examples\n\n"; - read_write_csv_tasks(); - serialize_to_tab_delimited_file(); - serialize_array_of_arrays_to_comma_delimited(); - serialize_books_to_csv_file(); - serialize_books_to_csv_file_with_reorder(); - last_column_repeats(); - last_two_columns_repeat(); - decode_csv_string(); - decode_csv_stream(); - encode_csv_file_from_books(); - decode_encode_csv_tasks(); - - csv_decode_without_type_inference(); - csv_parser_type_inference(); - - decode_csv_with_subfields(); - csv_source_to_json_value(); - csv_source_to_cpp_object(); - std::cout << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/data_model_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/data_model_examples.cpp deleted file mode 100644 index 42bd2ea1de..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/data_model_examples.cpp +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under Boost license - -#include -#include -#include - -using namespace jsoncons; - -void data_model_example1() -{ - json j = json::array(); - - j.emplace_back("foo"); - j.emplace_back(byte_string{ 'b','a','r' }); - j.emplace_back("-18446744073709551617", semantic_tag::bigint); - j.emplace_back("273.15", semantic_tag::bigdec); - j.emplace_back("2018-10-19 12:41:07-07:00", semantic_tag::datetime); - j.emplace_back(1431027667, semantic_tag::timestamp); - j.emplace_back(-1431027667, semantic_tag::timestamp); - j.emplace_back(1431027667.5, semantic_tag::timestamp); - - std::cout << "(1)\n" << pretty_print(j) << "\n\n"; - - std::vector bytes; - cbor::encode_cbor(j, bytes); - std::cout << "(2)\n"; - for (auto c : bytes) - { - std::cout << std::hex << std::noshowbase << std::setprecision(2) << std::setw(2) - << std::setfill('0') << static_cast(c); - } - std::cout << "\n\n"; -/* -88 -- Array of length 8 - 63 -- String value of length 3 - 666f6f -- "foo" - 43 -- Byte string value of length 3 - 626172 -- 'b''a''r' - c3 -- Tag 3 (negative bignum) - 49 Byte string value of length 9 - 010000000000000000 -- Bytes content - c4 - Tag 4 (decimal fraction) - 82 -- Array of length 2 - 21 -- -2 - 19 6ab3 -- 27315 - c0 -- Tag 0 (date-time) - 78 19 -- Length (25) - 323031382d31302d31392031323a34313a30372d30373a3030 -- "2018-10-19 12:41:07-07:00" - c1 -- Tag 1 (epoch time) - 1a -- uint32_t - 554bbfd3 -- 1431027667 - c1 - 3a - 554bbfd2 - c1 - fb - 41d552eff4e00000 -*/ -} - -void data_model_example2() -{ - std::vector bytes; - cbor::cbor_bytes_encoder encoder(bytes); - encoder.begin_array(); // indefinite length outer array - encoder.string_value("foo"); - encoder.byte_string_value(byte_string{'b','a','r'}); - encoder.string_value("-18446744073709551617", semantic_tag::bigint); - encoder.string_value("273.15", semantic_tag::bigdec); - encoder.string_value("2018-10-19 12:41:07-07:00", semantic_tag::datetime) ; - encoder.int64_value(1431027667, semantic_tag::timestamp); - encoder.int64_value(-1431027667, semantic_tag::timestamp); - encoder.double_value(1431027667.5, semantic_tag::timestamp); - encoder.end_array(); - encoder.flush(); - - std::cout << "(1)\n"; - for (auto c : bytes) - { - std::cout << std::hex << std::setprecision(2) << std::setw(2) - << std::noshowbase << std::setfill('0') << static_cast(c); - } - std::cout << "\n\n"; - -/* -9f -- Start indefinite length array - 63 -- String value of length 3 - 666f6f -- "foo" - 43 -- Byte string value of length 3 - 626172 -- 'b''a''r' - c3 -- Tag 3 (negative bignum) - 49 Byte string value of length 9 - 010000000000000000 -- Bytes content - c4 - Tag 4 (decimal fraction) - 82 -- Array of length 2 - 21 -- -2 - 19 6ab3 -- 27315 - c0 -- Tag 0 (date-time) - 78 19 -- Length (25) - 323031382d31302d31392031323a34313a30372d30373a3030 -- "2018-10-19 12:41:07-07:00" - c1 -- Tag 1 (epoch time) - 1a -- uint32_t - 554bbfd3 -- 1431027667 - c1 - 3a - 554bbfd2 - c1 - fb - 41d552eff4e00000 - ff -- "break" -*/ - - json j = cbor::decode_cbor(bytes); - - std::cout << "(2)\n" << pretty_print(j) << "\n\n"; -} - -void data_model_examples() -{ - std::cout << "\ndata model examples\n\n"; - data_model_example1(); - data_model_example2(); - std::cout << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/example_types.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/example_types.hpp deleted file mode 100644 index 6899157221..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/example_types.hpp +++ /dev/null @@ -1,105 +0,0 @@ -#ifndef EXAMPLE_TYPES -#define EXAMPLE_TYPES - -#include -#include -#include - -// book example - -namespace ns { - struct book - { - std::string author; - std::string title; - double price; - }; -} // namespace ns - -namespace jsoncons { - - template - struct json_type_traits - { - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - return j.is_object() && j.contains("author") && - j.contains("title") && j.contains("price"); - } - static ns::book as(const Json& j) - { - ns::book val; - val.author = j.at("author").template as(); - val.title = j.at("title").template as(); - val.price = j.at("price").template as(); - return val; - } - static Json to_json(const ns::book& val, - allocator_type allocator=allocator_type()) - { - Json j(allocator); - j.try_emplace("author", val.author); - j.try_emplace("title", val.title); - j.try_emplace("price", val.price); - return j; - } - }; -} // namespace jsoncons - -// reputon example - -namespace ns { - struct reputon - { - std::string rater; - std::string assertion; - std::string rated; - double rating; - - friend bool operator==(const reputon& lhs, const reputon& rhs) - { - return lhs.rater == rhs.rater && lhs.assertion == rhs.assertion && - lhs.rated == rhs.rated && lhs.rating == rhs.rating; - } - - friend bool operator!=(const reputon& lhs, const reputon& rhs) - { - return !(lhs == rhs); - }; - }; - - class reputation_object - { - std::string application; - std::vector reputons; - - // Make json_type_traits specializations friends to give accesses to private members - JSONCONS_TYPE_TRAITS_FRIEND; - public: - reputation_object() - { - } - reputation_object(const std::string& application, const std::vector& reputons) - : application(application), reputons(reputons) - {} - - friend bool operator==(const reputation_object& lhs, const reputation_object& rhs) - { - return (lhs.application == rhs.application) && (lhs.reputons == rhs.reputons); - } - - friend bool operator!=(const reputation_object& lhs, const reputation_object& rhs) - { - return !(lhs == rhs); - }; - }; - -} // ns - -// Declare the traits. Specify which data members need to be serialized. -JSONCONS_MEMBER_TRAITS_DECL(ns::reputon, rater, assertion, rated, rating) -JSONCONS_MEMBER_TRAITS_DECL(ns::reputation_object, application, reputons) - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/examples.cpp deleted file mode 100644 index c74b923317..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/examples.cpp +++ /dev/null @@ -1,588 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -void readme_examples(); -void basics_examples(); -void basics_wexamples(); -void json_filter_examples(); -void array_examples(); -void container_examples(); -void wjson_examples(); -void serialization_examples(); -void type_extensibility_examples(); -void type_extensibility_examples2(); -void ojson_examples(); -void unicode_examples(); -void csv_examples(); -void jsonpath_examples(); -void json_accessor_examples(); -void msgpack_examples(); -void jsonpointer_examples(); -void jsonpatch_examples(); -void cbor_examples(); -void ubjson_examples(); -void json_parser_examples(); -void byte_string_examples(); -void pull_parser_examples(); -void data_model_examples(); -void staj_iterator_examples(); -void bson_examples(); -void polymorphic_examples(); - -void comment_example() -{ - std::string s = R"( - { - // Single line comments - /* - Multi line comments - */ - } - )"; - - // Default - json j = json::parse(s); - std::cout << "(1) " << j << std::endl; - - // Strict - try - { - strict_parse_error_handler err_handler; - json j = json::parse(s, err_handler); - } - catch (const ser_error& e) - { - std::cout << "(2) " << e.what() << std::endl; - } -} - -void first_example_a() -{ - std::string path = "./input/books.json"; - std::fstream is(path); - if (!is) - { - std::cout << "Cannot open " << path << std::endl; - return; - } - json books = json::parse(is); - - for (size_t i = 0; i < books.size(); ++i) - { - try - { - json& book = books[i]; - std::string author = book["author"].as(); - std::string title = book["title"].as(); - double price = book["price"].as(); - std::cout << author << ", " << title << ", " << price << std::endl; - } - catch (const std::exception& e) - { - std::cerr << e.what() << std::endl; - } - } -} - -void first_example_b() -{ - std::string path = "./input/books.json"; - std::fstream is(path); - if (!is) - { - std::cout << "Cannot open " << path << std::endl; - return; - } - json books = json::parse(is); - - for (size_t i = 0; i < books.size(); ++i) - { - try - { - json& book = books[i]; - std::string author = book["author"].as(); - std::string title = book["title"].as(); - std::string price = book.get_with_default("price", "N/A"); - std::cout << author << ", " << title << ", " << price << std::endl; - } - catch (const std::exception& e) - { - std::cerr << e.what() << std::endl; - } - } -} - -void first_example_c() -{ - const json books = json::parse(R"( - [ - { - "title" : "Kafka on the Shore", - "author" : "Haruki Murakami", - "price" : 25.17 - }, - { - "title" : "Women: A Novel", - "author" : "Charles Bukowski", - "price" : 12.00 - }, - { - "title" : "Cutter's Way", - "author" : "Ivan Passer" - } - ] - )"); - - json_options options; - - for (const auto& book : books.array_range()) - { - try - { - std::string author = book["author"].as(); - std::string title = book["title"].as(); - std::string price; - book.get_with_default("price", "N/A").dump(price,options); - std::cout << author << ", " << title << ", " << price << std::endl; - } - catch (const ser_error& e) - { - std::cerr << e.what() << std::endl; - } - } -} - -void first_example_d() -{ - std::string path = "./input/books.json"; - std::fstream is(path); - if (!is) - { - std::cout << "Cannot open " << path << std::endl; - return; - } - json books = json::parse(is); - - json_options options; - //options.floatfield(std::ios::fixed); - options.precision(2); - - for (size_t i = 0; i < books.size(); ++i) - { - try - { - json& book = books[i]; - std::string author = book["author"].as(); - std::string title = book["title"].as(); - if (book.contains("price") && book["price"].is_number()) - { - double price = book["price"].as(); - std::cout << author << ", " << title << ", " << price << std::endl; - } - else - { - std::cout << author << ", " << title << ", " << "n/a" << std::endl; - } - } - catch (const std::exception& e) - { - std::cerr << e.what() << std::endl; - } - } - -} - -void second_example_a() -{ - try - { - json books = json::array(); - - { - json book; - book["title"] = "Kafka on the Shore"; - book["author"] = "Haruki Murakami"; - book["price"] = 25.17; - books.push_back(std::move(book)); - } - - { - json book; - book["title"] = "Women: A Novel"; - book["author"] = "Charles Bukowski"; - book["price"] = 12.00; - books.push_back(std::move(book)); - } - - { - json book; - book["title"] = "Cutter's Way"; - book["author"] = "Ivan Passer"; - books.push_back(std::move(book)); - } - - std::cout << pretty_print(books) << std::endl; - } - catch (const std::exception& e) - { - std::cerr << e.what() << std::endl; - } -} - -void json_constructor_examples() -{ - json j1; // An empty object - std::cout << "(1) " << j1 << std::endl; - - json j2 = json::object({{"baz", "qux"}, {"foo", "bar"}}); // An object - std::cout << "(2) " << j2 << std::endl; - - json j3 = json::array({"bar", "baz"}); // An array - std::cout << "(3) " << j3 << std::endl; - - json j4(json::null()); // A null value - std::cout << "(4) " << j4 << std::endl; - - json j5(true); // A boolean value - std::cout << "(5) " << j5 << std::endl; - - double x = 1.0/7.0; - - json j6(x); // A double value - std::cout << "(6) " << j6 << std::endl; - - json j7(x,4); // A double value with specified precision - std::cout << "(7) " << j7 << std::endl; - - json j8("Hello"); // A text string - std::cout << "(8) " << j8 << std::endl; - - byte_string bs = {'H','e','l','l','o'}; - json j9(bs); // A byte string - std::cout << "(9) " << j9 << std::endl; - - std::vector v = {10,20,30}; - json j10 = v; // From a sequence container - std::cout << "(10) " << j10 << std::endl; - - std::map m{ {"one", 1}, {"two", 2}, {"three", 3} }; - json j11 = m; // From an associative container - std::cout << "(11) " << j11 << std::endl; - - // An object value with four members - json obj; - obj["first_name"] = "Jane"; - obj["last_name"] = "Roe"; - obj["events_attended"] = 10; - obj["accept_waiver_of_liability"] = true; - - std::string first_name = obj["first_name"].as(); - std::string last_name = obj.at("last_name").as(); - int events_attended = obj["events_attended"].as(); - bool accept_waiver_of_liability = obj["accept_waiver_of_liability"].as(); - - // An array value with four elements - json arr = json::array(); - arr.push_back(j1); - arr.push_back(j2); - arr.push_back(j3); - arr.push_back(j4); - - json_options options; - std::cout << pretty_print(arr) << std::endl; -} - -void mulitple_json_objects() -{ - std::ifstream is("./input/multiple-json-objects.json"); - if (!is.is_open()) - { - throw std::runtime_error("Cannot open file"); - } - - json_decoder decoder; - json_reader reader(is, decoder); - - while (!reader.eof()) - { - reader.read_next(); - if (!reader.eof()) - { - json val = decoder.get_result(); - std::cout << val << std::endl; - } - } -} - -void object_range_based_for_loop() -{ - json j = json::parse(R"( -{ - "category" : "Fiction", - "title" : "Pulp", - "author" : "Charles Bukowski", - "date" : "2004-07-08", - "price" : 22.48, - "isbn" : "1852272007" -} -)"); - - for (const auto& member : j.object_range()) - { - std::cout << member.key() << " => " << member.value().as() << std::endl; - } -} - -void more_examples() -{ - json file_settings = json::object{ - {"Image Format", "JPEG"}, - {"Color Space", "sRGB"}, - { "Limit File Size", true}, - {"Limit File Size To", 10000} - }; - - json image_sizing; - image_sizing.insert_or_assign("Resize To Fit",true); // a boolean - image_sizing.insert_or_assign("Resize Unit", "pixels"); // a string - image_sizing.insert_or_assign("Resize What", "long_edge"); // a string - image_sizing.insert_or_assign("Dimension 1",9.84); // a double - std::cout << pretty_print(image_sizing) << std::endl; - - json image_formats = json::array{"JPEG","PSD","TIFF","DNG"}; - - json color_spaces = json::array(); - color_spaces.push_back("sRGB"); - color_spaces.push_back("AdobeRGB"); - color_spaces.push_back("ProPhoto RGB"); - - json export_settings; - export_settings["File Format Options"]["Color Spaces"] = std::move(color_spaces); - export_settings["File Format Options"]["Image Formats"] = std::move(image_formats); - export_settings["File Settings"] = std::move(file_settings); - export_settings["Image Sizing"] = std::move(image_sizing); - - // Write to stream - std::ofstream os("./output/export_settings.json"); - os << export_settings; - - // Read from stream - std::ifstream is("./output/export_settings.json"); - json j = json::parse(is); - - // Pretty print - std::cout << "(1)\n" << pretty_print(j) << "\n\n"; - - // Does object member exist? - std::cout << "(2) " << std::boolalpha << j.contains("Image Sizing") << "\n\n"; - - // Get reference to object member - const json& val = j["Image Sizing"]; - - // Access member as double - std::cout << "(3) " << "Dimension 1 = " << val["Dimension 1"].as() << "\n\n"; - - // Try access member with default - std::cout << "(4) " << "Dimension 2 = " << val.get_with_default("Dimension 2",0.0) << "\n\n"; - -} - -void parse_error_example() -{ - std::string s = "[1,2,3,4,]"; - try - { - jsoncons::json val = jsoncons::json::parse(s); - } - catch(const jsoncons::ser_error& e) - { - std::cout << "Caught ser_error with category " << e.code().category().name() - << ", code " << e.code().value() - << " and message " << e.what() << std::endl; - } -} - -void validation_example() -{ - std::string s = R"( -{ - "StartDate" : "2017-03-01", - "MaturityDate" "2020-12-30" -} - )"; - std::stringstream is(s); - - json_reader reader(is); - - std::error_code ec; - reader.read(ec); - if (ec) - { - std::cout << ec.message() - << " on line " << reader.line() - << " and column " << reader.column() - << std::endl; - } -} - -void max_nesting_path_example() -{ - std::string s = "[[[[[[[[[[[[[[[[[[[[[\"Too deep\"]]]]]]]]]]]]]]]]]]]]]"; - try - { - json_options options; - options.max_nesting_depth(20); - json::parse(s, options); - } - catch (const ser_error& e) - { - std::cout << e.what() << std::endl; - } -} - -void get_example() -{ - json j = json::parse(R"( - { - "application": "hiking", - "reputons": [ - { - "rater": "HikingAsylum.example.com", - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rating": 0.90 - } - ] - } - )"); - - // Using index or `at` accessors - std::string result1 = j["reputons"][0]["rated"].as(); - std::cout << "(1) " << result1 << std::endl; - std::string result2 = j.at("reputons").at(0).at("rated").as(); - std::cout << "(2) " << result2 << std::endl; - - // Using JSON Pointer - std::string result3 = jsonpointer::get(j, "/reputons/0/rated").as(); - std::cout << "(3) " << result3 << std::endl; - - // Using JSONPath - json result4 = jsonpath::json_query(j, "$.reputons.0.rated"); - if (result4.size() > 0) - { - std::cout << "(4) " << result4[0].as() << std::endl; - } - json result5 = jsonpath::json_query(j, "$..0.rated"); - if (result5.size() > 0) - { - std::cout << "(5) " << result5[0].as() << std::endl; - } -} - -int main() -{ - try - { - - std::cout << "jsoncons version: " << version() << std::endl; - - object_range_based_for_loop(); - - basics_examples(); - basics_wexamples(); - ojson_examples(); - - first_example_a(); - first_example_b(); - first_example_c(); - first_example_d(); - - second_example_a(); - - array_examples(); - container_examples(); - - csv_examples(); - - mulitple_json_objects(); - - wjson_examples(); - - unicode_examples(); - - parse_error_example(); - - type_extensibility_examples2(); - - json_filter_examples(); - - msgpack_examples(); - - validation_example(); - - comment_example(); - - json_constructor_examples(); - - json_accessor_examples(); - - json_accessor_examples(); - - jsonpatch_examples(); - - max_nesting_path_example(); - - get_example(); - - json_parser_examples(); - - more_examples(); - - data_model_examples(); - - pull_parser_examples(); - - staj_iterator_examples(); - - bson_examples(); - - serialization_examples(); - - jsonpointer_examples(); - - cbor_examples(); - - byte_string_examples(); - - readme_examples(); - - type_extensibility_examples(); - - cbor_examples(); - - csv_examples(); - - ubjson_examples(); - - jsonpath_examples(); - - polymorphic_examples(); - } - catch (const std::exception& e) - { - std::cout << e.what() << std::endl; - } - - return 0; -} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/json_accessor_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/json_accessor_examples.cpp deleted file mode 100644 index cc14474351..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/json_accessor_examples.cpp +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include - -using namespace jsoncons; - -void is_as_examples() -{ - json j = json::parse(R"( - { - "k1" : 2147483647, - "k2" : 2147483648, - "k3" : -10, - "k4" : 10.5, - "k5" : true, - "k6" : "10.5" - } - )"); - - std::cout << std::boolalpha << "(1) " << j["k1"].is() << '\n'; - std::cout << std::boolalpha << "(2) " << j["k2"].is() << '\n'; - std::cout << std::boolalpha << "(3) " << j["k2"].is() << '\n'; - std::cout << std::boolalpha << "(4) " << j["k3"].is() << '\n'; - std::cout << std::boolalpha << "(5) " << j["k3"].is() << '\n'; - std::cout << std::boolalpha << "(6) " << j["k4"].is() << '\n'; - std::cout << std::boolalpha << "(7) " << j["k4"].is() << '\n'; - std::cout << std::boolalpha << "(8) " << j["k5"].is() << '\n'; - std::cout << std::boolalpha << "(9) " << j["k5"].is() << '\n'; - std::cout << std::boolalpha << "(10) " << j["k6"].is() << '\n'; - std::cout << '\n'; - std::cout << "(1) " << j["k1"].as() << '\n'; - std::cout << "(2) " << j["k2"].as() << '\n'; - std::cout << "(3) " << j["k2"].as() << '\n'; - std::cout << "(4) " << j["k3"].as() << '\n'; - std::cout << "(5) " << j["k3"].as() << '\n'; - std::cout << "(6) " << j["k4"].as() << '\n'; - std::cout << "(7) " << j["k4"].as() << '\n'; - std::cout << std::boolalpha << "(8) " << j["k5"].as() << '\n'; - std::cout << std::boolalpha << "(9) " << j["k5"].as() << '\n'; - std::cout << "(10) " << j["k6"].as() << '\n'; -} - -void byte_string_from_initializer_list() -{ - json j(byte_string({'H','e','l','l','o'})); - byte_string bs = j.as(); - - std::cout << "(1) "<< bs << "\n\n"; - - std::cout << "(2) "; - for (auto b : bs) - { - std::cout << (char)b; - } - std::cout << "\n\n"; - - std::cout << "(3) " << j << std::endl; -} - -void byte_string_from_char_array() -{ - json j(byte_string("Hello")); - byte_string bs = j.as(); - - std::cout << "(1) "<< bs << "\n\n"; - - std::cout << "(2) "; - for (auto b : bs) - { - std::cout << (char)b; - } - std::cout << "\n\n"; - - std::cout << "(3) " << j << std::endl; -} - -void introspection_example() -{ - std::string path = "./input/books.json"; - std::fstream is(path); - if (!is) - { - std::cout << "Cannot open " << path << std::endl; - return; - } - json val = json::parse(is); - std::cout << std::boolalpha; - std::cout << "Is this an object? " << val.is() << ", or an array? " << val.is() << std::endl; - - if (val.is()) - { - for (size_t i = 0; i < val.size(); ++i) - { - json& elem = val[i]; - std::cout << "Is element " << i << " an object? " << elem.is() << std::endl; - if (elem.is()) - { - for (auto it = elem.object_range().begin(); it != elem.object_range().end(); ++it){ - std::cout << "Is member " << it->key() << " a string? " << it->value().is() << ", or a double? " << it->value().is() << ", or perhaps an int? " << it->value().is() << std::endl; - - } - } - } - } -} - -void json_accessor_examples() -{ - is_as_examples(); - - introspection_example(); - - byte_string_from_initializer_list(); - - byte_string_from_char_array(); -} - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/json_cursor_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/json_cursor_examples.cpp deleted file mode 100644 index 7494f7132a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/json_cursor_examples.cpp +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include - -using namespace jsoncons; - -// Example JSON text -const std::string example = R"( -[ - { - "author" : "Haruki Murakami", - "title" : "Hard-Boiled Wonderland and the End of the World", - "isbn" : "0679743464", - "publisher" : "Vintage", - "date" : "1993-03-02", - "price": 18.90 - }, - { - "author" : "Graham Greene", - "title" : "The Comedians", - "isbn" : "0099478374", - "publisher" : "Vintage Classics", - "date" : "2005-09-21", - "price": 15.74 - } -] -)"; - -// In the example, the application pulls the next event in the -// JSON input stream by calling next(). - -void reading_a_json_stream() -{ - std::istringstream is(example); - - json_cursor reader(is); - - for (; !reader.done(); reader.next()) - { - const auto& event = reader.current(); - switch (event.event_type()) - { - case staj_event_type::begin_array: - std::cout << "begin_array\n"; - break; - case staj_event_type::end_array: - std::cout << "end_array\n"; - break; - case staj_event_type::begin_object: - std::cout << "begin_object\n"; - break; - case staj_event_type::end_object: - std::cout << "end_object\n"; - break; - case staj_event_type::name: - // If underlying type is string, can return as string_view - std::cout << "name: " << event.as() << "\n"; - break; - case staj_event_type::string_value: - std::cout << "string_value: " << event.as() << "\n"; - break; - case staj_event_type::null_value: - std::cout << "null_value: " << event.as() << "\n"; - break; - case staj_event_type::bool_value: - std::cout << "bool_value: " << event.as() << "\n"; - break; - case staj_event_type::int64_value: - std::cout << "int64_value: " << event.as() << "\n"; - break; - case staj_event_type::uint64_value: - std::cout << "uint64_value: " << event.as() << "\n"; - break; - case staj_event_type::double_value: - // Return as string, could also use event.as() - std::cout << "double_value: " << event.as() << "\n"; - break; - default: - std::cout << "Unhandled event type\n"; - break; - } - } -} - -class author_filter : public staj_filter -{ - bool accept_next_ = false; -public: - bool accept(const staj_event& event, const ser_context& context) override - { - if (event.event_type() == staj_event_type::name && - event.as() == "author") - { - accept_next_ = true; - return false; - } - else if (accept_next_) - { - accept_next_ = false; - return true; - } - else - { - accept_next_ = false; - return false; - } - } -}; - -// Filtering the stream -void filtering_a_json_stream() -{ - std::istringstream is(example); - - author_filter filter; - json_cursor reader(is, filter); - - for (; !reader.done(); reader.next()) - { - const auto& event = reader.current(); - switch (event.event_type()) - { - case staj_event_type::string_value: - std::cout << event.as() << "\n"; - break; - } - } -} - -void pull_parser_examples() -{ - std::cout << "\nPull parser examples\n\n"; - - reading_a_json_stream(); - std::cout << "\n"; - filtering_a_json_stream(); - - std::cout << "\n"; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/json_filter_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/json_filter_examples.cpp deleted file mode 100644 index 6224fda910..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/json_filter_examples.cpp +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include - -using namespace jsoncons; - -class name_fix_up_filter : public json_filter -{ -public: - name_fix_up_filter(json_content_handler& handler) - : json_filter(handler) - { - } - -private: - bool do_name(const string_view_type& name, - const ser_context& context) override - { - member_name_ = std::string(name); - if (member_name_ != "name") - { - this->to_handler().name(name, context); - } - return true; - } - - bool do_string_value(const string_view_type& s, - semantic_tag tag, - const ser_context& context) override - { - if (member_name_ == "name") - { - size_t end_first = s.find_first_of(" \t"); - size_t start_last = s.find_first_not_of(" \t", end_first); - this->to_handler().name("first-name", context); - string_view_type first = s.substr(0, end_first); - this->to_handler().string_value(first, tag, context); - if (start_last != string_view_type::npos) - { - this->to_handler().name("last-name", context); - string_view_type last = s.substr(start_last); - this->to_handler().string_value(last, tag, context); - } - else - { - std::cerr << "Incomplete name \"" << s - << "\" at line " << context.line() - << " and column " << context.column() << std::endl; - } - } - else - { - this->to_handler().string_value(s, tag, context); - } - return true; - } - - std::string member_name_; -}; - -void name_fix_up_example1() -{ - std::string in_file = "./input/address-book.json"; - std::string out_file = "./output/new-address-book1.json"; - std::ifstream is(in_file); - std::ofstream os(out_file); - - json_encoder encoder(os); - name_fix_up_filter filter(encoder); - json_reader reader(is, filter); - reader.read_next(); -} - -void name_fix_up_example2() -{ - std::string in_file = "./input/address-book.json"; - std::string out_file = "./output/new-address-book2.json"; - std::ifstream is(in_file); - std::ofstream os(out_file); - - json j; - is >> j; - - json_encoder encoder(os); - name_fix_up_filter filter(encoder); - j.dump(filter); -} - -void change_member_name_example() -{ - std::string s = R"({"first":1,"second":2,"fourth":3,"fifth":4})"; - - json_encoder encoder(std::cout); - - // Filters can be chained - rename_object_member_filter filter2("fifth", "fourth", encoder); - rename_object_member_filter filter1("fourth", "third", filter2); - - // A filter can be passed to any function that takes - // a json_content_handler ... - std::cout << "(1) "; - std::istringstream is(s); - json_reader reader(is, filter1); - reader.read(); - std::cout << std::endl; - - // or a json_content_handler - std::cout << "(2) "; - ojson j = ojson::parse(s); - j.dump(filter1); - std::cout << std::endl; -} - -void json_filter_examples() -{ - std::cout << "\njson_filter examples\n\n"; - name_fix_up_example1(); - name_fix_up_example2(); - change_member_name_example(); - - std::cout << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/json_parser_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/json_parser_examples.cpp deleted file mode 100644 index dfa3337cdd..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/json_parser_examples.cpp +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include - -using namespace jsoncons; - -void incremental_parsing_example1() -{ - jsoncons::json_decoder decoder; - json_parser parser; - try - { - parser.update("[fal"); - parser.parse_some(decoder); - std::cout << "(1) done: " << std::boolalpha << parser.done() << ", source_exhausted: " << parser.source_exhausted() << "\n\n"; - - parser.update("se]"); - parser.parse_some(decoder); - std::cout << "(2) done: " << std::boolalpha << parser.done() << ", source_exhausted: " << parser.source_exhausted() << "\n\n"; - - parser.finish_parse(decoder); - std::cout << "(3) done: " << std::boolalpha << parser.done() << ", source_exhausted: " << parser.source_exhausted() << "\n\n"; - - parser.check_done(); - std::cout << "(4) done: " << std::boolalpha << parser.done() << ", source_exhausted: " << parser.source_exhausted() << "\n\n"; - - json j = decoder.get_result(); - std::cout << "(5) " << j << "\n\n"; - } - catch (const ser_error& e) - { - std::cout << e.what() << std::endl; - } -} - -void incremental_parsing_example2() -{ - jsoncons::json_decoder decoder; - json_parser parser; - try - { - parser.update("10"); - parser.parse_some(decoder); - std::cout << "(1) done: " << std::boolalpha << parser.done() << ", source_exhausted: " << parser.source_exhausted() << "\n\n"; - - parser.update(".5"); - parser.parse_some(decoder); // This is the end, but the parser can't tell - std::cout << "(2) done: " << std::boolalpha << parser.done() << ", source_exhausted: " << parser.source_exhausted() << "\n\n"; - - parser.finish_parse(decoder); // Indicates that this is the end - std::cout << "(3) done: " << std::boolalpha << parser.done() << ", source_exhausted: " << parser.source_exhausted() << "\n\n"; - - parser.check_done(); // Checks if there are any unconsumed - // non-whitespace characters in the input - std::cout << "(4) done: " << std::boolalpha << parser.done() << ", source_exhausted: " << parser.source_exhausted() << "\n\n"; - - json j = decoder.get_result(); - std::cout << "(5) " << j << "\n"; - } - catch (const ser_error& e) - { - std::cout << e.what() << std::endl; - } -} - -void incremental_parsing_example3() -{ - jsoncons::json_decoder decoder; - json_parser parser; - try - { - parser.update("[10"); - parser.parse_some(decoder); - std::cout << "(1) done: " << std::boolalpha << parser.done() << ", source_exhausted: " << parser.source_exhausted() << "\n\n"; - - parser.update(".5]{}"); - parser.parse_some(decoder); // The parser reaches the end at ']' - std::cout << "(2) done: " << std::boolalpha << parser.done() << ", source_exhausted: " << parser.source_exhausted() << "\n\n"; - - parser.finish_parse(decoder); // Indicates that this is the end - std::cout << "(3) done: " << std::boolalpha << parser.done() << ", source_exhausted: " << parser.source_exhausted() << "\n\n"; - - parser.check_done(); // Checks if there are any unconsumed - // non-whitespace characters in the input - // (there are) - } - catch (const ser_error& e) - { - std::cout << "(4) " << e.what() << std::endl; - } -} - -void parse_nan_replacement_example() -{ - std::string s = R"( - { - "A" : "NaN", - "B" : "Infinity", - "C" : "-Infinity" - } - )"; - - json_options options; - options.nan_to_str("NaN") - .inf_to_str("Infinity"); - - jsoncons::json_decoder decoder; - json_parser parser(options); - try - { - parser.update(s); - parser.parse_some(decoder); - parser.finish_parse(decoder); - parser.check_done(); - } - catch (const ser_error& e) - { - std::cout << e.what() << std::endl; - } - - json j = decoder.get_result(); // performs move - if (j["A"].is()) - { - std::cout << "A: " << j["A"].as() << std::endl; - } - if (j["B"].is()) - { - std::cout << "B: " << j["B"].as() << std::endl; - } - if (j["C"].is()) - { - std::cout << "C: " << j["C"].as() << std::endl; - } -} - -void json_parser_examples() -{ - std::cout << "\njson_parser examples\n\n"; - incremental_parsing_example1(); - incremental_parsing_example2(); - incremental_parsing_example3(); - parse_nan_replacement_example(); - - std::cout << std::endl; -} - - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/jsonpatch_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/jsonpatch_examples.cpp deleted file mode 100644 index 1846f9f7b2..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/jsonpatch_examples.cpp +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under Boost license - -#include -#include - -using namespace jsoncons; - -void jsonpatch_add_add() -{ - // Apply a JSON Patch - - json doc = R"( - { "foo": "bar"} - )"_json; - - json doc2 = doc; - - json patch = R"( - [ - { "op": "add", "path": "/baz", "value": "qux" }, - { "op": "add", "path": "/foo", "value": [ "bar", "baz" ] } - ] - )"_json; - - std::error_code ec; - jsonpatch::apply_patch(doc, patch, ec); - - std::cout << "(1)\n" << pretty_print(doc) << std::endl; - - // Create a JSON Patch - - auto patch2 = jsonpatch::from_diff(doc2,doc); - - std::cout << "(2)\n" << pretty_print(patch2) << std::endl; - - jsonpatch::apply_patch(doc2,patch2,ec); - - std::cout << "(3)\n" << pretty_print(doc2) << std::endl; -} - -void jsonpatch_add_add_add_failed1() -{ - json target = R"( - { "foo": "bar"} - )"_json; - - json patch = R"( - [ - { "op": "add", "path": "/baz", "value": "qux" }, - { "op": "add", "path": "/foo", "value": [ "bar", "baz" ] }, - { "op": "add", "path": "/baz/bat", "value": "qux" } // nonexistent target - ] - )"_json; - - try - { - jsonpatch::apply_patch(target, patch); - } - catch (const jsonpatch::jsonpatch_error& e) - { - std::cout << "(1) " << e.what() << std::endl; - std::cout << "(2) " << target << std::endl; - } -} - -void jsonpatch_add_add_add_failed2() -{ - json target = R"( - { "foo": "bar"} - )"_json; - - json patch = R"( - [ - { "op": "add", "path": "/baz", "value": "qux" }, - { "op": "add", "path": "/foo", "value": [ "bar", "baz" ] }, - { "op": "add", "path": "/baz/bat", "value": "qux" } // nonexistent target - ] - )"_json; - - std::error_code ec; - jsonpatch::apply_patch(target, patch, ec); - - std::cout << "(1) " << std::error_code(ec).message() << std::endl; - std::cout << "(2) " << target << std::endl; -} - -void create_a_json_patch() -{ - json source = R"( - {"/": 9, "foo": "bar"} - )"_json; - - json target = R"( - { "baz":"qux", "foo": [ "bar", "baz" ]} - )"_json; - - auto patch = jsonpatch::from_diff(source, target); - - std::error_code ec; - jsonpatch::apply_patch(source, patch, ec); - - std::cout << "(1)\n" << pretty_print(patch) << std::endl; - std::cout << "(2)\n" << pretty_print(source) << std::endl; -} - -void jsonpatch_examples() -{ - std::cout << "\njsonpatch examples\n\n"; - create_a_json_patch(); - jsonpatch_add_add(); - jsonpatch_add_add_add_failed2(); - jsonpatch_add_add_add_failed1(); - std::cout << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/jsonpath_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/jsonpath_examples.cpp deleted file mode 100644 index 113787791a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/jsonpath_examples.cpp +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include - -using namespace jsoncons; - -void json_query_examples() -{ - std::ifstream is("./input/booklist.json"); - json booklist = json::parse(is); - - // The authors of books that are cheaper than $10 - json result1 = jsonpath::json_query(booklist, "$.store.book[?(@.price < 10)].author"); - std::cout << "(1) " << result1 << "\n"; - - // The number of books - json result2 = jsonpath::json_query(booklist, "$..book.length"); - std::cout << "(2) " << result2 << "\n"; - - // The third book - json result3 = jsonpath::json_query(booklist, "$..book[2]"); - std::cout << "(3)\n" << pretty_print(result3) << "\n"; - - // All books whose author's name starts with Evelyn - json result4 = jsonpath::json_query(booklist, "$.store.book[?(@.author =~ /Evelyn.*?/)]"); - std::cout << "(4)\n" << pretty_print(result4) << "\n"; - - // The titles of all books that have isbn number - json result5 = jsonpath::json_query(booklist, "$..book[?(@.isbn)].title"); - std::cout << "(5) " << result5 << "\n"; - - // All authors and titles of books - json result6 = jsonpath::json_query(booklist, "$['store']['book']..['author','title']"); - std::cout << "(6)\n" << pretty_print(result6) << "\n"; - - // Union of two ranges of book titles - json result7 = jsonpath::json_query(booklist, "$..book[1:2,2:4].title"); - std::cout << "(7)\n" << pretty_print(result7) << "\n"; - - // Union of a subset of book titles identified by index - json result8 = jsonpath::json_query(booklist, "$.store[book[0].title,book[1].title,book[3].title]"); - std::cout << "(8)\n" << pretty_print(result8) << "\n"; - - // Union of third book title and all book titles with price > 10 - json result9 = jsonpath::json_query(booklist, "$.store[book[3].title,book[?(@.price > 10)].title]"); - std::cout << "(9)\n" << pretty_print(result9) << "\n"; - - // Intersection of book titles with category fiction and price < 15 - json result10 = jsonpath::json_query(booklist, "$.store.book[?(@.category == 'fiction')][?(@.price < 15)].title"); - std::cout << "(10)\n" << pretty_print(result10) << "\n"; - - // Normalized path expressions - json result11 = jsonpath::json_query(booklist, "$.store.book[?(@.author =~ /Evelyn.*?/)]", jsonpath::result_type::path); - std::cout << "(11)\n" << pretty_print(result11) << "\n"; - - // All titles whose author's second name is 'Waugh' - json result12 = jsonpath::json_query(booklist,"$.store.book[?(tokenize(@.author,'\\\\s+')[1] == 'Waugh')].title"); - std::cout << "(12)\n" << result12 << "\n"; - - // All keys in the second book - json result13 = jsonpath::json_query(booklist,"keys($.store.book[1])[*]"); - std::cout << "(13)\n" << result13 << "\n"; -} - -void json_replace_example1() -{ - std::ifstream is("./input/booklist.json"); - json booklist = json::parse(is); - - jsonpath::json_replace(booklist,"$.store.book[?(@.isbn == '0-553-21311-3')].price",10.0); - std::cout << pretty_print(booklist) << "\n"; -} - -void json_replace_example2() -{ - json j; - try - { - j = json::parse(R"( -{"store": -{"book": [ -{"category": "reference", -"author": "Margaret Weis", -"title": "Dragonlance Series", -"price": 31.96}, -{"category": "reference", -"author": "Brent Weeks", -"title": "Night Angel Trilogy", -"price": 14.70 -}]}} -)"); - } - catch (const std::exception& e) - { - std::cout << e.what() << "\n"; - } - - std::cout << ("1\n") << pretty_print(j) << "\n"; - - jsonpath::json_replace(j,"$..book[?(@.price==31.96)].price", 30.9); - - std::cout << ("2\n") << pretty_print(j) << "\n"; -} - -void jsonpath_complex_examples() -{ - const json j = json::parse(R"( - [ - { - "root": { - "id" : 10, - "second": [ - { - "names": [ - 2 - ], - "complex": [ - { - "names": [ - 1 - ], - "panels": [ - { - "result": [ - 1 - ] - }, - { - "result": [ - 1, - 2, - 3, - 4 - ] - }, - { - "result": [ - 1 - ] - } - ] - } - ] - } - ] - } - }, - { - "root": { - "id" : 20, - "second": [ - { - "names": [ - 2 - ], - "complex": [ - { - "names": [ - 1 - ], - "panels": [ - { - "result": [ - 1 - ] - }, - { - "result": [ - 3, - 4, - 5, - 6 - ] - }, - { - "result": [ - 1 - ] - } - ] - } - ] - } - ] - } - } - ] - )"); - - // Find all arrays of elements where result.length is 4 - json result1 = jsonpath::json_query(j,"$..[?(@.result.length == 4)].result"); - std::cout << "(1) " << result1 << "\n"; - - // Find array of elements that has id 10 and result.length is 4 - json result2 = jsonpath::json_query(j,"$..[?(@.id == 10)]..[?(@.result.length == 4)].result"); - std::cout << "(2) " << result2 << "\n"; - - // Find all arrays of elements where result.length is 4 and that have value 3 - json result3 = jsonpath::json_query(j,"$..[?(@.result.length == 4 && (@.result[0] == 3 || @.result[1] == 3 || @.result[2] == 3 || @.result[3] == 3))].result"); - std::cout << "(3) " << result3 << "\n"; -} - -void jsonpath_union() -{ - const json root = json::parse(R"( -{ - "firstName": "John", - "lastName" : "doe", - "age" : 26, - "address" : { - "streetAddress": "naist street", - "city" : "Nara", - "postalCode" : "630-0192" - }, - "phoneNumbers": [ - { - "type" : "iPhone", - "number": "0123-4567-8888" - }, - { - "type" : "home", - "number": "0123-4567-8910" - } - ] -} )"); - - std::string path = "$..[firstName,address.city]"; - json result = jsonpath::json_query(root,path); - - std::cout << result << "\n"; -} - -void jsonpath_examples() -{ - std::cout << "\nJsonPath examples\n\n"; - std::cout << "Here\n"; - json_replace_example1(); - json_replace_example2(); - jsonpath_complex_examples(); - jsonpath_union(); - json_query_examples(); - std::cout << "\n"; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/jsonpointer_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/jsonpointer_examples.cpp deleted file mode 100644 index 2e6045bddd..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/jsonpointer_examples.cpp +++ /dev/null @@ -1,474 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under Boost license - -#include -#include - -using namespace jsoncons; - -void jsonpointer_select_RFC6901() -{ - // Example from RFC 6901 - auto j = json::parse(R"( - { - "foo": ["bar", "baz"], - "": 0, - "a/b": 1, - "c%d": 2, - "e^f": 3, - "g|h": 4, - "i\\j": 5, - "k\"l": 6, - " ": 7, - "m~n": 8 - } - )"); - - try - { - const json& result1 = jsonpointer::get(j, ""); - std::cout << "(1) " << result1 << std::endl; - const json& result2 = jsonpointer::get(j, "/foo"); - std::cout << "(2) " << result2 << std::endl; - const json& result3 = jsonpointer::get(j, "/foo/0"); - std::cout << "(3) " << result3 << std::endl; - const json& result4 = jsonpointer::get(j, "/"); - std::cout << "(4) " << result4 << std::endl; - const json& result5 = jsonpointer::get(j, "/a~1b"); - std::cout << "(5) " << result5 << std::endl; - const json& result6 = jsonpointer::get(j, "/c%d"); - std::cout << "(6) " << result6 << std::endl; - const json& result7 = jsonpointer::get(j, "/e^f"); - std::cout << "(7) " << result7 << std::endl; - const json& result8 = jsonpointer::get(j, "/g|h"); - std::cout << "(8) " << result8 << std::endl; - const json& result9 = jsonpointer::get(j, "/i\\j"); - std::cout << "(9) " << result9 << std::endl; - const json& result10 = jsonpointer::get(j, "/k\"l"); - std::cout << "(10) " << result10 << std::endl; - const json& result11 = jsonpointer::get(j, "/ "); - std::cout << "(11) " << result11 << std::endl; - const json& result12 = jsonpointer::get(j, "/m~0n"); - std::cout << "(12) " << result12 << std::endl; - } - catch (const jsonpointer::jsonpointer_error& e) - { - std::cerr << e.what() << std::endl; - } -} - -void jsonpointer_contains() -{ - // Example from RFC 6901 - auto j = json::parse(R"( - { - "foo": ["bar", "baz"], - "": 0, - "a/b": 1, - "c%d": 2, - "e^f": 3, - "g|h": 4, - "i\\j": 5, - "k\"l": 6, - " ": 7, - "m~n": 8 - } - )"); - - std::cout << "(1) " << jsonpointer::contains(j, "/foo/0") << std::endl; - std::cout << "(2) " << jsonpointer::contains(j, "e^g") << std::endl; -} - -void jsonpointer_select_author() -{ - auto j = json::parse(R"( - [ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - } - ] - )"); - - // Using exceptions to report errors - try - { - json result = jsonpointer::get(j, "/1/author"); - std::cout << "(1) " << result << std::endl; - } - catch (const jsonpointer::jsonpointer_error& e) - { - std::cout << e.what() << std::endl; - } - - // Using error codes to report errors - std::error_code ec; - const json& result = jsonpointer::get(j, "/0/title", ec); - - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << "(2) " << result << std::endl; - } -} - -void jsonpointer_add_member_to_object() -{ - auto target = json::parse(R"( - { "foo": "bar"} - )"); - - std::error_code ec; - jsonpointer::insert(target, "/baz", json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} - -void jsonpointer_add_element_to_array() -{ - auto target = json::parse(R"( - { "foo": [ "bar", "baz" ] } - )"); - - std::error_code ec; - jsonpointer::insert(target, "/foo/1", json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} - -void jsonpointer_add_element_to_end_array() -{ - auto target = json::parse(R"( - { "foo": [ "bar", "baz" ] } - )"); - - std::error_code ec; - jsonpointer::insert(target, "/foo/-", json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} - -void jsonpointer_insert_name_exists() -{ - auto target = json::parse(R"( - { "foo": "bar", "baz" : "abc"} - )"); - - std::error_code ec; - jsonpointer::insert(target, "/baz", json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} - -void jsonpointer_add_element_outside_range() -{ - auto target = json::parse(R"( - { "foo": [ "bar", "baz" ] } - )"); - - std::error_code ec; - jsonpointer::insert_or_assign(target, "/foo/3", json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} - -void jsonpointer_insert_or_assign_name_exists() -{ - auto target = json::parse(R"( - { "foo": "bar", "baz" : "abc"} - )"); - - std::error_code ec; - jsonpointer::insert_or_assign(target, "/baz", json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} - -void jsonpointer_remove_object_member() -{ - auto target = json::parse(R"( - { "foo": "bar", "baz" : "qux"} - )"); - - std::error_code ec; - jsonpointer::remove(target, "/baz", ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} - -void jsonpointer_remove_array_element() -{ - auto target = json::parse(R"( - { "foo": [ "bar", "qux", "baz" ] } - )"); - - std::error_code ec; - jsonpointer::remove(target, "/foo/1", ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} - -void jsonpointer_replace_object_value() -{ - auto target = json::parse(R"( - { - "baz": "qux", - "foo": "bar" - } - )"); - - std::error_code ec; - jsonpointer::replace(target, "/baz", json("boo"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} - -void jsonpointer_replace_array_value() -{ - auto target = json::parse(R"( - { "foo": [ "bar", "baz" ] } - )"); - - std::error_code ec; - jsonpointer::replace(target, "/foo/1", json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << pretty_print(target) << std::endl; - } -} - -void jsonpointer_error_example() -{ - auto j = json::parse(R"( - [ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - } - ] - )"); - - try - { - json result = jsonpointer::get(j, "/1/isbn"); - std::cout << "succeeded?" << std::endl; - std::cout << result << std::endl; - } - catch (const jsonpointer::jsonpointer_error& e) - { - std::cout << "Caught jsonpointer_error with category " << e.code().category().name() - << ", code " << e.code().value() - << " and message \"" << e.what() << "\"" << std::endl; - } -} - -void jsonpointer_get_examples() -{ - { - json j = json::array{"baz","foo"}; - json& item = jsonpointer::get(j,"/0"); - std::cout << "(1) " << item << std::endl; - - //std::vector u; - //cbor::encode_cbor(j,u); - //for (auto c : u) - //{ - // std::cout << "0x" << std::hex << (int)c << ","; - //} - //std::cout << std::endl; - } - { - const json j = json::array{"baz","foo"}; - const json& item = jsonpointer::get(j,"/1"); - std::cout << "(2) " << item << std::endl; - } - { - json j = json::array{"baz","foo"}; - - std::error_code ec; - json& item = jsonpointer::get(j,"/1",ec); - std::cout << "(4) " << item << std::endl; - } - { - const json j = json::array{"baz","foo"}; - - std::error_code ec; - const json& item = jsonpointer::get(j,"/0",ec); - std::cout << "(5) " << item << std::endl; - } -} - -void jsonpointer_address_example() -{ - auto j = json::parse(R"( - { - "a/b": ["bar", "baz"], - "m~n": ["foo", "qux"] - } - )"); - - jsonpointer::address addr; - addr /= "m~n"; - addr /= "1"; - - std::cout << "(1) " << addr << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& item : addr) - { - std::cout << item << "\n"; - } - std::cout << "\n"; - - json item = jsonpointer::get(j, addr); - std::cout << "(3) " << item << "\n"; -} - -void jsonpointer_address_iterator_example() -{ - jsonpointer::address addr("/store/book/1/author"); - - std::cout << "(1) " << addr << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& token : addr) - { - std::cout << token << "\n"; - } - - std::cout << "\n"; -} - -void jsonpointer_address_append_tokens() -{ - jsonpointer::address addr; - - addr /= "a/b"; - addr /= ""; - addr /= "m~n"; - - std::cout << "(1) " << addr << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& token : addr) - { - std::cout << token << "\n"; - } - - std::cout << "\n"; -} - -void jsonpointer_address_concatentae() -{ - jsonpointer::address addr("/a~1b"); - - addr += jsonpointer::address("//m~0n"); - - std::cout << "(1) " << addr << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& token : addr) - { - std::cout << token << "\n"; - } - - std::cout << "\n"; -} - -void jsonpointer_examples() -{ - std::cout << "\njsonpointer examples\n\n"; - jsonpointer_select_author(); - jsonpointer_address_example(); - jsonpointer_select_RFC6901(); - jsonpointer_add_member_to_object(); - jsonpointer_add_element_to_array(); - jsonpointer_add_element_to_end_array(); - jsonpointer_add_element_outside_range(); - jsonpointer_remove_object_member(); - jsonpointer_remove_array_element(); - jsonpointer_replace_object_value(); - jsonpointer_replace_array_value(); - jsonpointer_contains(); - jsonpointer_error_example(); - jsonpointer_insert_name_exists(); - jsonpointer_insert_or_assign_name_exists(); - jsonpointer_get_examples(); - jsonpointer_address_iterator_example(); - jsonpointer_address_append_tokens(); - jsonpointer_address_concatentae(); -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/jsonpointer_examples.cpp.orig b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/jsonpointer_examples.cpp.orig deleted file mode 100644 index e0c0c579a8..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/jsonpointer_examples.cpp.orig +++ /dev/null @@ -1,494 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under Boost license - -#include -#include - -namespace jp = jsoncons::jsonpointer; - -void jsonpointer_select_RFC6901() -{ - // Example from RFC 6901 - auto j = jsoncons::json::parse(R"( - { - "foo": ["bar", "baz"], - "": 0, - "a/b": 1, - "c%d": 2, - "e^f": 3, - "g|h": 4, - "i\\j": 5, - "k\"l": 6, - " ": 7, - "m~n": 8 - } - )"); - - try - { - const jsoncons::json& result1 = jp::get(j, ""); - std::cout << "(1) " << result1 << std::endl; - const jsoncons::json& result2 = jp::get(j, "/foo"); - std::cout << "(2) " << result2 << std::endl; - const jsoncons::json& result3 = jp::get(j, "/foo/0"); - std::cout << "(3) " << result3 << std::endl; - const jsoncons::json& result4 = jp::get(j, "/"); - std::cout << "(4) " << result4 << std::endl; - const jsoncons::json& result5 = jp::get(j, "/a~1b"); - std::cout << "(5) " << result5 << std::endl; - const jsoncons::json& result6 = jp::get(j, "/c%d"); - std::cout << "(6) " << result6 << std::endl; - const jsoncons::json& result7 = jp::get(j, "/e^f"); - std::cout << "(7) " << result7 << std::endl; - const jsoncons::json& result8 = jp::get(j, "/g|h"); - std::cout << "(8) " << result8 << std::endl; - const jsoncons::json& result9 = jp::get(j, "/i\\j"); - std::cout << "(9) " << result9 << std::endl; - const jsoncons::json& result10 = jp::get(j, "/k\"l"); - std::cout << "(10) " << result10 << std::endl; - const jsoncons::json& result11 = jp::get(j, "/ "); - std::cout << "(11) " << result11 << std::endl; - const jsoncons::json& result12 = jp::get(j, "/m~0n"); - std::cout << "(12) " << result12 << std::endl; - } - catch (const jp::jsonpointer_error& e) - { - std::cerr << e.what() << std::endl; - } -} - -void jsonpointer_contains() -{ - // Example from RFC 6901 - auto j = jsoncons::json::parse(R"( - { - "foo": ["bar", "baz"], - "": 0, - "a/b": 1, - "c%d": 2, - "e^f": 3, - "g|h": 4, - "i\\j": 5, - "k\"l": 6, - " ": 7, - "m~n": 8 - } - )"); - - std::cout << "(1) " << jp::contains(j, "/foo/0") << std::endl; - std::cout << "(2) " << jp::contains(j, "e^g") << std::endl; -} - -void jsonpointer_select_author() -{ - auto j = jsoncons::json::parse(R"( - [ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - } - ] - )"); - - // Using exceptions to report errors - try - { - jsoncons::json result = jp::get(j, "/1/author"); - std::cout << "(1) " << result << std::endl; - } - catch (const jp::jsonpointer_error& e) - { - std::cout << e.what() << std::endl; - } - - // Using error codes to report errors - std::error_code ec; - const jsoncons::json& result = jp::get(j, "/0/title", ec); - - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << "(2) " << result << std::endl; - } -} - -void jsonpointer_add_member_to_object() -{ - auto target = jsoncons::json::parse(R"( - { "foo": "bar"} - )"); - - std::error_code ec; - jp::insert(target, "/baz", jsoncons::json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} - -void jsonpointer_add_element_to_array() -{ - auto target = jsoncons::json::parse(R"( - { "foo": [ "bar", "baz" ] } - )"); - - std::error_code ec; - jp::insert(target, "/foo/1", jsoncons::json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} - -void jsonpointer_add_element_to_end_array() -{ - auto target = jsoncons::json::parse(R"( - { "foo": [ "bar", "baz" ] } - )"); - - std::error_code ec; - jp::insert(target, "/foo/-", jsoncons::json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} - -void jsonpointer_insert_name_exists() -{ - auto target = jsoncons::json::parse(R"( - { "foo": "bar", "baz" : "abc"} - )"); - - std::error_code ec; - jp::insert(target, "/baz", jsoncons::json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} - -void jsonpointer_add_element_outside_range() -{ - auto target = jsoncons::json::parse(R"( - { "foo": [ "bar", "baz" ] } - )"); - - std::error_code ec; - jp::insert_or_assign(target, "/foo/3", jsoncons::json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} - -void jsonpointer_insert_or_assign_name_exists() -{ - auto target = jsoncons::json::parse(R"( - { "foo": "bar", "baz" : "abc"} - )"); - - std::error_code ec; - jp::insert_or_assign(target, "/baz", jsoncons::json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} - -void jsonpointer_remove_object_member() -{ - auto target = jsoncons::json::parse(R"( - { "foo": "bar", "baz" : "qux"} - )"); - - std::error_code ec; - jp::remove(target, "/baz", ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} - -void jsonpointer_remove_array_element() -{ - auto target = jsoncons::json::parse(R"( - { "foo": [ "bar", "qux", "baz" ] } - )"); - - std::error_code ec; - jp::remove(target, "/foo/1", ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} - -void jsonpointer_replace_object_value() -{ - auto target = jsoncons::json::parse(R"( - { - "baz": "qux", - "foo": "bar" - } - )"); - - std::error_code ec; - jp::replace(target, "/baz", jsoncons::json("boo"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << target << std::endl; - } -} - -void jsonpointer_replace_array_value() -{ - auto target = jsoncons::json::parse(R"( - { "foo": [ "bar", "baz" ] } - )"); - - std::error_code ec; - jp::replace(target, "/foo/1", jsoncons::json("qux"), ec); - if (ec) - { - std::cout << ec.message() << std::endl; - } - else - { - std::cout << pretty_print(target) << std::endl; - } -} - -void jsonpointer_error_example() -{ - auto j = jsoncons::json::parse(R"( - [ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - } - ] - )"); - - try - { - jsoncons::json result = jp::get(j, "/1/isbn"); - std::cout << "succeeded?" << std::endl; - std::cout << result << std::endl; - } - catch (const jp::jsonpointer_error& e) - { - std::cout << "Caught jsonpointer_error with category " << e.code().category().name() - << ", code " << e.code().value() - << " and message \"" << e.what() << "\"" << std::endl; - } -} - -void jsonpointer_get_examples() -{ - { - jsoncons::json j = jsoncons::json::array{"baz","foo"}; - jsoncons::json& item = jp::get(j,"/0"); - std::cout << "(1) " << item << std::endl; - - //std::vector u; - //cbor::encode_cbor(j,u); - //for (auto c : u) - //{ - // std::cout << "0x" << std::hex << (int)c << ","; - //} - //std::cout << std::endl; - } - { - const jsoncons::json j = jsoncons::json::array{"baz","foo"}; - const jsoncons::json& item = jp::get(j,"/1"); - std::cout << "(2) " << item << std::endl; - } - { - jsoncons::json j = jsoncons::json::array{"baz","foo"}; - - std::error_code ec; - jsoncons::json& item = jp::get(j,"/1",ec); - std::cout << "(4) " << item << std::endl; - } - { - const jsoncons::json j = jsoncons::json::array{"baz","foo"}; - - std::error_code ec; - const jsoncons::json& item = jp::get(j,"/0",ec); - std::cout << "(5) " << item << std::endl; - } -} - -void jsonpointer_address_example() -{ - auto j = jsoncons::json::parse(R"( - { - "a/b": ["bar", "baz"], - "m~n": ["foo", "qux"] - } - )"); - -<<<<<<< HEAD - jp::address addr; - addr /= "m~n"; - addr /= "1"; -======= - jsonpointer::address p; - p /= "m~n"; - p /= "1"; ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0 - - std::cout << "(1) " << addr << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& item : addr) - { - std::cout << item << "\n"; - } - std::cout << "\n"; - - jsoncons::json item = jp::get(j, addr); - std::cout << "(3) " << item << "\n"; -} - -void jsonpointer_address_iterator_example() -{ -<<<<<<< HEAD - jp::address addr("/store/book/1/author"); -======= - jp::address p("/store/book/1/author"); ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0 - - std::cout << "(1) " << addr << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& token : addr) - { - std::cout << token << "\n"; - } - - std::cout << "\n"; -} - -void jsonpointer_address_append_tokens() -{ -<<<<<<< HEAD - jp::address addr; -======= - jp::address p; ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0 - - addr /= "a/b"; - addr /= ""; - addr /= "m~n"; - - std::cout << "(1) " << addr << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& token : addr) - { - std::cout << token << "\n"; - } - - std::cout << "\n"; -} - -void jsonpointer_address_concatentae() -{ -<<<<<<< HEAD - jp::address addr("/a~1b"); - - addr += jp::address("//m~0n"); -======= - jp::address p("/a~1b"); - - p += jp::address("//m~0n"); ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0 - - std::cout << "(1) " << addr << "\n\n"; - - std::cout << "(2)\n"; - for (const auto& token : addr) - { - std::cout << token << "\n"; - } - - std::cout << "\n"; -} - -void jsonpointer_examples() -{ - std::cout << "\njsonpointer examples\n\n"; - jsonpointer_select_author(); - jsonpointer_address_example(); - jsonpointer_select_RFC6901(); - jsonpointer_add_member_to_object(); - jsonpointer_add_element_to_array(); - jsonpointer_add_element_to_end_array(); - jsonpointer_add_element_outside_range(); - jsonpointer_remove_object_member(); - jsonpointer_remove_array_element(); - jsonpointer_replace_object_value(); - jsonpointer_replace_array_value(); - jsonpointer_contains(); - jsonpointer_error_example(); - jsonpointer_insert_name_exists(); - jsonpointer_insert_or_assign_name_exists(); - jsonpointer_get_examples(); - jsonpointer_address_iterator_example(); - jsonpointer_address_append_tokens(); - jsonpointer_address_concatentae(); -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/msgpack_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/msgpack_examples.cpp deleted file mode 100644 index 4dcef22ee9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/msgpack_examples.cpp +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include - -using namespace jsoncons; - -void message_pack_example1() -{ -ojson j1 = ojson::parse(R"( -[ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - } -] -)"); - - std::vector v; - msgpack::encode_msgpack(j1, v); - - ojson j2 = msgpack::decode_msgpack(v); - - std::cout << pretty_print(j2) << std::endl; - - json j3 = msgpack::decode_msgpack(v); - - std::cout << pretty_print(j3) << std::endl; - - std::cout << std::endl; - - //wjson j4 = msgpack::decode_msgpack(v); - - //std::wcout << pretty_print(j4) << std::endl; - - //std::cout << std::endl; -} - -void message_pack_example2() -{ - ojson j1; - j1["zero"] = 0; - j1["one"] = 1; - j1["two"] = 2; - j1["null"] = null_type(); - j1["true"] = true; - j1["false"] = false; - j1["max int64_t"] = (std::numeric_limits::max)(); - j1["max uint64_t"] = (std::numeric_limits::max)(); - j1["min int64_t"] = (std::numeric_limits::lowest)(); - j1["max int32_t"] = (std::numeric_limits::max)(); - j1["max uint32_t"] = (std::numeric_limits::max)(); - j1["min int32_t"] = (std::numeric_limits::lowest)(); - j1["max int16_t"] = (std::numeric_limits::max)(); - j1["max uint16_t"] = (std::numeric_limits::max)(); - j1["min int16_t"] = (std::numeric_limits::lowest)(); - j1["max int8_t"] = (std::numeric_limits::max)(); - j1["max uint8_t"] = (std::numeric_limits::max)(); - j1["min int8_t"] = (std::numeric_limits::lowest)(); - j1["max double"] = (std::numeric_limits::max)(); - j1["min double"] = (std::numeric_limits::lowest)(); - j1["max float"] = (std::numeric_limits::max)(); - j1["zero float"] = 0.0; - j1["min float"] = (std::numeric_limits::lowest)(); - j1["Key too long for small string optimization"] = "String too long for small string optimization"; - - std::vector v; - msgpack::encode_msgpack(j1, v); - - ojson j2 = msgpack::decode_msgpack(v); - - std::cout << pretty_print(j2) << std::endl; - - std::cout << std::endl; -} - -void msgpack_examples() -{ - std::cout << "\nmsgpack examples\n\n"; - message_pack_example1(); - message_pack_example2(); - std::cout << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/ojson_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/ojson_examples.cpp deleted file mode 100644 index 41650ce788..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/ojson_examples.cpp +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include - -using namespace jsoncons; - -void ojson_examples() -{ - std::cout << "\nojson examples\n\n"; - - ojson o = ojson::parse(R"( - { - "street_number" : "100", - "street_name" : "Queen St W", - "city" : "Toronto", - "country" : "Canada" - } - )"); - - std::cout << pretty_print(o) << std::endl; - - o.insert_or_assign("postal_code", "M5H 2N2"); - std::cout << pretty_print(o) << std::endl; - - ojson o2 = o; - - ojson o3 = o; - o3["street_name"] = "Queen St W"; - - auto it = o.find("country"); - o.insert_or_assign(it,"province","Ontario"); - - o.insert_or_assign("unit_type","O"); - - std::cout << pretty_print(o) << std::endl; - - o.erase("unit_type"); - - std::cout << pretty_print(o) << std::endl; - - std::cout << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/polymorphic_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/polymorphic_examples.cpp deleted file mode 100644 index bdacfebb1a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/polymorphic_examples.cpp +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright 2016 Daniel Parker -// Distributed under Boost license - -#include "example_types.hpp" -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -namespace ns { - -class Employee -{ - std::string firstName_; - std::string lastName_; -public: - Employee(const std::string& firstName, const std::string& lastName) - : firstName_(firstName), lastName_(lastName) - { - } - virtual ~Employee() = default; - - virtual double calculatePay() const = 0; - - const std::string& firstName() const {return firstName_;} - const std::string& lastName() const {return lastName_;} -}; - -class HourlyEmployee : public Employee -{ - double wage_; - unsigned hours_; -public: - HourlyEmployee(const std::string& firstName, const std::string& lastName, - double wage, unsigned hours) - : Employee(firstName, lastName), - wage_(wage), hours_(hours) - { - } - HourlyEmployee(const HourlyEmployee&) = default; - HourlyEmployee(HourlyEmployee&&) = default; - HourlyEmployee& operator=(const HourlyEmployee&) = default; - HourlyEmployee& operator=(HourlyEmployee&&) = default; - - double wage() const {return wage_;} - - unsigned hours() const {return hours_;} - - double calculatePay() const override - { - return wage_*hours_; - } -}; - -class CommissionedEmployee : public Employee -{ - double baseSalary_; - double commission_; - unsigned sales_; -public: - CommissionedEmployee(const std::string& firstName, const std::string& lastName, - double baseSalary, double commission, unsigned sales) - : Employee(firstName, lastName), - baseSalary_(baseSalary), commission_(commission), sales_(sales) - { - } - CommissionedEmployee(const CommissionedEmployee&) = default; - CommissionedEmployee(CommissionedEmployee&&) = default; - CommissionedEmployee& operator=(const CommissionedEmployee&) = default; - CommissionedEmployee& operator=(CommissionedEmployee&&) = default; - - double baseSalary() const - { - return baseSalary_; - } - - double commission() const - { - return commission_; - } - - unsigned sales() const - { - return sales_; - } - - double calculatePay() const override - { - return baseSalary_ + commission_*sales_; - } -}; -} // ns - -JSONCONS_GETTER_CTOR_TRAITS_DECL(ns::HourlyEmployee, firstName, lastName, wage, hours) -JSONCONS_GETTER_CTOR_TRAITS_DECL(ns::CommissionedEmployee, firstName, lastName, baseSalary, commission, sales) - -namespace jsoncons { - -template -struct json_type_traits> -{ - static bool is(const Json& j) noexcept - { - return j.is() || j.is(); - } - static std::shared_ptr as(const Json& j) - { - if (j.at("type").as() == "Hourly") - { - return std::make_shared(j.as()); - } - else if (j.at("type").as() == "Commissioned") - { - return std::make_shared(j.as()); - } - else - { - throw std::runtime_error("Not an employee"); - } - } - static Json to_json(const std::shared_ptr& ptr) - { - if (ns::HourlyEmployee* p = dynamic_cast(ptr.get())) - { - Json j(*p); - j.try_emplace("type","Hourly"); - return j; - } - else if (ns::CommissionedEmployee* p = dynamic_cast(ptr.get())) - { - Json j(*p); - j.try_emplace("type","Commissioned"); - return j; - } - else - { - throw std::runtime_error("Not an employee"); - } - } -}; - -} // jsoncons - -void employee_polymorphic_example() -{ - std::vector> v; - - v.push_back(std::make_shared("John", "Smith", 40.0, 1000)); - v.push_back(std::make_shared("Jane", "Doe", 30000, 0.25, 1000)); - - json j(v); - std::cout << pretty_print(j) << "\n\n"; - - assert(j[0].is()); - assert(!j[0].is()); - assert(!j[1].is()); - assert(j[1].is()); - - - for (size_t i = 0; i < j.size(); ++i) - { - auto p = j[i].as>(); - assert(p->firstName() == v[i]->firstName()); - assert(p->lastName() == v[i]->lastName()); - assert(p->calculatePay() == v[i]->calculatePay()); - } -} - -void polymorphic_examples() -{ - std::cout << "\nType extensibility examples\n\n"; - - employee_polymorphic_example(); - - std::cout << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/readme_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/readme_examples.cpp deleted file mode 100644 index 3725d6d757..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/readme_examples.cpp +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include - -namespace readme -{ - using namespace jsoncons; - - void playing_around() - { - // Construct some CBOR using the streaming API - std::vector b; - cbor::cbor_bytes_encoder encoder(b); - encoder.begin_array(); // indefinite length outer array - encoder.begin_array(3); // a fixed length array - encoder.string_value("foo"); - encoder.byte_string_value(byte_string{'P','u','s','s'}); // no suggested conversion - encoder.string_value("-18446744073709551617", semantic_tag::bigint); - encoder.end_array(); - encoder.end_array(); - encoder.flush(); - - // Print bytes - std::cout << "(1) "; - for (auto c : b) - { - std::cout << std::hex << std::setprecision(2) << std::setw(2) - << std::setfill('0') << static_cast(c); - } - std::cout << "\n\n"; -/* - 9f -- Start indefinte length array - 83 -- Array of length 3 - 63 -- String value of length 3 - 666f6f -- "foo" - 44 -- Byte string value of length 4 - 50757373 -- 'P''u''s''s' - c3 -- Tag 3 (negative bignum) - 49 -- Byte string value of length 9 - 010000000000000000 -- Bytes content - ff -- "break" -*/ - // Unpack bytes into a json variant value, and add some more elements - json j = cbor::decode_cbor(b); - - // Loop over the rows - std::cout << "(2)\n"; - for (const json& row : j.array_range()) - { - std::cout << row << "\n"; - } - std::cout << "\n"; - - // Get bignum value at position 0/2 using jsonpointer - json& v = jsonpointer::get(j, "/0/2"); - std::cout << "(3) " << v.as() << "\n\n"; - - // Print JSON representation with default options - std::cout << "(4)\n"; - std::cout << pretty_print(j) << "\n\n"; - - // Print JSON representation with different options - json_options options; - options.byte_string_format(byte_string_chars_format::base64) - .bigint_format(bigint_chars_format::base64url); - std::cout << "(5)\n"; - std::cout << pretty_print(j, options) << "\n\n"; - - // Add some more elements - - json another_array = json::array(); - another_array.emplace_back(byte_string({'P','u','s','s'}), - semantic_tag::base64); // suggested conversion to base64 - another_array.emplace_back("273.15", semantic_tag::bigdec); - another_array.emplace(another_array.array_range().begin(),"bar"); // place at front - - j.push_back(std::move(another_array)); - std::cout << "(6)\n"; - std::cout << pretty_print(j) << "\n\n"; - - // Get big decimal value at position /1/2 using jsonpointer - json& ref = jsonpointer::get(j, "/1/2"); - std::cout << "(7) " << ref.as() << "\n\n"; - -#if (defined(__GNUC__) || defined(__clang__)) && (!defined(__STRICT_ANSI__) && defined(_GLIBCXX_USE_INT128)) - // e.g. if code compiled with GCC and std=gnu++11 (rather than std=c++11) - __int128 i = j[1][2].as<__int128>(); -#endif - - // Get byte string value at position /1/1 as a byte_string - byte_string bs = j[1][1].as(); - std::cout << "(8) " << bs << "\n\n"; - - // or alternatively as a std::vector - std::vector u = j[1][1].as>(); - - // Repack bytes - std::vector b2; - cbor::encode_cbor(j, b2); - - // Print the repacked bytes - std::cout << "(9) "; - for (auto c : b2) - { - std::cout << std::hex << std::setprecision(2) << std::setw(2) - << std::setfill('0') << static_cast(c); - } - std::cout << "\n\n"; -/* - 82 -- Array of length 2 - 83 -- Array of length 3 - 63 -- String value of length 3 - 666f6f -- "foo" - 44 -- Byte string value of length 4 - 50757373 -- 'P''u''s''s' - c3 -- Tag 3 (negative bignum) - 49 -- Byte string value of length 9 - 010000000000000000 -- Bytes content - 83 -- Another array of length 3 - 63 -- String value of length 3 - 626172 -- "bar" - d6 - Expected conversion to base64 - 44 -- Byte string value of length 4 - 50757373 -- 'P''u''s''s' - c4 -- Tag 4 (decimal fraction) - 82 -- Array of length 2 - 21 -- -2 - 19 6ab3 -- 27315 -*/ - // Encode to CSV - csv::csv_options csv_options; - csv_options.column_names("Column 1,Column 2,Column 3"); - - std::cout << "(10)\n"; - csv::encode_csv(j, std::cout, csv_options); - } -} - -void readme_examples() -{ - std::cout << "\nReadme examples\n\n"; - - readme::playing_around(); - - std::cout << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/serialization_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/serialization_examples.cpp deleted file mode 100644 index b7704a16c8..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/serialization_examples.cpp +++ /dev/null @@ -1,461 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include - -using namespace jsoncons; - -void serialization_example1() -{ - json val = json::parse(R"( -{ - "sfm_data_version": "0.2", - "root_path": "D:\\Lagring\\Plugg\\Examensarbete\\Data\\images", - "views": [], - "intrinsics": [], - "extrinsics": [ - { - "key": 0, - "value": { - "rotation": [ - [ - 0.89280214808572156, - 0.35067276062587932, - -0.28272413998197254 - ], - [ - -0.090429686592667424, - 0.75440463553446824, - 0.65015084224113584 - ], - [ - 0.44127859245183554, - -0.5548894131618759, - 0.70524530697098287 - ] - ], - "center": [ - -0.60959634064871249, - 0.24123645392011658, - 0.57783384588917808 - ] - } - } - ] -} - -)"); - - std::cout << "Default pretty print" << std::endl; - std::cout << pretty_print(val) << std::endl; - - - std::cout << "array_array_line_splits(line_split_kind::new_line)" << std::endl; - std::cout << "array_object_line_splits(line_split_kind::new_line)" << std::endl; - - json_options options; - - options.array_array_line_splits(line_split_kind::new_line) - .array_object_line_splits(line_split_kind::new_line); - std::cout << pretty_print(val,options) << std::endl; -} - -void serialization_example2() -{ - - json val; - - val["verts"] = json::array{1, 2, 3}; - val["normals"] = json::array{1, 0, 1}; - val["uvs"] = json::array{0, 0, 1, 1}; - - std::cout << "Default object-array same line options" << std::endl; - std::cout << pretty_print(val) << std::endl; - - std::cout << "object_array_line_splits(line_split_kind::same_line)" << std::endl; - json_options options1; - options1.object_array_line_splits(line_split_kind::same_line); - std::cout << pretty_print(val,options1) << std::endl; - - std::cout << "object_array_line_splits(line_split_kind::new_line)" << std::endl; - json_options options2; - options2 .object_array_line_splits(line_split_kind::new_line); - std::cout << pretty_print(val,options2 ) << std::endl; - - std::cout << "object_array_line_splits(line_split_kind::multi_line)" << std::endl; - json_options options3; - options3.object_array_line_splits(line_split_kind::multi_line); - std::cout << pretty_print(val,options3) << std::endl; -} - -void serialization_example3() -{ - { - json val = json::parse(R"( - [ - {"first-name" : "John", - "last-name" : "Doe"}, - {"first-name" : "Jane", - "last-name" : "Doe"} - ] - )"); - - json_options options; - options.array_object_line_splits(line_split_kind::same_line); - std::cout << "array_object_line_splits(line_split_kind::same_line)" << std::endl; - std::cout << pretty_print(val,options) << std::endl; - } - - { - json val = json::parse(R"({ - "verts" : [1, 2, 3], - - "normals" : [1, 0, 1], - - "uvs" : [ 0, 0, 1, 1 ] - } - )"); - std::cout << "Default print" << std::endl; - std::cout << print(val) << std::endl; - - std::cout << "Default pretty print" << std::endl; - std::cout << pretty_print(val) << std::endl; - - json_options options1; - options1.array_array_line_splits(line_split_kind::same_line); - std::cout << pretty_print(val,options1) << std::endl; - - json_options options; - options.object_object_line_splits(line_split_kind::new_line); - std::cout << pretty_print(val,options) << std::endl; - } - - { - json val2 = json::parse(R"( - { - "data": - { - "item": [[2],[4,5,2,3],[4],[4,5,2,3],[2],[4,5,3],[2],[4,3]], //A two-dimensional array - //blank line - "id": [0,1,2,3,4,5,6,7] //A one-dimensional array - } - } - )"); - - std::cout << "Default" << std::endl; - std::cout << pretty_print(val2) << std::endl; - - std::cout << "array_array_line_splits(line_split_kind::new_line)" << std::endl; - json_options options2; - options2 .array_array_line_splits(line_split_kind::new_line); - std::cout << pretty_print(val2,options2 ) << std::endl; - - std::cout << "array_array_line_splits(line_split_kind::same_line)" << std::endl; - json_options options4; - options4.array_array_line_splits(line_split_kind::same_line); - std::cout << pretty_print(val2, options4) << std::endl; - - std::cout << "array_array_line_splits(line_split_kind::same_line)" << std::endl; - json_options options5; - options5.array_array_line_splits(line_split_kind::same_line); - std::cout << pretty_print(val2, options5) << std::endl; - } - - json val3 = json::parse(R"( - { - "data": - { - "item": [[2]] //A two-dimensional array - } - } -)"); - std::cout << "array_array_line_splits(line_split_kind::new_line)" << std::endl; - json_options options6; - options6.array_array_line_splits(line_split_kind::new_line); - std::cout << pretty_print(val3,options6) << std::endl; -} - -void serialization_example4() -{ - json val; - val["data"]["id"] = json::array{0,1,2,3,4,5,6,7}; - val["data"]["item"] = json::array{json::array{2}, - json::array{4,5,2,3}, - json::array{4}, - json::array{4,5,2,3}, - json::array{2}, - json::array{4,5,3}, - json::array{2}, - json::array{4,3}}; - - std::cout << "Default array-array split line options" << std::endl; - std::cout << pretty_print(val) << std::endl; - - std::cout << "Array-array same line options" << std::endl; - json_options options1; - options1.array_array_line_splits(line_split_kind::same_line); - std::cout << pretty_print(val, options1) << std::endl; - - std::cout << "object_array_line_splits(line_split_kind::new_line)" << std::endl; - std::cout << "array_array_line_splits(line_split_kind::same_line)" << std::endl; - json_options options2; - options2 .object_array_line_splits(line_split_kind::new_line) - .array_array_line_splits(line_split_kind::same_line); - std::cout << pretty_print(val, options2 ) << std::endl; - - std::cout << "object_array_line_splits(line_split_kind::new_line)" << std::endl; - std::cout << "array_array_line_splits(line_split_kind::multi_line)" << std::endl; - json_options options3; - options3.object_array_line_splits(line_split_kind::new_line) - .array_array_line_splits(line_split_kind::multi_line); - std::cout << pretty_print(val, options3) << std::endl; - - { - json val = json::parse(R"( - { - "header" : {"properties": {}}, - "data": - { - "tags" : [], - "id" : [1,2,3], - "item": [[1,2,3]] - } - } - )"); - std::cout << "Default" << std::endl; - std::cout << pretty_print(val) << std::endl; - - std::string style1 = "array_array_line_splits(line_split_kind:same_line)"; - std::cout << style1 << std::endl; - json_options options1; - options1.array_array_line_splits(line_split_kind::same_line); - std::cout << pretty_print(val,options1) << std::endl; - - std::string style2 = "array_array_line_splits(line_split_kind::new_line)"; - std::cout << style2 << std::endl; - json_options options2; - options2 .array_array_line_splits(line_split_kind::new_line); - std::cout << pretty_print(val,options2 ) << std::endl; - - std::string style3 = "array_array_line_splits(line_split_kind::multi_line)"; - std::cout << style3 << std::endl; - json_options options3; - options3.array_array_line_splits(line_split_kind::multi_line); - std::cout << pretty_print(val,options3) << std::endl; - - std::string style4 = "object_array_line_splits(line_split_kind:same_line)"; - std::cout << style4 << std::endl; - json_options options4; - options4.object_array_line_splits(line_split_kind::same_line); - std::cout << pretty_print(val,options4) << std::endl; - - std::string style5 = "object_array_line_splits(line_split_kind::new_line)"; - std::cout << style5 << std::endl; - json_options options5; - options5.object_array_line_splits(line_split_kind::new_line); - std::cout << pretty_print(val,options5) << std::endl; - - std::string style6 = "object_array_line_splits(line_split_kind::multi_line)"; - std::cout << style6 << std::endl; - json_options options6; - options6.object_array_line_splits(line_split_kind::multi_line); - std::cout << pretty_print(val,options6) << std::endl; - } -} - -void dump_json_fragments() -{ - const json some_books = json::parse(R"( - [ - { - "title" : "Kafka on the Shore", - "author" : "Haruki Murakami", - "price" : 25.17 - }, - { - "title" : "Women: A Novel", - "author" : "Charles Bukowski", - "price" : 12.00 - } - ] - )"); - - const json more_books = json::parse(R"( - [ - { - "title" : "A Wild Sheep Chase: A Novel", - "author" : "Haruki Murakami", - "price" : 9.01 - }, - { - "title" : "Cutter's Way", - "author" : "Ivan Passer", - "price" : 8.00 - } - ] - )"); - - json_encoder encoder(std::cout); // pretty print - encoder.begin_array(); - for (const auto& book : some_books.array_range()) - { - book.dump(encoder); - } - for (const auto& book : more_books.array_range()) - { - book.dump(encoder); - } - encoder.end_array(); - encoder.flush(); -} - -void nan_inf_replacement() -{ - json j; - j["field1"] = std::sqrt(-1.0); - j["field2"] = 1.79e308 * 1000; - j["field3"] = -1.79e308 * 1000; - - json_options options; - options.nan_to_str("NaN") - .inf_to_str("Inf"); - - std::ostringstream os; - os << pretty_print(j, options); - - std::cout << "(1)\n" << os.str() << std::endl; - - json j2 = json::parse(os.str(),options); - - std::cout << "\n(2) " << j2["field1"].as() << std::endl; - std::cout << "(3) " << j2["field2"].as() << std::endl; - std::cout << "(4) " << j2["field3"].as() << std::endl; - - std::cout << "\n(5)\n" << pretty_print(j2,options) << std::endl; -} - -void bignum_serialization_examples1() -{ - std::string s = "-18446744073709551617"; - - json j(bignum(s.c_str())); - - std::cout << "(default) "; - j.dump(std::cout); - std::cout << "\n\n"; - - std::cout << "(integer) "; - json_options options1; - options1.bigint_format(bigint_chars_format::number); - j.dump(std::cout, options1); - std::cout << "\n\n"; - - std::cout << "(base64) "; - json_options options3; - options3.bigint_format(bigint_chars_format::base64); - j.dump(std::cout, options3); - std::cout << "\n\n"; - - std::cout << "(base64url) "; - json_options options4; - options4.bigint_format(bigint_chars_format::base64url); - j.dump(std::cout, options4); - std::cout << "\n\n"; -} - -void bignum_serialization_examples2() -{ - std::string s = "-18446744073709551617"; - - json j = json::parse(s); - - std::cout << "(1) "; - j.dump(std::cout); - std::cout << "\n\n"; - - std::cout << "(2) "; - json_options options1; - options1.bigint_format(bigint_chars_format::number); - j.dump(std::cout, options1); - std::cout << "\n\n"; - - std::cout << "(3) "; - json_options options2; - options2.bigint_format(bigint_chars_format::base64url); - j.dump(std::cout, options2); - std::cout << "\n\n"; -} - -void bignum_access_examples() -{ - std::string input = "-18446744073709551617"; - - json j = json::parse(input); - - // Access as string - std::string s = j.as(); - std::cout << "(1) " << s << "\n\n"; - - // Access as double - double d = j.as(); - std::cout << "(2) " << std::setprecision(17) << d << "\n\n"; - - // Access as jsoncons::bignum - jsoncons::bignum bn = j.as(); - std::cout << "(3) " << bn << "\n\n"; - - // If your compiler supports extended integral types for which std::numeric_limits is specialized -#if (defined(__GNUC__) || defined(__clang__)) && (!defined(__STRICT_ANSI__) && defined(_GLIBCXX_USE_INT128)) - __int128 i = j.as<__int128>(); - std::cout << "(4) " << i << "\n\n"; -#endif -} - -void decimal_precision_examples() -{ - std::string s = R"( - { - "a" : 12.00, - "b" : 1.23456789012345678901234567890 - } - )"; - - // Default - json j = json::parse(s); - - std::cout.precision(15); - - // Access as string - std::cout << "(1) a: " << j["a"].as() << ", b: " << j["b"].as() << "\n"; - // Access as double - std::cout << "(2) a: " << j["a"].as() << ", b: " << j["b"].as() << "\n\n"; - - // Using lossless_number option - json_options options; - options.lossless_number(true); - - json j2 = json::parse(s, options); - // Access as string - std::cout << "(3) a: " << j2["a"].as() << ", b: " << j2["b"].as() << "\n"; - // Access as double - std::cout << "(4) a: " << j2["a"].as() << ", b: " << j2["b"].as() << "\n\n"; -} - -void serialization_examples() -{ - std::cout << "\nSerialization examples\n\n"; - serialization_example1(); - serialization_example2(); - serialization_example3(); - serialization_example4(); - dump_json_fragments(); - nan_inf_replacement(); - bignum_serialization_examples2(); - bignum_serialization_examples1(); - decimal_precision_examples(); - bignum_access_examples(); - std::cout << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/staj_iterator_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/staj_iterator_examples.cpp deleted file mode 100644 index 23fcd6f995..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/staj_iterator_examples.cpp +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include - -using namespace jsoncons; - -// Example JSON text -const std::string array_example = R"( -[ - { - "employeeNo" : "101", - "name" : "Tommy Cochrane", - "title" : "Supervisor" - }, - { - "employeeNo" : "102", - "name" : "Bill Skeleton", - "title" : "Line manager" - } -] -)"; - -const std::string object_example = R"( -{ - "application": "hiking", - "reputons": [ - { - "rater": "HikingAsylum.array_example.com", - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rating": 0.90 - } - ] -} -)"; - -void staj_array_iterator_example() -{ - std::istringstream is(array_example); - - json_cursor reader(is); - - staj_array_iterator it(reader); - - for (const auto& j : it) - { - std::cout << pretty_print(j) << "\n"; - } - std::cout << "\n\n"; -} - -struct employee -{ - std::string employeeNo; - std::string name; - std::string title; -}; - -namespace jsoncons -{ - template - struct json_type_traits - { - static bool is(const Json& j) noexcept - { - return j.is_object() && j.contains("employeeNo") && j.contains("name") && j.contains("title"); - } - static employee as(const Json& j) - { - employee val; - val.employeeNo = j["employeeNo"].template as(); - val.name = j["name"].template as(); - val.title = j["title"].template as(); - return val; - } - static Json to_json(const employee& val) - { - Json j; - j["employeeNo"] = val.employeeNo; - j["name"] = val.name; - j["title"] = val.title; - return j; - } - }; -} - -void staj_array_iterator_example2() -{ - std::istringstream is(array_example); - - json_cursor reader(is); - - staj_array_iterator it(reader); - - for (const auto& val : it) - { - std::cout << val.employeeNo << ", " << val.name << ", " << val.title << "\n"; - } - std::cout << "\n\n"; -} - -void staj_object_iterator_example() -{ - - std::istringstream is(object_example); - - json_cursor reader(is); - - staj_object_iterator it(reader); - - for (const auto& kv : it) - { - std::cout << kv.first << ":\n" << pretty_print(kv.second) << "\n"; - } - std::cout << "\n\n"; -} - -void staj_iterator_examples() -{ - std::cout << "\nstaj_iterator examples\n\n"; - - staj_array_iterator_example(); - - staj_array_iterator_example2(); - - staj_object_iterator_example(); - - std::cout << "\n"; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/type_extensibility_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/type_extensibility_examples.cpp deleted file mode 100644 index 35fa87eb10..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/type_extensibility_examples.cpp +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright 2016 Daniel Parker -// Distributed under Boost license - -#include "example_types.hpp" -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -void book_extensibility_example() -{ - using jsoncons::json; - - ns::book book1{"Haruki Murakami", "Kafka on the Shore", 25.17}; - - json j = book1; - - std::cout << "(1) " << std::boolalpha << j.is() << "\n\n"; - - std::cout << "(2) " << pretty_print(j) << "\n\n"; - - ns::book temp = j.as(); - std::cout << "(3) " << temp.author << "," - << temp.title << "," - << temp.price << "\n\n"; - - ns::book book2{"Charles Bukowski", "Women: A Novel", 12.0}; - - std::vector book_array{book1, book2}; - - json ja = book_array; - - std::cout << "(4) " << std::boolalpha - << ja.is>() << "\n\n"; - - std::cout << "(5)\n" << pretty_print(ja) << "\n\n"; - - auto book_list = ja.as>(); - - std::cout << "(6)" << std::endl; - for (auto b : book_list) - { - std::cout << b.author << ", " - << b.title << ", " - << b.price << std::endl; - } -} - -void book_extensibility_example2() -{ - const std::string s = R"( - [ - { - "author" : "Haruki Murakami", - "title" : "Kafka on the Shore", - "price" : 25.17 - }, - { - "author" : "Charles Bukowski", - "title" : "Pulp", - "price" : 22.48 - } - ] - )"; - - std::vector book_list = decode_json>(s); - - std::cout << "(1)\n"; - for (const auto& item : book_list) - { - std::cout << item.author << ", " - << item.title << ", " - << item.price << "\n"; - } - - std::cout << "\n(2)\n"; - encode_json(book_list, std::cout, indenting::indent); - std::cout << "\n\n"; -} - -void reputons_extensibility_example() -{ - ns::reputation_object val("hiking", { ns::reputon{"HikingAsylum.example.com","strong-hiker","Marilyn C",0.90} }); - - std::string s; - encode_json(val, s, indenting::indent); - std::cout << s << "\n"; - - auto val2 = decode_json(s); - - assert(val2 == val); -} - -//own vector will always be of an even length -struct own_vector : std::vector { using std::vector::vector; }; - -namespace jsoncons { - -template -struct json_type_traits -{ - static bool is(const Json& j) noexcept - { - return j.is_object() && j.size() % 2 == 0; - } - static own_vector as(const Json& j) - { - own_vector v; - for (auto& item : j.object_range()) - { - std::string s(item.key()); - v.push_back(std::strtol(s.c_str(),nullptr,10)); - v.push_back(item.value().template as()); - } - return v; - } - static Json to_json(const own_vector& val){ - Json j; - for(size_t i=0;i -struct is_json_type_traits_declared : public std::true_type -{}; -} // jsoncons - -void own_vector_extensibility_example() -{ - using jsoncons::json; - - json j = json::object{ {"1",2},{"3",4} }; - assert(j.is()); - auto v = j.as(); - json j2 = v; - - std::cout << j2 << "\n"; -} - -void type_extensibility_examples() -{ - std::cout << std::setprecision(6); - - std::cout << "\nType extensibility examples\n\n"; - - book_extensibility_example(); - - own_vector_extensibility_example(); - - book_extensibility_example2(); - - reputons_extensibility_example(); - - std::cout << std::endl; -} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/type_extensibility_examples2.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/type_extensibility_examples2.cpp deleted file mode 100644 index c77fa236c3..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/type_extensibility_examples2.cpp +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include - -using namespace jsoncons; -using namespace jsoncons::literals; - -class Employee -{ - std::string name_; -public: - Employee(const std::string& name) - : name_(name) - { - } - virtual ~Employee() = default; - const std::string& name() const - { - return name_; - } - virtual double calculatePay() const = 0; -}; - -class HourlyEmployee : public Employee -{ -public: - HourlyEmployee(const std::string& name) - : Employee(name) - { - } - double calculatePay() const override - { - return 10000; - } -}; - -class CommissionedEmployee : public Employee -{ -public: - CommissionedEmployee(const std::string& name) - : Employee(name) - { - } - double calculatePay() const override - { - return 20000; - } -}; - -class EmployeeRegistry -{ - typedef std::unordered_map> employee_map; - employee_map employees_; -public: - EmployeeRegistry() - { - employees_.try_emplace("John Smith",std::make_shared("John Smith")); - employees_.try_emplace("Jane Doe",std::make_shared("Jane Doe")); - } - - bool contains(const std::string& name) const - { - return employees_.count(name) > 0; - } - - std::shared_ptr get(const std::string& name) const - { - auto it = employees_.find(name); - if (it == employees_.end()) - { - throw std::runtime_error("Employee not found"); - } - return it->second; - } -}; - -namespace jsoncons -{ - template - struct json_type_traits> - { - static bool is(const Json& j, const EmployeeRegistry& registry) noexcept - { - return j.is_string() && registry.contains(j.as()); - } - static std::shared_ptr as(const Json& j, - const EmployeeRegistry& registry) - { - return registry.get(j.as()); - } - static Json to_json(std::shared_ptr val) - { - Json j(val->name()); - return j; - } - }; -}; - -void type_extensibility_examples2() -{ - std::cout << "\nType extensibility examples 2\n\n"; - - json j = R"( - { - "EmployeeName" : "John Smith" - } - )"_json; - - EmployeeRegistry registry; - - std::shared_ptr employee = j["EmployeeName"].as>(registry); - - std::cout << "(1) " << employee->name() << " => " - << employee->calculatePay() << std::endl; - - // j does not have a key "SalesRep", so get_with_default returns "Jane Doe" - // The template parameter is explicitly specified as json, to return a json string - // json::as is then applied to the returned json string - std::shared_ptr salesRep = j.get_with_default("SalesRep","Jane Doe") - .as>(registry); - - std::cout << "(2) " << salesRep->name() << " => " - << salesRep->calculatePay() << std::endl; - - json j2; - j2["EmployeeName"] = employee; - j2["SalesRep"] = salesRep; - - std::cout << "(3)\n" << pretty_print(j2) << std::endl; -} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/ubjson_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/ubjson_examples.cpp deleted file mode 100644 index fbbbc5081d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/ubjson_examples.cpp +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under Boost license - -#include "example_types.hpp" -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -void to_from_ubjson_using_basic_json() -{ - ojson j1 = ojson::parse(R"( - { - "application": "hiking", - "reputons": [ - { - "rater": "HikingAsylum.example.com", - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rating": 0.90 - } - ] - } - )"); - - // Encode a basic_json value to UBJSON - std::vector data; - ubjson::encode_ubjson(j1, data); - - // Decode UBJSON to a basic_json value - ojson j2 = ubjson::decode_ubjson(data); - std::cout << "(1)\n" << pretty_print(j2) << "\n\n"; - - // Accessing the data items - - const ojson& reputons = j2["reputons"]; - - std::cout << "(2)\n"; - for (auto element : reputons.array_range()) - { - std::cout << element.at("rated").as() << ", "; - std::cout << element.at("rating").as() << "\n"; - } - std::cout << std::endl; - - // Get a UBJSON value for a nested data item with jsonpointer - std::error_code ec; - auto const& rated = jsonpointer::get(j2, "/reputons/0/rated", ec); - if (!ec) - { - std::cout << "(3) " << rated.as_string() << "\n"; - } - - std::cout << std::endl; -} - -void to_from_ubjson_using_example_type() -{ - ns::reputation_object val("hiking", { ns::reputon{"HikingAsylum.example.com","strong-hiker","Marilyn C",0.90} }); - - // Encode a ns::reputation_object value to UBJSON - std::vector data; - ubjson::encode_ubjson(val, data); - - // Decode UBJSON to a ns::reputation_object value - ns::reputation_object val2 = ubjson::decode_ubjson(data); - - assert(val2 == val); -} - -void ubjson_examples() -{ - std::cout << "\nubjson examples\n\n"; - to_from_ubjson_using_basic_json(); - to_from_ubjson_using_example_type(); - - std::cout << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/unicode_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/unicode_examples.cpp deleted file mode 100644 index f3000ffb25..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/unicode_examples.cpp +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include - -using namespace jsoncons; - -void read_and_write_escaped_unicode() -{ - std::string input = "[\"\\u8A73\\u7D30\\u95B2\\u89A7\\uD800\\uDC01\\u4E00\"]"; - json value = json::parse(input); - json_options options; - options.escape_all_non_ascii(true); - std::string output; - value.dump(output,options); - - std::cout << "Input:" << std::endl; - std::cout << input << std::endl; - std::cout << std::endl; - std::cout << "Output:" << std::endl; - std::cout << output << std::endl; -} - -void unicode_examples() -{ - std::cout << "\nUnicode examples\n\n"; - read_and_write_escaped_unicode(); - std::cout << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/wjson_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/wjson_examples.cpp deleted file mode 100644 index 8c75577cfb..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples/src/wjson_examples.cpp +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#ifdef _MSC_VER -#include -#endif -#include -#include -#include - -using namespace jsoncons; - -void wjson_object() -{ - wjson root; - root[L"field1"] = L"test"; - root[L"field2"] = 3.9; - root[L"field3"] = true; - std::wcout << root << L"\n"; -} - -void wjson_escape_u2() -{ -#ifdef _MSC_VER - std::wstring input = L"[\"\\u007F\\u07FF\\u0800\"]"; - std::wistringstream is(input); - - wjson val = wjson::parse(is); - - std::wstring s = val[0].as(); - std::cout << "length=" << s.length() << std::endl; - std::cout << "Hex dump: ["; - for (size_t i = 0; i < s.size(); ++i) - { - if (i != 0) - std::cout << " "; - uint32_t u(s[i] >= 0 ? s[i] : 256 + s[i] ); - std::cout << "0x" << std::hex<< std::setfill('0') << std::setw(2) << u; - } - std::cout << "]" << std::endl; - - std::wofstream os("./output/xxx.txt"); - os.imbue(std::locale(os.getloc(), new std::codecvt_utf8_utf16)); - - wjson_options options; - options.escape_all_non_ascii(true); - - os << pretty_print(val,options) << L"\n"; -#endif -} - -void wjson_surrogate_pair() -{ -#ifdef _MSC_VER - std::wstring input = L"[\"\\uD950\\uDF21\"]"; - std::wistringstream is(input); - - wjson val = wjson::parse(is); - - std::wstring s = val[0].as(); - std::cout << "length=" << s.length() << std::endl; - std::cout << "Hex dump: ["; - for (size_t i = 0; i < s.size(); ++i) - { - if (i != 0) - std::cout << " "; - uint32_t u(s[i] >= 0 ? s[i] : 256 + s[i] ); - std::cout << "0x" << std::hex<< std::setfill('0') << std::setw(2) << u; - } - std::cout << "]" << std::endl; -#endif -} - -void wjson_examples() -{ - std::cout << "\nwjson examples\n\n"; - wjson_object(); - wjson_escape_u2(); - wjson_surrogate_pair(); - std::cout << std::endl; -} - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples_boost/interprocess_allocator/shared_memory.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples_boost/interprocess_allocator/shared_memory.cpp deleted file mode 100644 index d053d6854d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples_boost/interprocess_allocator/shared_memory.cpp +++ /dev/null @@ -1,107 +0,0 @@ -#include -#include -#include -#include -#include //std::system -#include - -using namespace jsoncons; - -typedef boost::interprocess::allocator shmem_allocator; - -struct boost_sorted_policy : public sorted_policy -{ - template - using sequence_container_type = boost::interprocess::vector; - - template - using key_storage = boost::interprocess::basic_string; - - template - using string_storage = boost::interprocess::basic_string; -}; - -typedef basic_json shm_json; - -int main(int argc, char *argv[]) -{ - typedef std::pair MyType; - - if(argc == 1){ //Parent process - //Remove shared memory on construction and destruction - struct shm_remove - { - shm_remove() { boost::interprocess::shared_memory_object::remove("MySharedMemory"); } - ~shm_remove(){ boost::interprocess::shared_memory_object::remove("MySharedMemory"); } - } remover; - - //Construct managed shared memory - boost::interprocess::managed_shared_memory segment(boost::interprocess::create_only, - "MySharedMemory", 65536); - - //Initialize shared memory STL-compatible allocator - const shmem_allocator allocator(segment.get_segment_manager()); - - // Create json value with all dynamic allocations in shared memory - - shm_json* j = segment.construct("my json")(shm_json::array(allocator)); - j->push_back(10); - - shm_json o(allocator); - //o.try_emplace("category", "reference",allocator); - //o.try_emplace("author", "Nigel Rees",allocator); - //o.try_emplace("title", "Sayings of the Century",allocator); - //o.try_emplace("price", 8.95, allocator); - o.insert_or_assign("category", "reference"); - o.insert_or_assign("author", "Nigel Rees"); - o.insert_or_assign("title", "Sayings of the Century"); - o.insert_or_assign("price", 8.95); - - j->push_back(o); - - shm_json a = shm_json::array(2,shm_json::object(allocator),allocator); - a[0]["first"] = 1; - - j->push_back(a); - - std::pair res; - res = segment.find("my json"); - - std::cout << "Parent:" << std::endl; - std::cout << pretty_print(*(res.first)) << std::endl; - - //Launch child process - std::string s(argv[0]); s += " child "; - if(0 != std::system(s.c_str())) - return 1; - - - //Check child has destroyed all objects - if(segment.find("my json").first) - return 1; - } - else{ - //Open managed shared memory - boost::interprocess::managed_shared_memory segment(boost::interprocess::open_only, - "MySharedMemory"); - - std::pair res; - res = segment.find("my json"); - - if (res.first != nullptr) - { - std::cout << "Child:" << std::endl; - std::cout << pretty_print(*(res.first)) << std::endl; - } - else - { - std::cout << "Result is null" << std::endl; - } - - //We're done, delete all the objects - segment.destroy("my json"); - } - return 0; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples_boost/more_examples/src/extensibility.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples_boost/more_examples/src/extensibility.cpp deleted file mode 100644 index 1275c350bf..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples_boost/more_examples/src/extensibility.cpp +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace jsoncons -{ - template - struct json_type_traits - { - static bool is(const Json& val) noexcept - { - if (!val.is_string()) - { - return false; - } - std::string s = val.template as(); - try - { - boost::gregorian::from_simple_string(s); - return true; - } - catch (...) - { - return false; - } - } - - static boost::gregorian::date as(const Json& val) - { - std::string s = val.template as(); - return boost::gregorian::from_simple_string(s); - } - - static Json to_json(boost::gregorian::date val) - { - return Json(to_iso_extended_string(val)); - } - }; - - template - struct json_type_traits> - { - typedef boost::multiprecision::number multiprecision_type; - - static bool is(const Json& val) noexcept - { - if (!(val.is_string() && val.get_semantic_tag() == semantic_tag::bigdec)) - { - return false; - } - else - { - return true; - } - } - - static multiprecision_type as(const Json& val) - { - return multiprecision_type(val.template as()); - } - - static Json to_json(multiprecision_type val) - { - return Json(val.str(), semantic_tag::bigdec); - } - }; - - template - struct json_type_traits> - { - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& val) noexcept - { - if (!val.is_array()) - { - return false; - } - if (val.size() > 0) - { - size_t n = val[0].size(); - for (const auto& a: val.array_range()) - { - if (!(a.is_array() && a.size() == n)) - { - return false; - } - for (auto x: a.array_range()) - { - if (!x.template is()) - { - return false; - } - } - } - } - return true; - } - - static boost::numeric::ublas::matrix as(const Json& val) - { - if (val.is_array() && val.size() > 0) - { - size_t m = val.size(); - size_t n = 0; - for (const auto& a : val.array_range()) - { - if (a.size() > n) - { - n = a.size(); - } - } - - boost::numeric::ublas::matrix A(m,n,T()); - for (size_t i = 0; i < m; ++i) - { - const auto& a = val[i]; - for (size_t j = 0; j < a.size(); ++j) - { - A(i,j) = a[j].template as(); - } - } - return A; - } - else - { - boost::numeric::ublas::matrix A; - return A; - } - } - - static Json to_json(const boost::numeric::ublas::matrix& val, - allocator_type allocator = allocator_type()) - { - Json a = Json::template make_array<2>(val.size1(), val.size2(), T()); - for (size_t i = 0; i < val.size1(); ++i) - { - for (size_t j = 0; j < val.size1(); ++j) - { - a[i][j] = val(i,j); - } - } - return a; - } - }; -} - -using namespace jsoncons; -using boost::numeric::ublas::matrix; - -void boost_date_conversions() -{ - using boost::gregorian::date; - - json deal; - deal["maturity"] = date(2014,10,14); - - json observation_dates = json::array(); - observation_dates.push_back(date(2014,2,14)); - observation_dates.push_back(date(2014,2,21)); - - deal["observationDates"] = std::move(observation_dates); - - date maturity = deal["maturity"].as(); - - assert(deal["maturity"].as() == date(2014,10,14)); - assert(deal["observationDates"].is_array()); - assert(deal["observationDates"].size() == 2); - assert(deal["observationDates"][0].as() == date(2014,2,14)); - assert(deal["observationDates"][1].as() == date(2014,2,21)); - - std::cout << pretty_print(deal) << std::endl; -} - -void boost_matrix_conversions() -{ - matrix A(2, 2); - A(0, 0) = 1.1; - A(0, 1) = 2.1; - A(1, 0) = 3.1; - A(1, 1) = 4.1; - - json a = A; - - assert(a.is>()); - assert(!a.is>()); - - assert(a[0][0].as()==A(0,0)); - assert(a[0][1].as()==A(0,1)); - assert(a[1][0].as()==A(1,0)); - assert(a[1][1].as()==A(1,1)); - - matrix B = a.as>(); - - assert(B.size1() ==a.size()); - assert(B.size2() ==a[0].size()); - - assert(a[0][0].as()==B(0,0)); - assert(a[0][1].as()==B(0,1)); - assert(a[1][0].as()==B(1,0)); - assert(a[1][1].as()==B(1,1)); -} - - -void boost_multiprecison_conversions() -{ - typedef boost::multiprecision::number multiprecision_type; - - std::string s = "[100000000000000000000000000000000.1234]"; - json_options options; - options.lossless_number(true); - json j = json::parse(s, options); - - multiprecision_type x = j[0].as(); - - std::cout << "(1) " << std::setprecision(std::numeric_limits::max_digits10) - << x << "\n"; - - json j2 = json::array{x}; - std::cout << "(2) " << j2[0].as() << "\n"; -} - -void extensibility_examples() -{ - std::cout << "extensibility examples\n\n"; - - boost_date_conversions(); - boost_matrix_conversions(); - boost_multiprecison_conversions(); -} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples_boost/more_examples/src/more_examples.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples_boost/more_examples/src/more_examples.cpp deleted file mode 100644 index 247a2c9446..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples_boost/more_examples/src/more_examples.cpp +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under Boost license - -#include - -void extensibility_examples(); -void pool_allocator_examples(); - -int main() -{ - try - { - extensibility_examples(); - pool_allocator_examples(); - } - catch (const std::exception& e) - { - std::cout << e.what() << std::endl; - } - - return 0; -} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples_boost/more_examples/src/pool_allocator.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/examples_boost/more_examples/src/pool_allocator.cpp deleted file mode 100644 index 460640b201..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/examples_boost/more_examples/src/pool_allocator.cpp +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -typedef basic_json> my_json; - -void pool_allocator_examples() -{ - std::cout << "pool_allocator examples\n\n"; - - jsoncons::json_decoder> decoder; - - static std::string s("[1,2,3,4,5,6]"); - std::istringstream is(s); - - basic_json_reader,boost::pool_allocator> reader(is,decoder); - reader.read(); - - my_json j = decoder.get_result(); - - std::cout << j << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/basic_json.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/basic_json.hpp deleted file mode 100644 index 76e58e278f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/basic_json.hpp +++ /dev/null @@ -1,4984 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_BASIC_JSON_HPP -#define JSONCONS_BASIC_JSON_HPP - -#include // std::numeric_limits -#include -#include -#include -#include -#include -#include // std::allocator -#include -#include // std::memcpy -#include // std::swap -#include // std::initializer_list -#include // std::move -#include // std::enable_if -#include // std::basic_istream -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace jsoncons { - -struct sorted_policy -{ - static constexpr bool preserve_order = false; - - template - using sequence_container_type = std::vector; - - template - using key_storage = std::basic_string; - - template - using string_storage = std::basic_string; - - typedef default_parse_error_handler parse_error_handler_type; -}; - -struct preserve_order_policy : public sorted_policy -{ - static constexpr bool preserve_order = true; -}; - -template -class range -{ - IteratorT first_; - IteratorT last_; -public: - range(const IteratorT& first, const IteratorT& last) - : first_(first), last_(last) - { - } - -public: - IteratorT begin() - { - return first_; - } - IteratorT end() - { - return last_; - } -}; - -enum class storage_type : uint8_t -{ - null_val = 0x00, - bool_val = 0x01, - int64_val = 0x02, - uint64_val = 0x03, - double_val = 0x04, - short_string_val = 0x05, - long_string_val = 0x06, - byte_string_val = 0x07, - array_val = 0x08, - empty_object_val = 0x09, - object_val = 0x0a, - tag_val = 0x0b -}; - -template -class basic_json -{ -public: - - typedef Allocator allocator_type; - - typedef ImplementationPolicy implementation_policy; - - typedef typename ImplementationPolicy::parse_error_handler_type parse_error_handler_type; - - typedef CharT char_type; - typedef typename std::char_traits char_traits_type; - typedef basic_string_view string_view_type; - - typedef typename std::allocator_traits:: template rebind_alloc char_allocator_type; - - typedef std::basic_string string_type; - - typedef basic_json json_type; - - typedef json_type& reference; - typedef const json_type& const_reference; - typedef json_type* pointer; - typedef const json_type* const_pointer; - - typedef key_value key_value_type; - -#if !defined(JSONCONS_NO_DEPRECATED) - typedef json_type value_type; - typedef string_type key_type; - typedef key_value_type kvp_type; - typedef key_value_type member_type; - typedef jsoncons::null_type null_type; -#endif - - typedef typename std::allocator_traits:: template rebind_alloc byte_allocator_type; - using byte_string_storage_type = typename implementation_policy::template sequence_container_type; - - typedef json_array array; - - typedef typename std::allocator_traits:: template rebind_alloc key_value_allocator_type; - - typedef json_object object; - - typedef typename std::allocator_traits:: template rebind_alloc array_allocator; - typedef typename std::allocator_traits:: template rebind_alloc object_allocator; - - typedef typename object::iterator object_iterator; - typedef typename object::const_iterator const_object_iterator; - typedef typename array::iterator array_iterator; - typedef typename array::const_iterator const_array_iterator; - - struct variant - { - class data_base - { - static const uint8_t major_type_shift = 0x04; - static const uint8_t additional_information_mask = (1U << 4) - 1; - - uint8_t type_; - public: - data_base(uint8_t type) - : type_(type) - {} - - data_base(storage_type data_type, semantic_tag semantic_type) - : type_((static_cast(data_type) << major_type_shift) | static_cast(semantic_type)) - {} - - uint8_t type() const - { - return type_; - } - - storage_type get_storage_type() const - { - - uint8_t value = type_ >> major_type_shift; - return static_cast(value); - } - - semantic_tag get_semantic_tag() const - { - uint8_t value = type_ & additional_information_mask; - return static_cast(value); - } - }; - - class null_data final : public data_base - { - public: - null_data() - : data_base(storage_type::null_val, semantic_tag::none) - { - } - null_data(semantic_tag tag) - : data_base(storage_type::null_val, tag) - { - } - }; - - class empty_object_data final : public data_base - { - public: - empty_object_data(semantic_tag tag) - : data_base(storage_type::empty_object_val, tag) - { - } - }; - - class bool_data final : public data_base - { - bool val_; - public: - bool_data(bool val, semantic_tag tag) - : data_base(storage_type::bool_val, tag),val_(val) - { - } - - bool_data(const bool_data& val) - : data_base(val.type()),val_(val.val_) - { - } - - bool value() const - { - return val_; - } - - }; - - class int64_data final : public data_base - { - int64_t val_; - public: - int64_data(int64_t val, - semantic_tag tag = semantic_tag::none) - : data_base(storage_type::int64_val, tag),val_(val) - { - } - - int64_data(const int64_data& val) - : data_base(val.type()),val_(val.val_) - { - } - - int64_t value() const - { - return val_; - } - }; - - class uint64_data final : public data_base - { - uint64_t val_; - public: - uint64_data(uint64_t val, - semantic_tag tag = semantic_tag::none) - : data_base(storage_type::uint64_val, tag),val_(val) - { - } - - uint64_data(const uint64_data& val) - : data_base(val.type()),val_(val.val_) - { - } - - uint64_t value() const - { - return val_; - } - }; - - class double_data final : public data_base - { - double val_; - public: - double_data(double val, - semantic_tag tag = semantic_tag::none) - : data_base(storage_type::double_val, tag), - val_(val) - { - } - - double_data(const double_data& val) - : data_base(val.type()), - val_(val.val_) - { - } - - double value() const - { - return val_; - } - }; - - class short_string_data final : public data_base - { - static const size_t capacity = 14/sizeof(char_type); - uint8_t length_; - char_type data_[capacity]; - public: - static const size_t max_length = (14 / sizeof(char_type)) - 1; - - short_string_data(semantic_tag tag, const char_type* p, uint8_t length) - : data_base(storage_type::short_string_val, tag), length_(length) - { - JSONCONS_ASSERT(length <= max_length); - std::memcpy(data_,p,length*sizeof(char_type)); - data_[length] = 0; - } - - short_string_data(const short_string_data& val) - : data_base(val.type()), length_(val.length_) - { - std::memcpy(data_,val.data_,val.length_*sizeof(char_type)); - data_[length_] = 0; - } - - uint8_t length() const - { - return length_; - } - - const char_type* data() const - { - return data_; - } - - const char_type* c_str() const - { - return data_; - } - }; - - // long_string_data - class long_string_data final : public data_base - { - typedef typename jsoncons::detail::heap_only_string_factory::string_pointer pointer; - - pointer ptr_; - public: - - long_string_data(semantic_tag tag, const char_type* data, size_t length, const Allocator& a) - : data_base(storage_type::long_string_val, tag) - { - ptr_ = jsoncons::detail::heap_only_string_factory::create(data,length,a); - } - - long_string_data(const long_string_data& val) - : data_base(val.type()) - { - ptr_ = jsoncons::detail::heap_only_string_factory::create(val.data(),val.length(),val.get_allocator()); - } - - long_string_data(long_string_data&& val) - : data_base(val.type()), ptr_(nullptr) - { - std::swap(val.ptr_,ptr_); - } - - long_string_data(const long_string_data& val, const Allocator& a) - : data_base(val.type()) - { - ptr_ = jsoncons::detail::heap_only_string_factory::create(val.data(),val.length(),a); - } - - ~long_string_data() - { - if (ptr_ != nullptr) - { - jsoncons::detail::heap_only_string_factory::destroy(ptr_); - } - } - - void swap(long_string_data& val) - { - std::swap(val.ptr_,ptr_); - } - - const char_type* data() const - { - return ptr_->data(); - } - - const char_type* c_str() const - { - return ptr_->c_str(); - } - - size_t length() const - { - return ptr_->length(); - } - - allocator_type get_allocator() const - { - return ptr_->get_allocator(); - } - }; - - // byte_string_data - class byte_string_data final : public data_base - { - typedef typename std::allocator_traits:: template rebind_alloc string_holder_allocator_type; - typedef typename std::allocator_traits::pointer pointer; - - pointer ptr_; - - template - void create(string_holder_allocator_type allocator, Args&& ... args) - { - typename std::allocator_traits:: template rebind_alloc alloc(allocator); - ptr_ = alloc.allocate(1); - try - { - std::allocator_traits:: template rebind_traits::construct(alloc, jsoncons::detail::to_plain_pointer(ptr_), std::forward(args)...); - } - catch (...) - { - alloc.deallocate(ptr_,1); - throw; - } - } - public: - - byte_string_data(semantic_tag semantic_type, - const uint8_t* data, size_t length, - const Allocator& a) - : data_base(storage_type::byte_string_val, semantic_type) - { - create(string_holder_allocator_type(a), data, data+length, a); - } - - byte_string_data(const byte_string_data& val) - : data_base(val.type()) - { - create(val.ptr_->get_allocator(), *(val.ptr_)); - } - - byte_string_data(byte_string_data&& val) - : data_base(val.type()), ptr_(nullptr) - { - std::swap(val.ptr_,ptr_); - } - - byte_string_data(const byte_string_data& val, const Allocator& a) - : data_base(val.type()) - { - create(string_holder_allocator_type(a), *(val.ptr_), a); - } - - ~byte_string_data() - { - if (ptr_ != nullptr) - { - typename std::allocator_traits:: template rebind_alloc alloc(ptr_->get_allocator()); - std::allocator_traits:: template rebind_traits::destroy(alloc, jsoncons::detail::to_plain_pointer(ptr_)); - alloc.deallocate(ptr_,1); - } - } - - void swap(byte_string_data& val) - { - std::swap(val.ptr_,ptr_); - } - - const uint8_t* data() const - { - return ptr_->data(); - } - - size_t length() const - { - return ptr_->size(); - } - - const uint8_t* begin() const - { - return ptr_->data(); - } - - const uint8_t* end() const - { - return ptr_->data() + ptr_->size(); - } - - allocator_type get_allocator() const - { - return ptr_->get_allocator(); - } - }; - - // array_data - class array_data final : public data_base - { - typedef typename std::allocator_traits::pointer pointer; - pointer ptr_; - - template - void create(array_allocator allocator, Args&& ... args) - { - typename std::allocator_traits:: template rebind_alloc alloc(allocator); - ptr_ = alloc.allocate(1); - try - { - std::allocator_traits:: template rebind_traits::construct(alloc, jsoncons::detail::to_plain_pointer(ptr_), std::forward(args)...); - } - catch (...) - { - alloc.deallocate(ptr_,1); - throw; - } - } - public: - array_data(const array& val, semantic_tag tag) - : data_base(storage_type::array_val, tag) - { - create(val.get_allocator(), val); - } - - array_data(const array& val, semantic_tag tag, const Allocator& a) - : data_base(storage_type::array_val, tag) - { - create(array_allocator(a), val, a); - } - - array_data(const array_data& val) - : data_base(val.type()) - { - create(val.ptr_->get_allocator(), *(val.ptr_)); - } - - array_data(array_data&& val) - : data_base(val.type()), ptr_(nullptr) - { - std::swap(val.ptr_, ptr_); - } - - array_data(const array_data& val, const Allocator& a) - : data_base(val.type()) - { - create(array_allocator(a), *(val.ptr_), a); - } - ~array_data() - { - if (ptr_ != nullptr) - { - typename std::allocator_traits:: template rebind_alloc alloc(ptr_->get_allocator()); - std::allocator_traits:: template rebind_traits::destroy(alloc, jsoncons::detail::to_plain_pointer(ptr_)); - alloc.deallocate(ptr_,1); - } - } - - allocator_type get_allocator() const - { - return ptr_->get_allocator(); - } - - void swap(array_data& val) - { - std::swap(val.ptr_,ptr_); - } - - array& value() - { - return *ptr_; - } - - const array& value() const - { - return *ptr_; - } - }; - - // object_data - class object_data final : public data_base - { - typedef typename std::allocator_traits::pointer pointer; - pointer ptr_; - - template - void create(Allocator allocator, Args&& ... args) - { - typename std::allocator_traits:: template rebind_alloc alloc(allocator); - ptr_ = alloc.allocate(1); - try - { - std::allocator_traits:: template rebind_traits::construct(alloc, jsoncons::detail::to_plain_pointer(ptr_), std::forward(args)...); - } - catch (...) - { - alloc.deallocate(ptr_,1); - throw; - } - } - public: - explicit object_data(const object& val, semantic_tag tag) - : data_base(storage_type::object_val, tag) - { - create(val.get_allocator(), val); - } - - explicit object_data(const object& val, semantic_tag tag, const Allocator& a) - : data_base(storage_type::object_val, tag) - { - create(object_allocator(a), val, a); - } - - explicit object_data(const object_data& val) - : data_base(val.type()) - { - create(val.ptr_->get_allocator(), *(val.ptr_)); - } - - explicit object_data(object_data&& val) - : data_base(val.type()), ptr_(nullptr) - { - std::swap(val.ptr_,ptr_); - } - - explicit object_data(const object_data& val, const Allocator& a) - : data_base(val.type()) - { - create(object_allocator(a), *(val.ptr_), a); - } - - ~object_data() - { - if (ptr_ != nullptr) - { - typename std::allocator_traits:: template rebind_alloc alloc(ptr_->get_allocator()); - std::allocator_traits:: template rebind_traits::destroy(alloc, jsoncons::detail::to_plain_pointer(ptr_)); - alloc.deallocate(ptr_,1); - } - } - - void swap(object_data& val) - { - std::swap(val.ptr_,ptr_); - } - - object& value() - { - return *ptr_; - } - - const object& value() const - { - return *ptr_; - } - - allocator_type get_allocator() const - { - return ptr_->get_allocator(); - } - }; - - private: - static const size_t data_size = static_max::value; - static const size_t data_align = static_max::value; - - typedef typename std::aligned_storage::type data_t; - - data_t data_; - public: - variant(semantic_tag tag) - { - new(reinterpret_cast(&data_))empty_object_data(tag); - } - - explicit variant(null_type, semantic_tag tag) - { - new(reinterpret_cast(&data_))null_data(tag); - } - - explicit variant(bool val, semantic_tag tag) - { - new(reinterpret_cast(&data_))bool_data(val,tag); - } - explicit variant(int64_t val, semantic_tag tag) - { - new(reinterpret_cast(&data_))int64_data(val, tag); - } - explicit variant(uint64_t val, semantic_tag tag) - { - new(reinterpret_cast(&data_))uint64_data(val, tag); - } - - variant(double val, semantic_tag tag) - { - new(reinterpret_cast(&data_))double_data(val, tag); - } - - variant(const char_type* s, size_t length, semantic_tag tag) - { - if (length <= short_string_data::max_length) - { - new(reinterpret_cast(&data_))short_string_data(tag, s, static_cast(length)); - } - else - { - new(reinterpret_cast(&data_))long_string_data(tag, s, length, char_allocator_type()); - } - } - - variant(const char_type* s, size_t length, semantic_tag tag, const Allocator& alloc) - { - if (length <= short_string_data::max_length) - { - new(reinterpret_cast(&data_))short_string_data(tag, s, static_cast(length)); - } - else - { - new(reinterpret_cast(&data_))long_string_data(tag, s, length, char_allocator_type(alloc)); - } - } - - variant(const byte_string_view& bs, semantic_tag tag) - { - new(reinterpret_cast(&data_))byte_string_data(tag, bs.data(), bs.length(), byte_allocator_type()); - } - - variant(const byte_string_view& bs, semantic_tag tag, const Allocator& allocator) - { - new(reinterpret_cast(&data_))byte_string_data(tag, bs.data(), bs.length(), allocator); - } - - variant(const basic_bignum& n) - { - std::basic_string s; - n.dump(s); - - if (s.length() <= short_string_data::max_length) - { - new(reinterpret_cast(&data_))short_string_data(semantic_tag::bigint, s.data(), static_cast(s.length())); - } - else - { - new(reinterpret_cast(&data_))long_string_data(semantic_tag::bigint, s.data(), s.length(), char_allocator_type()); - } - } - - variant(const basic_bignum& n, const Allocator& allocator) - { - std::basic_string s; - n.dump(s); - - if (s.length() <= short_string_data::max_length) - { - new(reinterpret_cast(&data_))short_string_data(semantic_tag::bigint, s.data(), static_cast(s.length())); - } - else - { - new(reinterpret_cast(&data_))long_string_data(semantic_tag::bigint, s.data(), s.length(), char_allocator_type(allocator)); - } - } - variant(const object& val, semantic_tag tag) - { - new(reinterpret_cast(&data_))object_data(val, tag); - } - variant(const object& val, semantic_tag tag, const Allocator& alloc) - { - new(reinterpret_cast(&data_))object_data(val, tag, alloc); - } - variant(const array& val, semantic_tag tag) - { - new(reinterpret_cast(&data_))array_data(val, tag); - } - variant(const array& val, semantic_tag tag, const Allocator& alloc) - { - new(reinterpret_cast(&data_))array_data(val, tag, alloc); - } - - variant(const variant& val) - { - Init_(val); - } - - variant(const variant& val, const Allocator& allocator) - { - Init_(val,allocator); - } - - variant(variant&& val) noexcept - { - Init_rv_(std::forward(val)); - } - - variant(variant&& val, const Allocator& allocator) noexcept - { - Init_rv_(std::forward(val), allocator, - typename std::allocator_traits::propagate_on_container_move_assignment()); - } - - ~variant() - { - Destroy_(); - } - - void Destroy_() - { - switch (get_storage_type()) - { - case storage_type::long_string_val: - reinterpret_cast(&data_)->~long_string_data(); - break; - case storage_type::byte_string_val: - reinterpret_cast(&data_)->~byte_string_data(); - break; - case storage_type::array_val: - reinterpret_cast(&data_)->~array_data(); - break; - case storage_type::object_val: - reinterpret_cast(&data_)->~object_data(); - break; - default: - break; - } - } - - variant& operator=(const variant& val) - { - if (this !=&val) - { - Destroy_(); - switch (val.get_storage_type()) - { - case storage_type::null_val: - new(reinterpret_cast(&data_))null_data(*(val.null_data_cast())); - break; - case storage_type::empty_object_val: - new(reinterpret_cast(&data_))empty_object_data(*(val.empty_object_data_cast())); - break; - case storage_type::bool_val: - new(reinterpret_cast(&data_))bool_data(*(val.bool_data_cast())); - break; - case storage_type::int64_val: - new(reinterpret_cast(&data_))int64_data(*(val.int64_data_cast())); - break; - case storage_type::uint64_val: - new(reinterpret_cast(&data_))uint64_data(*(val.uint64_data_cast())); - break; - case storage_type::double_val: - new(reinterpret_cast(&data_))double_data(*(val.double_data_cast())); - break; - case storage_type::short_string_val: - new(reinterpret_cast(&data_))short_string_data(*(val.short_string_data_cast())); - break; - case storage_type::long_string_val: - new(reinterpret_cast(&data_))long_string_data(*(val.string_data_cast())); - break; - case storage_type::byte_string_val: - new(reinterpret_cast(&data_))byte_string_data(*(val.byte_string_data_cast())); - break; - case storage_type::array_val: - new(reinterpret_cast(&data_))array_data(*(val.array_data_cast())); - break; - case storage_type::object_val: - new(reinterpret_cast(&data_))object_data(*(val.object_data_cast())); - break; - default: - JSONCONS_UNREACHABLE(); - break; - } - } - return *this; - } - - variant& operator=(variant&& val) noexcept - { - if (this !=&val) - { - swap(val); - } - return *this; - } - - storage_type get_storage_type() const - { - return reinterpret_cast(&data_)->get_storage_type(); - } - - semantic_tag get_semantic_tag() const - { - return reinterpret_cast(&data_)->get_semantic_tag(); - } - - const null_data* null_data_cast() const - { - return reinterpret_cast(&data_); - } - - const empty_object_data* empty_object_data_cast() const - { - return reinterpret_cast(&data_); - } - - const bool_data* bool_data_cast() const - { - return reinterpret_cast(&data_); - } - - const int64_data* int64_data_cast() const - { - return reinterpret_cast(&data_); - } - - const uint64_data* uint64_data_cast() const - { - return reinterpret_cast(&data_); - } - - const double_data* double_data_cast() const - { - return reinterpret_cast(&data_); - } - - const short_string_data* short_string_data_cast() const - { - return reinterpret_cast(&data_); - } - - long_string_data* string_data_cast() - { - return reinterpret_cast(&data_); - } - - const long_string_data* string_data_cast() const - { - return reinterpret_cast(&data_); - } - - byte_string_data* byte_string_data_cast() - { - return reinterpret_cast(&data_); - } - - const byte_string_data* byte_string_data_cast() const - { - return reinterpret_cast(&data_); - } - - object_data* object_data_cast() - { - return reinterpret_cast(&data_); - } - - const object_data* object_data_cast() const - { - return reinterpret_cast(&data_); - } - - array_data* array_data_cast() - { - return reinterpret_cast(&data_); - } - - const array_data* array_data_cast() const - { - return reinterpret_cast(&data_); - } - - size_t size() const - { - switch (get_storage_type()) - { - case storage_type::array_val: - return array_data_cast()->value().size(); - case storage_type::object_val: - return object_data_cast()->value().size(); - default: - return 0; - } - } - - string_view_type as_string_view() const - { - switch (get_storage_type()) - { - case storage_type::short_string_val: - return string_view_type(short_string_data_cast()->data(),short_string_data_cast()->length()); - case storage_type::long_string_val: - return string_view_type(string_data_cast()->data(),string_data_cast()->length()); - default: - JSONCONS_THROW(json_runtime_error("Not a string")); - } - } - - template > - basic_byte_string as_byte_string() const - { - switch (get_storage_type()) - { - case storage_type::short_string_val: - case storage_type::long_string_val: - { - switch (get_semantic_tag()) - { - case semantic_tag::base16: - { - basic_byte_string bs; - auto s = as_string_view(); - decode_base16(s.begin(), s.end(), bs); - return bs; - } - case semantic_tag::base64: - { - basic_byte_string bs; - auto s = as_string_view(); - decode_base64(s.begin(), s.end(), bs); - return bs; - } - case semantic_tag::base64url: - { - basic_byte_string bs; - auto s = as_string_view(); - decode_base64url(s.begin(), s.end(), bs); - return bs; - } - default: - JSONCONS_THROW(json_runtime_error("Not a byte string")); - } - break; - } - case storage_type::byte_string_val: - return basic_byte_string(byte_string_data_cast()->data(),byte_string_data_cast()->length()); - default: - JSONCONS_THROW(json_runtime_error("Not a byte string")); - } - } - - byte_string_view as_byte_string_view() const - { - switch (get_storage_type()) - { - case storage_type::byte_string_val: - return byte_string_view(byte_string_data_cast()->data(),byte_string_data_cast()->length()); - default: - JSONCONS_THROW(json_runtime_error("Not a byte string")); - } - } - - template > - basic_bignum as_bignum() const - { - switch (get_storage_type()) - { - case storage_type::short_string_val: - case storage_type::long_string_val: - if (!jsoncons::detail::is_integer(as_string_view().data(), as_string_view().length())) - { - JSONCONS_THROW(json_runtime_error("Not an integer")); - } - return basic_bignum(as_string_view().data(), as_string_view().length()); - case storage_type::double_val: - return basic_bignum(double_data_cast()->value()); - case storage_type::int64_val: - return basic_bignum(int64_data_cast()->value()); - case storage_type::uint64_val: - return basic_bignum(uint64_data_cast()->value()); - case storage_type::bool_val: - return basic_bignum(bool_data_cast()->value() ? 1 : 0); - default: - JSONCONS_THROW(json_runtime_error("Not a bignum")); - } - } - - bool operator==(const variant& rhs) const - { - if (this ==&rhs) - { - return true; - } - switch (get_storage_type()) - { - case storage_type::null_val: - switch (rhs.get_storage_type()) - { - case storage_type::null_val: - return true; - default: - return false; - } - break; - case storage_type::empty_object_val: - switch (rhs.get_storage_type()) - { - case storage_type::empty_object_val: - return true; - case storage_type::object_val: - return rhs.size() == 0; - default: - return false; - } - break; - case storage_type::bool_val: - switch (rhs.get_storage_type()) - { - case storage_type::bool_val: - return bool_data_cast()->value() == rhs.bool_data_cast()->value(); - default: - return false; - } - break; - case storage_type::int64_val: - switch (rhs.get_storage_type()) - { - case storage_type::int64_val: - return int64_data_cast()->value() == rhs.int64_data_cast()->value(); - case storage_type::uint64_val: - return int64_data_cast()->value() >= 0 ? static_cast(int64_data_cast()->value()) == rhs.uint64_data_cast()->value() : false; - case storage_type::double_val: - return static_cast(int64_data_cast()->value()) == rhs.double_data_cast()->value(); - default: - return false; - } - break; - case storage_type::uint64_val: - switch (rhs.get_storage_type()) - { - case storage_type::int64_val: - return rhs.int64_data_cast()->value() >= 0 ? uint64_data_cast()->value() == static_cast(rhs.int64_data_cast()->value()) : false; - case storage_type::uint64_val: - return uint64_data_cast()->value() == rhs.uint64_data_cast()->value(); - case storage_type::double_val: - return static_cast(uint64_data_cast()->value()) == rhs.double_data_cast()->value(); - default: - return false; - } - break; - case storage_type::double_val: - switch (rhs.get_storage_type()) - { - case storage_type::int64_val: - return double_data_cast()->value() == static_cast(rhs.int64_data_cast()->value()); - case storage_type::uint64_val: - return double_data_cast()->value() == static_cast(rhs.uint64_data_cast()->value()); - case storage_type::double_val: - return double_data_cast()->value() == rhs.double_data_cast()->value(); - default: - return false; - } - break; - case storage_type::short_string_val: - switch (rhs.get_storage_type()) - { - case storage_type::short_string_val: - return as_string_view() == rhs.as_string_view(); - case storage_type::long_string_val: - return as_string_view() == rhs.as_string_view(); - default: - return false; - } - break; - case storage_type::long_string_val: - switch (rhs.get_storage_type()) - { - case storage_type::short_string_val: - return as_string_view() == rhs.as_string_view(); - case storage_type::long_string_val: - return as_string_view() == rhs.as_string_view(); - default: - return false; - } - break; - case storage_type::byte_string_val: - switch (rhs.get_storage_type()) - { - case storage_type::byte_string_val: - { - return as_byte_string_view() == rhs.as_byte_string_view(); - } - default: - return false; - } - break; - case storage_type::array_val: - switch (rhs.get_storage_type()) - { - case storage_type::array_val: - return array_data_cast()->value() == rhs.array_data_cast()->value(); - default: - return false; - } - break; - case storage_type::object_val: - switch (rhs.get_storage_type()) - { - case storage_type::empty_object_val: - return size() == 0; - case storage_type::object_val: - return object_data_cast()->value() == rhs.object_data_cast()->value(); - default: - return false; - } - break; - default: - JSONCONS_UNREACHABLE(); - break; - } - } - - bool operator!=(const variant& rhs) const - { - return !(*this == rhs); - } - - bool operator<(const variant& rhs) const - { - if (this == &rhs) - { - return false; - } - switch (get_storage_type()) - { - case storage_type::null_val: - return (int)get_storage_type() < (int)rhs.get_storage_type(); - case storage_type::empty_object_val: - switch (rhs.get_storage_type()) - { - case storage_type::empty_object_val: - return false; - case storage_type::object_val: - return rhs.size() != 0; - default: - return (int)get_storage_type() < (int)rhs.get_storage_type(); - } - break; - case storage_type::bool_val: - switch (rhs.get_storage_type()) - { - case storage_type::bool_val: - return bool_data_cast()->value() < rhs.bool_data_cast()->value(); - default: - return (int)get_storage_type() < (int)rhs.get_storage_type(); - } - break; - case storage_type::int64_val: - switch (rhs.get_storage_type()) - { - case storage_type::int64_val: - return int64_data_cast()->value() < rhs.int64_data_cast()->value(); - case storage_type::uint64_val: - return int64_data_cast()->value() >= 0 ? static_cast(int64_data_cast()->value()) < rhs.uint64_data_cast()->value() : true; - case storage_type::double_val: - return static_cast(int64_data_cast()->value()) < rhs.double_data_cast()->value(); - default: - return (int)get_storage_type() < (int)rhs.get_storage_type(); - } - break; - case storage_type::uint64_val: - switch (rhs.get_storage_type()) - { - case storage_type::int64_val: - return rhs.int64_data_cast()->value() >= 0 ? uint64_data_cast()->value() < static_cast(rhs.int64_data_cast()->value()) : true; - case storage_type::uint64_val: - return uint64_data_cast()->value() < rhs.uint64_data_cast()->value(); - case storage_type::double_val: - return static_cast(uint64_data_cast()->value()) < rhs.double_data_cast()->value(); - default: - return (int)get_storage_type() < (int)rhs.get_storage_type(); - } - break; - case storage_type::double_val: - switch (rhs.get_storage_type()) - { - case storage_type::int64_val: - return double_data_cast()->value() < static_cast(rhs.int64_data_cast()->value()); - case storage_type::uint64_val: - return double_data_cast()->value() < static_cast(rhs.uint64_data_cast()->value()); - case storage_type::double_val: - return double_data_cast()->value() < rhs.double_data_cast()->value(); - default: - return (int)get_storage_type() < (int)rhs.get_storage_type(); - } - break; - case storage_type::short_string_val: - switch (rhs.get_storage_type()) - { - case storage_type::short_string_val: - return as_string_view() < rhs.as_string_view(); - case storage_type::long_string_val: - return as_string_view() < rhs.as_string_view(); - default: - return (int)get_storage_type() < (int)rhs.get_storage_type(); - } - break; - case storage_type::long_string_val: - switch (rhs.get_storage_type()) - { - case storage_type::short_string_val: - return as_string_view() < rhs.as_string_view(); - case storage_type::long_string_val: - return as_string_view() < rhs.as_string_view(); - default: - return (int)get_storage_type() < (int)rhs.get_storage_type(); - } - break; - case storage_type::byte_string_val: - switch (rhs.get_storage_type()) - { - case storage_type::byte_string_val: - { - return as_byte_string_view() < rhs.as_byte_string_view(); - } - default: - return (int)get_storage_type() < (int)rhs.get_storage_type(); - } - break; - case storage_type::array_val: - switch (rhs.get_storage_type()) - { - case storage_type::array_val: - return array_data_cast()->value() < rhs.array_data_cast()->value(); - default: - return (int)get_storage_type() < (int)rhs.get_storage_type(); - } - break; - case storage_type::object_val: - switch (rhs.get_storage_type()) - { - case storage_type::empty_object_val: - return false; - case storage_type::object_val: - return object_data_cast()->value() < rhs.object_data_cast()->value(); - default: - return (int)get_storage_type() < (int)rhs.get_storage_type(); - } - break; - default: - JSONCONS_UNREACHABLE(); - break; - } - } - - template - typename std::enable_if::pointer>::value,void>::type - swap(variant& other) noexcept - { - if (this ==&other) - { - return; - } - - std::swap(data_,other.data_); - } - - template - typename std::enable_if::pointer>::value, void>::type - swap(variant& other) noexcept - { - if (this ==&other) - { - return; - } - - variant temp(other); - switch (get_storage_type()) - { - case storage_type::null_val: - new(reinterpret_cast(&(other.data_)))null_data(*null_data_cast()); - break; - case storage_type::empty_object_val: - new(reinterpret_cast(&(other.data_)))empty_object_data(*empty_object_data_cast()); - break; - case storage_type::bool_val: - new(reinterpret_cast(&(other.data_)))bool_data(*bool_data_cast()); - break; - case storage_type::int64_val: - new(reinterpret_cast(&(other.data_)))int64_data(*int64_data_cast()); - break; - case storage_type::uint64_val: - new(reinterpret_cast(&(other.data_)))uint64_data(*uint64_data_cast()); - break; - case storage_type::double_val: - new(reinterpret_cast(&(other.data_)))double_data(*double_data_cast()); - break; - case storage_type::short_string_val: - new(reinterpret_cast(&(other.data_)))short_string_data(*short_string_data_cast()); - break; - case storage_type::long_string_val: - new(reinterpret_cast(&other.data_))long_string_data(std::move(*string_data_cast())); - break; - case storage_type::byte_string_val: - new(reinterpret_cast(&other.data_))byte_string_data(std::move(*byte_string_data_cast())); - break; - case storage_type::array_val: - new(reinterpret_cast(&(other.data_)))array_data(std::move(*array_data_cast())); - break; - case storage_type::object_val: - new(reinterpret_cast(&(other.data_)))object_data(std::move(*object_data_cast())); - break; - default: - JSONCONS_UNREACHABLE(); - break; - } - switch (temp.get_storage_type()) - { - case storage_type::long_string_val: - new(reinterpret_cast(&data_))long_string_data(std::move(*temp.string_data_cast())); - break; - case storage_type::byte_string_val: - new(reinterpret_cast(&data_))byte_string_data(std::move(*temp.byte_string_data_cast())); - break; - case storage_type::array_val: - new(reinterpret_cast(&(data_)))array_data(std::move(*temp.array_data_cast())); - break; - case storage_type::object_val: - new(reinterpret_cast(&(data_)))object_data(std::move(*temp.object_data_cast())); - break; - default: - std::swap(data_,temp.data_); - break; - } - } - private: - - void Init_(const variant& val) - { - switch (val.get_storage_type()) - { - case storage_type::null_val: - new(reinterpret_cast(&data_))null_data(*(val.null_data_cast())); - break; - case storage_type::empty_object_val: - new(reinterpret_cast(&data_))empty_object_data(*(val.empty_object_data_cast())); - break; - case storage_type::bool_val: - new(reinterpret_cast(&data_))bool_data(*(val.bool_data_cast())); - break; - case storage_type::int64_val: - new(reinterpret_cast(&data_))int64_data(*(val.int64_data_cast())); - break; - case storage_type::uint64_val: - new(reinterpret_cast(&data_))uint64_data(*(val.uint64_data_cast())); - break; - case storage_type::double_val: - new(reinterpret_cast(&data_))double_data(*(val.double_data_cast())); - break; - case storage_type::short_string_val: - new(reinterpret_cast(&data_))short_string_data(*(val.short_string_data_cast())); - break; - case storage_type::long_string_val: - new(reinterpret_cast(&data_))long_string_data(*(val.string_data_cast())); - break; - case storage_type::byte_string_val: - new(reinterpret_cast(&data_))byte_string_data(*(val.byte_string_data_cast())); - break; - case storage_type::object_val: - new(reinterpret_cast(&data_))object_data(*(val.object_data_cast())); - break; - case storage_type::array_val: - new(reinterpret_cast(&data_))array_data(*(val.array_data_cast())); - break; - default: - break; - } - } - - void Init_(const variant& val, const Allocator& a) - { - switch (val.get_storage_type()) - { - case storage_type::null_val: - case storage_type::empty_object_val: - case storage_type::bool_val: - case storage_type::int64_val: - case storage_type::uint64_val: - case storage_type::double_val: - case storage_type::short_string_val: - Init_(val); - break; - case storage_type::long_string_val: - new(reinterpret_cast(&data_))long_string_data(*(val.string_data_cast()),a); - break; - case storage_type::byte_string_val: - new(reinterpret_cast(&data_))byte_string_data(*(val.byte_string_data_cast()),a); - break; - case storage_type::array_val: - new(reinterpret_cast(&data_))array_data(*(val.array_data_cast()),a); - break; - case storage_type::object_val: - new(reinterpret_cast(&data_))object_data(*(val.object_data_cast()),a); - break; - default: - break; - } - } - - void Init_rv_(variant&& val) noexcept - { - switch (val.get_storage_type()) - { - case storage_type::null_val: - case storage_type::empty_object_val: - case storage_type::double_val: - case storage_type::int64_val: - case storage_type::uint64_val: - case storage_type::bool_val: - case storage_type::short_string_val: - Init_(val); - break; - case storage_type::long_string_val: - { - new(reinterpret_cast(&data_))long_string_data(std::move(*val.string_data_cast())); - new(reinterpret_cast(&val.data_))null_data(); - } - break; - case storage_type::byte_string_val: - { - new(reinterpret_cast(&data_))byte_string_data(std::move(*val.byte_string_data_cast())); - new(reinterpret_cast(&val.data_))null_data(); - } - break; - case storage_type::array_val: - { - new(reinterpret_cast(&data_))array_data(std::move(*val.array_data_cast())); - new(reinterpret_cast(&val.data_))null_data(); - } - break; - case storage_type::object_val: - { - new(reinterpret_cast(&data_))object_data(std::move(*val.object_data_cast())); - new(reinterpret_cast(&val.data_))null_data(); - } - break; - default: - JSONCONS_UNREACHABLE(); - break; - } - } - - void Init_rv_(variant&& val, const Allocator&, std::true_type) noexcept - { - Init_rv_(std::forward(val)); - } - - void Init_rv_(variant&& val, const Allocator& a, std::false_type) noexcept - { - switch (val.get_storage_type()) - { - case storage_type::null_val: - case storage_type::empty_object_val: - case storage_type::double_val: - case storage_type::int64_val: - case storage_type::uint64_val: - case storage_type::bool_val: - case storage_type::short_string_val: - Init_(std::forward(val)); - break; - case storage_type::long_string_val: - { - if (a == val.string_data_cast()->get_allocator()) - { - Init_rv_(std::forward(val), a, std::true_type()); - } - else - { - Init_(val,a); - } - } - break; - case storage_type::byte_string_val: - { - if (a == val.byte_string_data_cast()->get_allocator()) - { - Init_rv_(std::forward(val), a, std::true_type()); - } - else - { - Init_(val,a); - } - } - break; - case storage_type::object_val: - { - if (a == val.object_data_cast()->get_allocator()) - { - Init_rv_(std::forward(val), a, std::true_type()); - } - else - { - Init_(val,a); - } - } - break; - case storage_type::array_val: - { - if (a == val.array_data_cast()->get_allocator()) - { - Init_rv_(std::forward(val), a, std::true_type()); - } - else - { - Init_(val,a); - } - } - break; - default: - break; - } - } - }; - - template - class json_proxy - { - private: - - ParentT& parent_; - const char_type* data_; - size_t length_; - - json_proxy() = delete; - json_proxy& operator = (const json_proxy& other) = delete; - - json_proxy(ParentT& parent, const char_type* data, size_t length) - : parent_(parent), data_(data), length_(length) - { - } - - basic_json& evaluate() - { - return parent_.evaluate(string_view_type(data_,length_)); - } - - const basic_json& evaluate() const - { - return parent_.evaluate(string_view_type(data_,length_)); - } - - basic_json& evaluate_with_default() - { - basic_json& val = parent_.evaluate_with_default(); - auto it = val.find(string_view_type(data_,length_)); - if (it == val.object_range().end()) - { - it = val.insert_or_assign(val.object_range().begin(),string_view_type(data_,length_),object(val.object_value().get_allocator())); - } - return it->value(); - } - - basic_json& evaluate(size_t index) - { - return evaluate().at(index); - } - - const basic_json& evaluate(size_t index) const - { - return evaluate().at(index); - } - - basic_json& evaluate(const string_view_type& index) - { - return evaluate().at(index); - } - - const basic_json& evaluate(const string_view_type& index) const - { - return evaluate().at(index); - } - public: - - friend class basic_json; - typedef json_proxy proxy_type; - - range object_range() - { - return evaluate().object_range(); - } - - range object_range() const - { - return evaluate().object_range(); - } - - range array_range() - { - return evaluate().array_range(); - } - - range array_range() const - { - return evaluate().array_range(); - } - - size_t size() const noexcept - { - return evaluate().size(); - } - - storage_type get_storage_type() const - { - return evaluate().get_storage_type(); - } - - semantic_tag get_semantic_tag() const - { - return evaluate().get_semantic_tag(); - } - - size_t count(const string_view_type& name) const - { - return evaluate().count(name); - } - - allocator_type get_allocator() const - { - return evaluate().get_allocator(); - } - - bool contains(const string_view_type& name) const - { - return evaluate().contains(name); - } - - bool is_null() const noexcept - { - return evaluate().is_null(); - } - - bool empty() const - { - return evaluate().empty(); - } - - size_t capacity() const - { - return evaluate().capacity(); - } - - void reserve(size_t n) - { - evaluate().reserve(n); - } - - void resize(size_t n) - { - evaluate().resize(n); - } - - template - void resize(size_t n, T val) - { - evaluate().resize(n,val); - } - - template - bool is(Args&&... args) const - { - return evaluate().template is(std::forward(args)...); - } - - bool is_string() const noexcept - { - return evaluate().is_string(); - } - - bool is_string_view() const noexcept - { - return evaluate().is_string_view(); - } - - bool is_byte_string() const noexcept - { - return evaluate().is_byte_string(); - } - - bool is_byte_string_view() const noexcept - { - return evaluate().is_byte_string_view(); - } - - bool is_bignum() const noexcept - { - return evaluate().is_bignum(); - } - - bool is_number() const noexcept - { - return evaluate().is_number(); - } - bool is_bool() const noexcept - { - return evaluate().is_bool(); - } - - bool is_object() const noexcept - { - return evaluate().is_object(); - } - - bool is_array() const noexcept - { - return evaluate().is_array(); - } - - bool is_int64() const noexcept - { - return evaluate().is_int64(); - } - - bool is_uint64() const noexcept - { - return evaluate().is_uint64(); - } - - bool is_double() const noexcept - { - return evaluate().is_double(); - } - - string_view_type as_string_view() const - { - return evaluate().as_string_view(); - } - - byte_string_view as_byte_string_view() const - { - return evaluate().as_byte_string_view(); - } - - basic_bignum as_bignum() const - { - return evaluate().as_bignum(); - } - - template > - string_type as_string() const - { - return evaluate().as_string(); - } - - template > - string_type as_string(const SAllocator& allocator) const - { - return evaluate().as_string(allocator); - } - - template > - basic_byte_string as_byte_string() const - { - return evaluate().template as_byte_string(); - } - - template > - string_type as_string(const basic_json_options& options) const - { - return evaluate().as_string(options); - } - - template > - string_type as_string(const basic_json_options& options, - const SAllocator& allocator) const - { - return evaluate().as_string(options,allocator); - } - - template - T as(Args&&... args) const - { - return evaluate().template as(std::forward(args)...); - } - - template - typename std::enable_if::value,T>::type - as(const char_allocator_type& allocator) const - { - return evaluate().template as(allocator); - } - bool as_bool() const - { - return evaluate().as_bool(); - } - - double as_double() const - { - return evaluate().as_double(); - } - - template - T as_integer() const - { - return evaluate().template as_integer(); - } - - template - json_proxy& operator=(T&& val) - { - parent_.evaluate_with_default().insert_or_assign(string_view_type(data_,length_), std::forward(val)); - return *this; - } - - bool operator==(const basic_json& rhs) const - { - return evaluate() == rhs; - } - - bool operator!=(const basic_json& rhs) const - { - return evaluate() != rhs; - } - - bool operator<(const basic_json& rhs) const - { - return evaluate() < rhs; - } - - bool operator<=(const basic_json& rhs) const - { - return !(rhs < evaluate()); - } - - bool operator>(const basic_json& rhs) const - { - return !(evaluate() <= rhs); - } - - bool operator>=(const basic_json& rhs) const - { - return !(evaluate() < rhs); - } - - basic_json& operator[](size_t i) - { - return evaluate_with_default().at(i); - } - - const basic_json& operator[](size_t i) const - { - return evaluate().at(i); - } - - json_proxy operator[](const string_view_type& key) - { - return json_proxy(*this,key.data(),key.length()); - } - - const basic_json& operator[](const string_view_type& name) const - { - return at(name); - } - - basic_json& at(const string_view_type& name) - { - return evaluate().at(name); - } - - const basic_json& at(const string_view_type& name) const - { - return evaluate().at(name); - } - - const basic_json& at(size_t index) - { - return evaluate().at(index); - } - - const basic_json& at(size_t index) const - { - return evaluate().at(index); - } - - object_iterator find(const string_view_type& name) - { - return evaluate().find(name); - } - - const_object_iterator find(const string_view_type& name) const - { - return evaluate().find(name); - } - - template - T get_with_default(const string_view_type& name, const T& default_val) const - { - return evaluate().template get_with_default(name,default_val); - } - - template > - T get_with_default(const string_view_type& name, const CharT* default_val) const - { - return evaluate().template get_with_default(name,default_val); - } - - void shrink_to_fit() - { - evaluate_with_default().shrink_to_fit(); - } - - void clear() - { - evaluate().clear(); - } - // Remove all elements from an array or object - - void erase(const_object_iterator pos) - { - evaluate().erase(pos); - } - // Remove a range of elements from an object - - void erase(const_object_iterator first, const_object_iterator last) - { - evaluate().erase(first, last); - } - // Remove a range of elements from an object - - void erase(const string_view_type& name) - { - evaluate().erase(name); - } - - void erase(const_array_iterator pos) - { - evaluate().erase(pos); - } - // Removes the element at pos - - void erase(const_array_iterator first, const_array_iterator last) - { - evaluate().erase(first, last); - } - // Remove a range of elements from an array - - // merge - - void merge(const basic_json& source) - { - return evaluate().merge(source); - } - - void merge(basic_json&& source) - { - return evaluate().merge(std::forward(source)); - } - - void merge(object_iterator hint, const basic_json& source) - { - return evaluate().merge(hint, source); - } - - void merge(object_iterator hint, basic_json&& source) - { - return evaluate().merge(hint, std::forward(source)); - } - - // merge_or_update - - void merge_or_update(const basic_json& source) - { - return evaluate().merge_or_update(source); - } - - void merge_or_update(basic_json&& source) - { - return evaluate().merge_or_update(std::forward(source)); - } - - void merge_or_update(object_iterator hint, const basic_json& source) - { - return evaluate().merge_or_update(hint, source); - } - - void merge_or_update(object_iterator hint, basic_json&& source) - { - return evaluate().merge_or_update(hint, std::forward(source)); - } - - template - std::pair insert_or_assign(const string_view_type& name, T&& val) - { - return evaluate().insert_or_assign(name,std::forward(val)); - } - - // emplace - - template - std::pair try_emplace(const string_view_type& name, Args&&... args) - { - return evaluate().try_emplace(name,std::forward(args)...); - } - - template - object_iterator insert_or_assign(object_iterator hint, const string_view_type& name, T&& val) - { - return evaluate().insert_or_assign(hint, name, std::forward(val)); - } - - template - object_iterator try_emplace(object_iterator hint, const string_view_type& name, Args&&... args) - { - return evaluate().try_emplace(hint, name, std::forward(args)...); - } - - template - array_iterator emplace(const_array_iterator pos, Args&&... args) - { - evaluate_with_default().emplace(pos, std::forward(args)...); - } - - template - basic_json& emplace_back(Args&&... args) - { - return evaluate_with_default().emplace_back(std::forward(args)...); - } - - template - void push_back(T&& val) - { - evaluate_with_default().push_back(std::forward(val)); - } - - template - array_iterator insert(const_array_iterator pos, T&& val) - { - return evaluate_with_default().insert(pos, std::forward(val)); - } - - template - array_iterator insert(const_array_iterator pos, InputIt first, InputIt last) - { - return evaluate_with_default().insert(pos, first, last); - } - - template - void insert(InputIt first, InputIt last) - { - evaluate_with_default().insert(first, last); - } - - template - void insert(sorted_unique_range_tag tag, InputIt first, InputIt last) - { - evaluate_with_default().insert(tag, first, last); - } - - template - void dump(std::basic_string& s) const - { - evaluate().dump(s); - } - - template - void dump(std::basic_string& s, - indenting line_indent) const - { - evaluate().dump(s, line_indent); - } - - template - void dump(std::basic_string& s, - const basic_json_options& options) const - { - evaluate().dump(s,options); - } - - template - void dump(std::basic_string& s, - const basic_json_options& options, - indenting line_indent) const - { - evaluate().dump(s,options,line_indent); - } - - void dump(basic_json_content_handler& handler) const - { - evaluate().dump(handler); - } - - void dump(std::basic_ostream& os) const - { - evaluate().dump(os); - } - - void dump(std::basic_ostream& os, indenting line_indent) const - { - evaluate().dump(os, line_indent); - } - - void dump(std::basic_ostream& os, const basic_json_options& options) const - { - evaluate().dump(os,options); - } - - void dump(std::basic_ostream& os, const basic_json_options& options, indenting line_indent) const - { - evaluate().dump(os,options,line_indent); - } -#if !defined(JSONCONS_NO_DEPRECATED) - - bool is_datetime() const noexcept - { - return evaluate().is_datetime(); - } - - bool is_epoch_time() const noexcept - { - return evaluate().is_epoch_time(); - } - - template - void add(T&& val) - { - evaluate_with_default().add(std::forward(val)); - } - - template - array_iterator add(const_array_iterator pos, T&& val) - { - return evaluate_with_default().add(pos, std::forward(val)); - } - - // set - - template - std::pair set(const string_view_type& name, T&& val) - { - return evaluate().set(name,std::forward(val)); - } - - template - object_iterator set(object_iterator hint, const string_view_type& name, T&& val) - { - return evaluate().set(hint, name, std::forward(val)); - } - - bool has_key(const string_view_type& name) const - { - return evaluate().has_key(name); - } - - bool is_integer() const noexcept - { - return evaluate().is_int64(); - } - - bool is_uinteger() const noexcept - { - return evaluate().is_uint64(); - } - - unsigned long long as_ulonglong() const - { - return evaluate().as_ulonglong(); - } - - uint64_t as_uinteger() const - { - return evaluate().as_uinteger(); - } - - void dump(std::basic_ostream& os, const basic_json_options& options, bool pprint) const - { - evaluate().dump(os,options,pprint); - } - - void dump(std::basic_ostream& os, bool pprint) const - { - evaluate().dump(os, pprint); - } - - string_type to_string(const char_allocator_type& allocator = char_allocator_type()) const noexcept - { - return evaluate().to_string(allocator); - } - void write(basic_json_content_handler& handler) const - { - evaluate().write(handler); - } - - void write(std::basic_ostream& os) const - { - evaluate().write(os); - } - - void write(std::basic_ostream& os, const basic_json_options& options) const - { - evaluate().write(os,options); - } - - void write(std::basic_ostream& os, const basic_json_options& options, bool pprint) const - { - evaluate().write(os,options,pprint); - } - - string_type to_string(const basic_json_options& options, char_allocator_type& allocator = char_allocator_type()) const - { - return evaluate().to_string(options,allocator); - } - - range members() - { - return evaluate().members(); - } - - range members() const - { - return evaluate().members(); - } - - range elements() - { - return evaluate().elements(); - } - - range elements() const - { - return evaluate().elements(); - } - void to_stream(basic_json_content_handler& handler) const - { - evaluate().to_stream(handler); - } - - void to_stream(std::basic_ostream& os) const - { - evaluate().to_stream(os); - } - - void to_stream(std::basic_ostream& os, const basic_json_options& options) const - { - evaluate().to_stream(os,options); - } - - void to_stream(std::basic_ostream& os, const basic_json_options& options, bool pprint) const - { - evaluate().to_stream(os,options,pprint); - } -#endif - void swap(basic_json& val) - { - evaluate_with_default().swap(val); - } - - friend std::basic_ostream& operator<<(std::basic_ostream& os, const json_proxy& o) - { - o.dump(os); - return os; - } - -#if !defined(JSONCONS_NO_DEPRECATED) - - void resize_array(size_t n) - { - evaluate().resize_array(n); - } - - template - void resize_array(size_t n, T val) - { - evaluate().resize_array(n,val); - } - - object_iterator begin_members() - { - return evaluate().begin_members(); - } - - const_object_iterator begin_members() const - { - return evaluate().begin_members(); - } - - object_iterator end_members() - { - return evaluate().end_members(); - } - - const_object_iterator end_members() const - { - return evaluate().end_members(); - } - - array_iterator begin_elements() - { - return evaluate().begin_elements(); - } - - const_array_iterator begin_elements() const - { - return evaluate().begin_elements(); - } - - array_iterator end_elements() - { - return evaluate().end_elements(); - } - - const_array_iterator end_elements() const - { - return evaluate().end_elements(); - } - - template - basic_json get(const string_view_type& name, T&& default_val) const - { - return evaluate().get(name,std::forward(default_val)); - } - - const basic_json& get(const string_view_type& name) const - { - return evaluate().get(name); - } - - bool is_ulonglong() const noexcept - { - return evaluate().is_ulonglong(); - } - - bool is_longlong() const noexcept - { - return evaluate().is_longlong(); - } - - int as_int() const - { - return evaluate().as_int(); - } - - unsigned int as_uint() const - { - return evaluate().as_uint(); - } - - long as_long() const - { - return evaluate().as_long(); - } - - unsigned long as_ulong() const - { - return evaluate().as_ulong(); - } - - long long as_longlong() const - { - return evaluate().as_longlong(); - } - - bool has_member(const string_type& name) const - { - return evaluate().has_member(name); - } - - // Remove a range of elements from an array - void remove_range(size_t from_index, size_t to_index) - { - evaluate().remove_range(from_index, to_index); - } - // Remove a range of elements from an array - void remove(const string_view_type& name) - { - evaluate().remove(name); - } - void remove_member(const string_view_type& name) - { - evaluate().remove(name); - } - bool is_empty() const noexcept - { - return empty(); - } - bool is_numeric() const noexcept - { - return is_number(); - } -#endif - }; - - static basic_json parse(std::basic_istream& is) - { - parse_error_handler_type err_handler; - return parse(is,err_handler); - } - - static basic_json parse(std::basic_istream& is, parse_error_handler& err_handler) - { - json_decoder> handler; - basic_json_reader> reader(is, handler, err_handler); - reader.read_next(); - reader.check_done(); - if (!handler.is_valid()) - { - JSONCONS_THROW(json_runtime_error("Failed to parse json stream")); - } - return handler.get_result(); - } - - static basic_json parse(const string_view_type& s) - { - parse_error_handler_type err_handler; - return parse(s,err_handler); - } - -#if !defined(JSONCONS_NO_DEPRECATED) - - void add(size_t index, const basic_json& value) - { - evaluate_with_default().add(index, value); - } - - void add(size_t index, basic_json&& value) - { - evaluate_with_default().add(index, std::forward(value)); - } - - static basic_json parse(const char_type* s, size_t length) - { - parse_error_handler_type err_handler; - return parse(s,length,err_handler); - } - - static basic_json parse(const char_type* s, size_t length, parse_error_handler& err_handler) - { - return parse(string_view_type(s,length),err_handler); - } -#endif - - static basic_json parse(const string_view_type& s, parse_error_handler& err_handler) - { - json_decoder decoder; - basic_json_parser parser(err_handler); - - auto result = unicons::skip_bom(s.begin(), s.end()); - if (result.ec != unicons::encoding_errc()) - { - throw ser_error(result.ec); - } - size_t offset = result.it - s.begin(); - parser.update(s.data()+offset,s.size()-offset); - parser.parse_some(decoder); - parser.finish_parse(decoder); - parser.check_done(); - if (!decoder.is_valid()) - { - JSONCONS_THROW(json_runtime_error("Failed to parse json string")); - } - return decoder.get_result(); - } - - static basic_json parse(std::basic_istream& is, const basic_json_options& options) - { - parse_error_handler_type err_handler; - return parse(is,options,err_handler); - } - - static basic_json parse(std::basic_istream& is, const basic_json_options& options, parse_error_handler& err_handler) - { - json_decoder> handler; - basic_json_reader> reader(is, handler, options, err_handler); - reader.read_next(); - reader.check_done(); - if (!handler.is_valid()) - { - JSONCONS_THROW(json_runtime_error("Failed to parse json stream")); - } - return handler.get_result(); - } - - static basic_json parse(const string_view_type& s, const basic_json_options& options) - { - parse_error_handler_type err_handler; - return parse(s,options,err_handler); - } - - static basic_json parse(const string_view_type& s, const basic_json_options& options, parse_error_handler& err_handler) - { - json_decoder decoder; - basic_json_parser parser(options,err_handler); - - auto result = unicons::skip_bom(s.begin(), s.end()); - if (result.ec != unicons::encoding_errc()) - { - throw ser_error(result.ec); - } - size_t offset = result.it - s.begin(); - parser.update(s.data()+offset,s.size()-offset); - parser.parse_some(decoder); - parser.finish_parse(decoder); - parser.check_done(); - if (!decoder.is_valid()) - { - JSONCONS_THROW(json_runtime_error("Failed to parse json string")); - } - return decoder.get_result(); - } - - static basic_json make_array() - { - return basic_json(array()); - } - - static basic_json make_array(const array& a) - { - return basic_json(a); - } - - static basic_json make_array(const array& a, allocator_type allocator) - { - return basic_json(variant(a, semantic_tag::none, allocator)); - } - - static basic_json make_array(std::initializer_list init, const Allocator& allocator = Allocator()) - { - return array(std::move(init),allocator); - } - - static basic_json make_array(size_t n, const Allocator& allocator = Allocator()) - { - return array(n,allocator); - } - - template - static basic_json make_array(size_t n, const T& val, const Allocator& allocator = Allocator()) - { - return basic_json::array(n, val,allocator); - } - - template - static typename std::enable_if::type make_array(size_t n) - { - return array(n); - } - - template - static typename std::enable_if::type make_array(size_t n, const T& val, const Allocator& allocator = Allocator()) - { - return array(n,val,allocator); - } - - template - static typename std::enable_if<(dim>1),basic_json>::type make_array(size_t n, Args... args) - { - const size_t dim1 = dim - 1; - - basic_json val = make_array(std::forward(args)...); - val.resize(n); - for (size_t i = 0; i < n; ++i) - { - val[i] = make_array(std::forward(args)...); - } - return val; - } - - static const basic_json& null() - { - static basic_json a_null = basic_json(null_type(), semantic_tag::none); - return a_null; - } - - variant var_; - - basic_json(semantic_tag tag = semantic_tag::none) - : var_(tag) - { - } - - explicit basic_json(const Allocator& allocator, semantic_tag tag = semantic_tag::none) - : var_(object(allocator),tag) - { - } - - basic_json(const basic_json& val) - : var_(val.var_) - { - } - - basic_json(const basic_json& val, const Allocator& allocator) - : var_(val.var_,allocator) - { - } - - basic_json(basic_json&& other) noexcept - : var_(std::move(other.var_)) - { - } - - basic_json(basic_json&& other, const Allocator&) noexcept - : var_(std::move(other.var_) /*,allocator*/ ) - { - } - - basic_json(const variant& val) - : var_(val) - { - } - - basic_json(variant&& other) - : var_(std::forward(other)) - { - } - - basic_json(const array& val, semantic_tag tag = semantic_tag::none) - : var_(val, tag) - { - } - - basic_json(array&& other, semantic_tag tag = semantic_tag::none) - : var_(std::forward(other), tag) - { - } - - basic_json(const object& other, semantic_tag tag = semantic_tag::none) - : var_(other, tag) - { - } - - basic_json(object&& other, semantic_tag tag = semantic_tag::none) - : var_(std::forward(other), tag) - { - } - - template - basic_json(const json_proxy& proxy) - : var_(proxy.evaluate().var_) - { - } - - template - basic_json(const json_proxy& proxy, const Allocator& allocator) - : var_(proxy.evaluate().var_,allocator) - { - } - - template - basic_json(const T& val) - : var_(json_type_traits::to_json(val).var_) - { - } - - template - basic_json(const T& val, const Allocator& allocator) - : var_(json_type_traits::to_json(val,allocator).var_) - { - } - - basic_json(const char_type* s, semantic_tag tag = semantic_tag::none) - : var_(s, char_traits_type::length(s), tag) - { - } - - basic_json(const char_type* s, const Allocator& allocator) - : var_(s, char_traits_type::length(s), semantic_tag::none, allocator) - { - } - -#if !defined(JSONCONS_NO_DEPRECATED) - basic_json(double val, uint8_t) - : var_(val, semantic_tag::none) - { - } - basic_json(double val, - const floating_point_options&, - semantic_tag tag = semantic_tag::none) - : var_(val, tag) - { - } -#endif - - basic_json(double val, semantic_tag tag) - : var_(val, tag) - { - } - - template - basic_json(T val, semantic_tag tag, typename std::enable_if::value && std::is_signed::value>::type* = 0) - : var_(static_cast(val), tag) - { - } - - template - basic_json(T val, semantic_tag tag, typename std::enable_if::value && !std::is_signed::value>::type* = 0) - : var_(static_cast(val), tag) - { - } - - basic_json(const char_type *s, size_t length, semantic_tag tag = semantic_tag::none) - : var_(s, length, tag) - { - } - - basic_json(const string_view_type& sv, semantic_tag tag) - : var_(sv.data(), sv.length(), tag) - { - } - - basic_json(null_type val, semantic_tag tag) - : var_(val, tag) - { - } - - basic_json(bool val, semantic_tag tag) - : var_(val, tag) - { - } - - basic_json(const string_view_type& sv, semantic_tag tag, const Allocator& allocator) - : var_(sv.data(), sv.length(), tag, allocator) - { - } - - basic_json(const char_type *s, size_t length, - semantic_tag tag, const Allocator& allocator) - : var_(s, length, tag, allocator) - { - } - -#if !defined(JSONCONS_NO_DEPRECATED) - - basic_json(const byte_string_view& bs, - byte_string_chars_format encoding_hint, - semantic_tag tag = semantic_tag::none) - : var_(bs, tag) - { - switch (encoding_hint) - { - { - case byte_string_chars_format::base16: - var_ = variant(bs, semantic_tag::base16); - break; - case byte_string_chars_format::base64: - var_ = variant(bs, semantic_tag::base64); - break; - case byte_string_chars_format::base64url: - var_ = variant(bs, semantic_tag::base64url); - break; - default: - break; - } - } - } -#endif - - explicit basic_json(const byte_string_view& bs, - semantic_tag tag = semantic_tag::none) - : var_(bs, tag) - { - } - - basic_json(const byte_string_view& bs, - semantic_tag tag, - const Allocator& allocator) - : var_(bs, tag, allocator) - { - } - - explicit basic_json(const basic_bignum& bs) - : var_(bs) - { - } - - explicit basic_json(const basic_bignum& bs, const Allocator& allocator) - : var_(bs, byte_allocator_type(allocator)) - { - } - -#if !defined(JSONCONS_NO_DEPRECATED) - - template - basic_json(InputIterator first, InputIterator last, const Allocator& allocator = Allocator()) - : var_(first,last,allocator) - { - } -#endif - - ~basic_json() - { - } - - basic_json& operator=(const basic_json& rhs) - { - if (this != &rhs) - { - var_ = rhs.var_; - } - return *this; - } - - basic_json& operator=(basic_json&& rhs) noexcept - { - if (this !=&rhs) - { - var_ = std::move(rhs.var_); - } - return *this; - } - - template - basic_json& operator=(const T& val) - { - var_ = json_type_traits::to_json(val).var_; - return *this; - } - - basic_json& operator=(const char_type* s) - { - var_ = variant(s, char_traits_type::length(s), semantic_tag::none); - return *this; - } - - friend bool operator==(const basic_json& lhs, const basic_json& rhs) - { - return lhs.var_ == rhs.var_; - } - - friend bool operator!=(const basic_json& lhs, const basic_json& rhs) - { - return !(lhs == rhs); - } - - friend bool operator<(const basic_json& lhs, const basic_json& rhs) - { - return lhs.var_ < rhs.var_; - } - - friend bool operator<=(const basic_json& lhs, const basic_json& rhs) - { - return !(rhs < lhs); - } - - friend bool operator>(const basic_json& lhs, const basic_json& rhs) - { - return !(lhs <= rhs); - } - - friend bool operator>=(const basic_json& lhs, const basic_json& rhs) - { - return !(lhs < rhs); - } - - size_t size() const noexcept - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - return 0; - case storage_type::object_val: - return object_value().size(); - case storage_type::array_val: - return array_value().size(); - default: - return 0; - } - } - - basic_json& operator[](size_t i) - { - return at(i); - } - - const basic_json& operator[](size_t i) const - { - return at(i); - } - - json_proxy operator[](const string_view_type& name) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - create_object_implicitly(); - JSONCONS_FALLTHROUGH; - case storage_type::object_val: - return json_proxy(*this, name.data(),name.length()); - break; - default: - JSONCONS_THROW(not_an_object(name.data(),name.length())); - break; - } - } - - const basic_json& operator[](const string_view_type& name) const - { - return at(name); - } - - template - void dump(std::basic_string& s) const - { - typedef std::basic_string string_type; - basic_json_compressed_encoder> encoder(s); - dump(encoder); - } - - template - void dump(std::basic_string& s, indenting line_indent) const - { - typedef std::basic_string string_type; - if (line_indent == indenting::indent) - { - basic_json_encoder> encoder(s); - dump(encoder); - } - else - { - basic_json_compressed_encoder> encoder(s); - dump(encoder); - } - } - - template - void dump(std::basic_string& s, - const basic_json_options& options) const - { - typedef std::basic_string string_type; - basic_json_compressed_encoder> encoder(s, options); - dump(encoder); - } - - template - void dump(std::basic_string& s, - const basic_json_options& options, - indenting line_indent) const - { - typedef std::basic_string string_type; - if (line_indent == indenting::indent) - { - basic_json_encoder> encoder(s, options); - dump(encoder); - } - else - { - basic_json_compressed_encoder> encoder(s, options); - dump(encoder); - } - } - - void dump(basic_json_content_handler& handler) const - { - dump_noflush(handler); - handler.flush(); - } - - void dump(std::basic_ostream& os) const - { - basic_json_compressed_encoder encoder(os); - dump(encoder); - } - - void dump(std::basic_ostream& os, indenting line_indent) const - { - if (line_indent == indenting::indent) - { - basic_json_encoder encoder(os); - dump(encoder); - } - else - { - basic_json_compressed_encoder encoder(os); - dump(encoder); - } - } - - void dump(std::basic_ostream& os, const basic_json_options& options) const - { - basic_json_compressed_encoder encoder(os, options); - dump(encoder); - } - - void dump(std::basic_ostream& os, const basic_json_options& options, indenting line_indent) const - { - if (line_indent == indenting::indent) - { - basic_json_encoder encoder(os, options); - dump(encoder); - } - else - { - basic_json_compressed_encoder encoder(os, options); - dump(encoder); - } - } - - string_type to_string(const char_allocator_type& allocator=char_allocator_type()) const noexcept - { - string_type s(allocator); - basic_json_compressed_encoder> encoder(s); - dump(encoder); - return s; - } - - string_type to_string(const basic_json_options& options, - const char_allocator_type& allocator=char_allocator_type()) const - { - string_type s(allocator); - basic_json_compressed_encoder> encoder(s,options); - dump(encoder); - return s; - } - -#if !defined(JSONCONS_NO_DEPRECATED) - void dump_fragment(basic_json_content_handler& handler) const - { - dump(handler); - } - - void dump_body(basic_json_content_handler& handler) const - { - dump(handler); - } - - void dump(std::basic_ostream& os, bool pprint) const - { - if (pprint) - { - basic_json_encoder encoder(os); - dump(encoder); - } - else - { - basic_json_compressed_encoder encoder(os); - dump(encoder); - } - } - - void dump(std::basic_ostream& os, const basic_json_options& options, bool pprint) const - { - if (pprint) - { - basic_json_encoder encoder(os, options); - dump(encoder); - } - else - { - basic_json_compressed_encoder encoder(os, options); - dump(encoder); - } - } - - void write_body(basic_json_content_handler& handler) const - { - dump(handler); - } - void write(basic_json_content_handler& handler) const - { - dump(handler); - } - - void write(std::basic_ostream& os) const - { - dump(os); - } - - void write(std::basic_ostream& os, const basic_json_options& options) const - { - dump(os,options); - } - - void write(std::basic_ostream& os, const basic_json_options& options, bool pprint) const - { - dump(os,options,pprint); - } - - void to_stream(basic_json_content_handler& handler) const - { - dump(handler); - } - - void to_stream(std::basic_ostream& os) const - { - dump(os); - } - - void to_stream(std::basic_ostream& os, const basic_json_options& options) const - { - dump(os,options); - } - - void to_stream(std::basic_ostream& os, const basic_json_options& options, bool pprint) const - { - dump(os,options,pprint ? indenting::indent : indenting::no_indent); - } -#endif - bool is_null() const noexcept - { - return var_.get_storage_type() == storage_type::null_val; - } - - allocator_type get_allocator() const - { - switch (var_.get_storage_type()) - { - case storage_type::long_string_val: - { - return var_.string_data_cast()->get_allocator(); - } - case storage_type::byte_string_val: - { - return var_.byte_string_data_cast()->get_allocator(); - } - case storage_type::array_val: - { - return var_.array_data_cast()->get_allocator(); - } - case storage_type::object_val: - { - return var_.object_data_cast()->get_allocator(); - } - default: - return allocator_type(); - } - } - - bool contains(const string_view_type& name) const - { - switch (var_.get_storage_type()) - { - case storage_type::object_val: - { - const_object_iterator it = object_value().find(name); - return it != object_range().end(); - } - break; - default: - return false; - } - } - - size_t count(const string_view_type& name) const - { - switch (var_.get_storage_type()) - { - case storage_type::object_val: - { - auto it = object_value().find(name); - if (it == object_range().end()) - { - return 0; - } - size_t count = 0; - while (it != object_range().end()&& it->key() == name) - { - ++count; - ++it; - } - return count; - } - break; - default: - return 0; - } - } - - template - bool is(Args&&... args) const - { - return json_type_traits::is(*this,std::forward(args)...); - } - - bool is_string() const noexcept - { - return (var_.get_storage_type() == storage_type::long_string_val) || (var_.get_storage_type() == storage_type::short_string_val); - } - - bool is_string_view() const noexcept - { - return is_string(); - } - - bool is_byte_string() const noexcept - { - return var_.get_storage_type() == storage_type::byte_string_val; - } - - bool is_byte_string_view() const noexcept - { - return is_byte_string(); - } - - bool is_bignum() const - { - switch (get_storage_type()) - { - case storage_type::short_string_val: - case storage_type::long_string_val: - return jsoncons::detail::is_integer(as_string_view().data(), as_string_view().length()); - case storage_type::int64_val: - case storage_type::uint64_val: - return true; - default: - return false; - } - } - - bool is_bool() const noexcept - { - return var_.get_storage_type() == storage_type::bool_val; - } - - bool is_object() const noexcept - { - return var_.get_storage_type() == storage_type::object_val || var_.get_storage_type() == storage_type::empty_object_val; - } - - bool is_array() const noexcept - { - return var_.get_storage_type() == storage_type::array_val; - } - - bool is_int64() const noexcept - { - return var_.get_storage_type() == storage_type::int64_val || (var_.get_storage_type() == storage_type::uint64_val&& (as_integer() <= static_cast((std::numeric_limits::max)()))); - } - - bool is_uint64() const noexcept - { - return var_.get_storage_type() == storage_type::uint64_val || (var_.get_storage_type() == storage_type::int64_val&& as_integer() >= 0); - } - - bool is_double() const noexcept - { - return var_.get_storage_type() == storage_type::double_val; - } - - bool is_number() const noexcept - { - switch (var_.get_storage_type()) - { - case storage_type::int64_val: - case storage_type::uint64_val: - case storage_type::double_val: - return true; - case storage_type::short_string_val: - case storage_type::long_string_val: - return var_.get_semantic_tag() == semantic_tag::bigint || - var_.get_semantic_tag() == semantic_tag::bigdec || - var_.get_semantic_tag() == semantic_tag::bigfloat; -#if !defined(JSONCONS_NO_DEPRECATED) - case storage_type::array_val: - return var_.get_semantic_tag() == semantic_tag::bigfloat; -#endif - default: - return false; - } - } - - bool empty() const noexcept - { - switch (var_.get_storage_type()) - { - case storage_type::byte_string_val: - return var_.byte_string_data_cast()->length() == 0; - break; - case storage_type::short_string_val: - return var_.short_string_data_cast()->length() == 0; - case storage_type::long_string_val: - return var_.string_data_cast()->length() == 0; - case storage_type::array_val: - return array_value().size() == 0; - case storage_type::empty_object_val: - return true; - case storage_type::object_val: - return object_value().size() == 0; - default: - return false; - } - } - - size_t capacity() const - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - return array_value().capacity(); - case storage_type::object_val: - return object_value().capacity(); - default: - return 0; - } - } - - template - void create_object_implicitly() - { - static_assert(is_stateless::value, "Cannot create object implicitly - allocator is stateful."); - var_ = variant(object(Allocator()), semantic_tag::none); - } - - void reserve(size_t n) - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - array_value().reserve(n); - break; - case storage_type::empty_object_val: - { - create_object_implicitly(); - object_value().reserve(n); - } - break; - case storage_type::object_val: - { - object_value().reserve(n); - } - break; - default: - break; - } - } - - void resize(size_t n) - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - array_value().resize(n); - break; - default: - break; - } - } - - template - void resize(size_t n, T val) - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - array_value().resize(n, val); - break; - default: - break; - } - } - - template - T as(Args&&... args) const - { - return json_type_traits::as(*this,std::forward(args)...); - } - - template - typename std::enable_if::value,T>::type - as(const char_allocator_type& allocator) const - { - return json_type_traits::as(*this,allocator); - } - - bool as_bool() const - { - switch (var_.get_storage_type()) - { - case storage_type::short_string_val: - case storage_type::long_string_val: - if (var_.get_semantic_tag() == semantic_tag::bigint) - { - return static_cast(var_.as_bignum()); - } - - try - { - basic_json j = basic_json::parse(as_string_view()); - return j.as_bool(); - } - catch (...) - { - JSONCONS_THROW(json_runtime_error("Not a bool")); - } - break; - case storage_type::bool_val: - return var_.bool_data_cast()->value(); - case storage_type::double_val: - return var_.double_data_cast()->value() != 0.0; - case storage_type::int64_val: - return var_.int64_data_cast()->value() != 0; - case storage_type::uint64_val: - return var_.uint64_data_cast()->value() != 0; - default: - JSONCONS_THROW(json_runtime_error("Not a bool")); - } - } - - template - T as_integer() const - { - switch (var_.get_storage_type()) - { - case storage_type::short_string_val: - case storage_type::long_string_val: - { - auto result = jsoncons::detail::to_integer(as_string_view().data(), as_string_view().length()); - if (result.ec != jsoncons::detail::to_integer_errc()) - { - JSONCONS_THROW(json_runtime_error(make_error_code(result.ec).message())); - } - return result.value; - } - case storage_type::double_val: - return static_cast(var_.double_data_cast()->value()); - case storage_type::int64_val: - return static_cast(var_.int64_data_cast()->value()); - case storage_type::uint64_val: - return static_cast(var_.uint64_data_cast()->value()); - case storage_type::bool_val: - return static_cast(var_.bool_data_cast()->value() ? 1 : 0); - default: - JSONCONS_THROW(json_runtime_error("Not an integer")); - } - } - -#if !defined(JSONCONS_NO_DEPRECATED) - size_t precision() const - { - switch (var_.get_storage_type()) - { - case storage_type::double_val: - return 0; - default: - JSONCONS_THROW(json_runtime_error("Not a double")); - } - } - - size_t decimal_places() const - { - switch (var_.get_storage_type()) - { - case storage_type::double_val: - return 0; - default: - JSONCONS_THROW(json_runtime_error("Not a double")); - } - } -#endif - - double as_double() const - { - switch (var_.get_storage_type()) - { - case storage_type::short_string_val: - case storage_type::long_string_val: - { - jsoncons::detail::string_to_double to_double; - // to_double() throws std::invalid_argument if conversion fails - return to_double(as_cstring(), as_string_view().length()); - } - case storage_type::double_val: - return var_.double_data_cast()->value(); - case storage_type::int64_val: - return static_cast(var_.int64_data_cast()->value()); - case storage_type::uint64_val: - return static_cast(var_.uint64_data_cast()->value()); -#if !defined(JSONCONS_NO_DEPRECATED) - case storage_type::array_val: - if (get_semantic_tag() == semantic_tag::bigfloat) - { - jsoncons::detail::string_to_double to_double; - string_type s = as_string(); - return to_double(s.c_str(), s.length()); - } - else - { - JSONCONS_THROW(json_runtime_error("Not a double")); - } -#endif - default: - JSONCONS_THROW(json_runtime_error("Not a double")); - } - } - - string_view_type as_string_view() const - { - return var_.as_string_view(); - } - - byte_string_view as_byte_string_view() const - { - return var_.as_byte_string_view(); - } - - template > - basic_byte_string as_byte_string() const - { - return var_.template as_byte_string(); - } - - basic_bignum as_bignum() const - { - return var_.as_bignum(); - } - - template > - string_type as_string() const - { - return as_string(basic_json_options(),SAllocator()); - } - - template > - string_type as_string(const SAllocator& allocator) const - { - return as_string(basic_json_options(),allocator); - } - - template > - string_type as_string(const basic_json_options& options) const - { - return as_string(options,SAllocator()); - } - - template > - string_type as_string(const basic_json_options& options, - const SAllocator& allocator) const - { - switch (var_.get_storage_type()) - { - case storage_type::short_string_val: - case storage_type::long_string_val: - { - return string_type(as_string_view().data(),as_string_view().length(),allocator); - } - case storage_type::byte_string_val: - { - string_type s(allocator); - byte_string_chars_format format = jsoncons::detail::resolve_byte_string_chars_format(options.byte_string_format(), - byte_string_chars_format::none, - byte_string_chars_format::base64url); - switch (format) - { - case byte_string_chars_format::base64: - encode_base64(var_.byte_string_data_cast()->begin(), - var_.byte_string_data_cast()->end(), - s); - break; - case byte_string_chars_format::base16: - encode_base16(var_.byte_string_data_cast()->begin(), - var_.byte_string_data_cast()->end(), - s); - break; - default: - encode_base64url(var_.byte_string_data_cast()->begin(), - var_.byte_string_data_cast()->end(), - s); - break; - } - return s; - } - case storage_type::array_val: - { - string_type s(allocator); -#if !defined(JSONCONS_NO_DEPRECATED) - if (get_semantic_tag() == semantic_tag::bigfloat) - { - JSONCONS_ASSERT(size() == 2); - int64_t exp = at(0).template as_integer(); - string_type mantissa = at(1).as_string(); - bignum n(mantissa); - int64_t new_exp = 0; - bignum five(5); - if (exp > 0) - { - new_exp = static_cast(std::floor(exp*std::log(2)/std::log(10))); - bignum five_power = power(five,(unsigned)new_exp); - uint64_t binShift = exp - new_exp; - n = ((n) << (unsigned)binShift)/five_power; - } - else - { - new_exp = static_cast(std::ceil(-exp*std::log(2)/std::log(10))); - bignum five_power = power(five,(unsigned)new_exp); - uint64_t binShift = -exp - new_exp; - n = (n*five_power) >> (unsigned)binShift; - } - - std::string str; - n.dump(str); - if (str[0] == '-') - { - s.push_back('-'); - jsoncons::detail::prettify_string(str.c_str()+1, str.size()-1, -(int)new_exp, -4, 17, s); - } - else - { - jsoncons::detail::prettify_string(str.c_str(), str.size(), -(int)new_exp, -4, 17, s); - } - } - else -#endif - { - basic_json_compressed_encoder> encoder(s,options); - dump(encoder); - } - return s; - } - default: - { - string_type s(allocator); - basic_json_compressed_encoder> encoder(s,options); - dump(encoder); - return s; - } - } - } - - const char_type* as_cstring() const - { - switch (var_.get_storage_type()) - { - case storage_type::short_string_val: - return var_.short_string_data_cast()->c_str(); - case storage_type::long_string_val: - return var_.string_data_cast()->c_str(); - default: - JSONCONS_THROW(json_runtime_error("Not a cstring")); - } - } - -#if !defined(JSONCONS_NO_DEPRECATED) - - bool is_datetime() const noexcept - { - return var_.get_semantic_tag() == semantic_tag::datetime; - } - - bool is_epoch_time() const noexcept - { - return var_.get_semantic_tag() == semantic_tag::timestamp; - } - - bool has_key(const string_view_type& name) const - { - return contains(name); - } - - bool is_integer() const noexcept - { - return var_.get_storage_type() == storage_type::int64_val || (var_.get_storage_type() == storage_type::uint64_val&& (as_integer() <= static_cast((std::numeric_limits::max)()))); - } - - bool is_uinteger() const noexcept - { - return var_.get_storage_type() == storage_type::uint64_val || (var_.get_storage_type() == storage_type::int64_val&& as_integer() >= 0); - } - - int64_t as_uinteger() const - { - return as_integer(); - } - - size_t double_precision() const - { - switch (var_.get_storage_type()) - { - case storage_type::double_val: - return 0; - default: - JSONCONS_THROW(json_runtime_error("Not a double")); - } - } -#endif - - basic_json& at(const string_view_type& name) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - JSONCONS_THROW(key_not_found(name.data(),name.length())); - case storage_type::object_val: - { - auto it = object_value().find(name); - if (it == object_range().end()) - { - JSONCONS_THROW(key_not_found(name.data(),name.length())); - } - return it->value(); - } - break; - default: - { - JSONCONS_THROW(not_an_object(name.data(),name.length())); - } - } - } - - basic_json& evaluate() - { - return *this; - } - - basic_json& evaluate_with_default() - { - return *this; - } - - const basic_json& evaluate() const - { - return *this; - } - basic_json& evaluate(const string_view_type& name) - { - return at(name); - } - - const basic_json& evaluate(const string_view_type& name) const - { - return at(name); - } - - const basic_json& at(const string_view_type& name) const - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - JSONCONS_THROW(key_not_found(name.data(),name.length())); - case storage_type::object_val: - { - auto it = object_value().find(name); - if (it == object_range().end()) - { - JSONCONS_THROW(key_not_found(name.data(),name.length())); - } - return it->value(); - } - break; - default: - { - JSONCONS_THROW(not_an_object(name.data(),name.length())); - } - } - } - - basic_json& at(size_t i) - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - if (i >= array_value().size()) - { - JSONCONS_THROW(json_runtime_error("Invalid array subscript")); - } - return array_value().operator[](i); - case storage_type::object_val: - return object_value().at(i); - default: - JSONCONS_THROW(json_runtime_error("Index on non-array value not supported")); - } - } - - const basic_json& at(size_t i) const - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - if (i >= array_value().size()) - { - JSONCONS_THROW(json_runtime_error("Invalid array subscript")); - } - return array_value().operator[](i); - case storage_type::object_val: - return object_value().at(i); - default: - JSONCONS_THROW(json_runtime_error("Index on non-array value not supported")); - } - } - - object_iterator find(const string_view_type& name) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - return object_range().end(); - case storage_type::object_val: - return object_value().find(name); - default: - { - JSONCONS_THROW(not_an_object(name.data(),name.length())); - } - } - } - - const_object_iterator find(const string_view_type& name) const - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - return object_range().end(); - case storage_type::object_val: - return object_value().find(name); - default: - { - JSONCONS_THROW(not_an_object(name.data(),name.length())); - } - } - } - - template - T get_with_default(const string_view_type& name, const T& default_val) const - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - { - return default_val; - } - case storage_type::object_val: - { - const_object_iterator it = object_value().find(name); - if (it != object_range().end()) - { - return it->value().template as(); - } - else - { - return default_val; - } - } - default: - { - JSONCONS_THROW(not_an_object(name.data(),name.length())); - } - } - } - - template> - T get_with_default(const string_view_type& name, const CharT* default_val) const - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - { - return T(default_val); - } - case storage_type::object_val: - { - const_object_iterator it = object_value().find(name); - if (it != object_range().end()) - { - return it->value().template as(); - } - else - { - return T(default_val); - } - } - default: - { - JSONCONS_THROW(not_an_object(name.data(),name.length())); - } - } - } - - // Modifiers - - void shrink_to_fit() - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - array_value().shrink_to_fit(); - break; - case storage_type::object_val: - object_value().shrink_to_fit(); - break; - default: - break; - } - } - - void clear() - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - array_value().clear(); - break; - case storage_type::object_val: - object_value().clear(); - break; - default: - break; - } - } - - void erase(const_object_iterator pos) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - break; - case storage_type::object_val: - object_value().erase(pos); - break; - default: - JSONCONS_THROW(json_runtime_error("Not an object")); - break; - } - } - - void erase(const_object_iterator first, const_object_iterator last) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - break; - case storage_type::object_val: - object_value().erase(first, last); - break; - default: - JSONCONS_THROW(json_runtime_error("Not an object")); - break; - } - } - - void erase(const_array_iterator pos) - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - array_value().erase(pos); - break; - default: - JSONCONS_THROW(json_runtime_error("Not an array")); - break; - } - } - - void erase(const_array_iterator first, const_array_iterator last) - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - array_value().erase(first, last); - break; - default: - JSONCONS_THROW(json_runtime_error("Not an array")); - break; - } - } - - // Removes all elements from an array value whose index is between from_index, inclusive, and to_index, exclusive. - - void erase(const string_view_type& name) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - break; - case storage_type::object_val: - object_value().erase(name); - break; - default: - JSONCONS_THROW(not_an_object(name.data(),name.length())); - break; - } - } - - template - std::pair insert_or_assign(const string_view_type& name, T&& val) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - create_object_implicitly(); - JSONCONS_FALLTHROUGH; - case storage_type::object_val: - return object_value().insert_or_assign(name, std::forward(val)); - default: - { - JSONCONS_THROW(not_an_object(name.data(),name.length())); - } - } - } - - template - std::pair try_emplace(const string_view_type& name, Args&&... args) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - create_object_implicitly(); - JSONCONS_FALLTHROUGH; - case storage_type::object_val: - return object_value().try_emplace(name, std::forward(args)...); - default: - { - JSONCONS_THROW(not_an_object(name.data(),name.length())); - } - } - } - - // merge - - void merge(const basic_json& source) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - create_object_implicitly(); - JSONCONS_FALLTHROUGH; - case storage_type::object_val: - return object_value().merge(source.object_value()); - default: - { - JSONCONS_THROW(json_runtime_error("Attempting to merge a value that is not an object")); - } - } - } - - void merge(basic_json&& source) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - create_object_implicitly(); - JSONCONS_FALLTHROUGH; - case storage_type::object_val: - return object_value().merge(std::move(source.object_value())); - default: - { - JSONCONS_THROW(json_runtime_error("Attempting to merge a value that is not an object")); - } - } - } - - void merge(object_iterator hint, const basic_json& source) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - create_object_implicitly(); - JSONCONS_FALLTHROUGH; - case storage_type::object_val: - return object_value().merge(hint, source.object_value()); - default: - { - JSONCONS_THROW(json_runtime_error("Attempting to merge a value that is not an object")); - } - } - } - - void merge(object_iterator hint, basic_json&& source) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - create_object_implicitly(); - JSONCONS_FALLTHROUGH; - case storage_type::object_val: - return object_value().merge(hint, std::move(source.object_value())); - default: - { - JSONCONS_THROW(json_runtime_error("Attempting to merge a value that is not an object")); - } - } - } - - // merge_or_update - - void merge_or_update(const basic_json& source) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - create_object_implicitly(); - JSONCONS_FALLTHROUGH; - case storage_type::object_val: - return object_value().merge_or_update(source.object_value()); - default: - { - JSONCONS_THROW(json_runtime_error("Attempting to merge or update a value that is not an object")); - } - } - } - - void merge_or_update(basic_json&& source) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - create_object_implicitly(); - JSONCONS_FALLTHROUGH; - case storage_type::object_val: - return object_value().merge_or_update(std::move(source.object_value())); - default: - { - JSONCONS_THROW(json_runtime_error("Attempting to merge or update a value that is not an object")); - } - } - } - - void merge_or_update(object_iterator hint, const basic_json& source) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - create_object_implicitly(); - JSONCONS_FALLTHROUGH; - case storage_type::object_val: - return object_value().merge_or_update(hint, source.object_value()); - default: - { - JSONCONS_THROW(json_runtime_error("Attempting to merge or update a value that is not an object")); - } - } - } - - void merge_or_update(object_iterator hint, basic_json&& source) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - create_object_implicitly(); - JSONCONS_FALLTHROUGH; - case storage_type::object_val: - return object_value().merge_or_update(hint, std::move(source.object_value())); - default: - { - JSONCONS_THROW(json_runtime_error("Attempting to merge or update a value that is not an object")); - } - } - } - - template - object_iterator insert_or_assign(object_iterator hint, const string_view_type& name, T&& val) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - create_object_implicitly(); - JSONCONS_FALLTHROUGH; - case storage_type::object_val: - return object_value().insert_or_assign(hint, name, std::forward(val)); - default: - { - JSONCONS_THROW(not_an_object(name.data(),name.length())); - } - } - } - - template - object_iterator try_emplace(object_iterator hint, const string_view_type& name, Args&&... args) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - create_object_implicitly(); - JSONCONS_FALLTHROUGH; - case storage_type::object_val: - return object_value().try_emplace(hint, name, std::forward(args)...); - default: - { - JSONCONS_THROW(not_an_object(name.data(),name.length())); - } - } - } - - template - array_iterator insert(const_array_iterator pos, T&& val) - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - return array_value().insert(pos, std::forward(val)); - break; - default: - { - JSONCONS_THROW(json_runtime_error("Attempting to insert into a value that is not an array")); - } - } - } - - template - array_iterator insert(const_array_iterator pos, InputIt first, InputIt last) - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - return array_value().insert(pos, first, last); - break; - default: - { - JSONCONS_THROW(json_runtime_error("Attempting to insert into a value that is not an array")); - } - } - } - - template - void insert(InputIt first, InputIt last) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - case storage_type::object_val: - return object_value().insert(first, last, get_key_value()); - break; - default: - { - JSONCONS_THROW(json_runtime_error("Attempting to insert into a value that is not an object")); - } - } - } - - template - void insert(sorted_unique_range_tag tag, InputIt first, InputIt last) - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - case storage_type::object_val: - return object_value().insert(tag, first, last, get_key_value()); - break; - default: - { - JSONCONS_THROW(json_runtime_error("Attempting to insert into a value that is not an object")); - } - } - } - - template - array_iterator emplace(const_array_iterator pos, Args&&... args) - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - return array_value().emplace(pos, std::forward(args)...); - break; - default: - { - JSONCONS_THROW(json_runtime_error("Attempting to insert into a value that is not an array")); - } - } - } - - template - basic_json& emplace_back(Args&&... args) - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - return array_value().emplace_back(std::forward(args)...); - default: - { - JSONCONS_THROW(json_runtime_error("Attempting to insert into a value that is not an array")); - } - } - } - - storage_type get_storage_type() const - { - return var_.get_storage_type(); - } - - semantic_tag get_semantic_tag() const - { - return var_.get_semantic_tag(); - } - - void swap(basic_json& b) - { - var_.swap(b.var_); - } - - friend void swap(basic_json& a, basic_json& b) - { - a.swap(b); - } - - template - void push_back(T&& val) - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - array_value().push_back(std::forward(val)); - break; - default: - { - JSONCONS_THROW(json_runtime_error("Attempting to insert into a value that is not an array")); - } - } - } - -#if !defined(JSONCONS_NO_DEPRECATED) - - template - void add(T&& val) - { - push_back(std::forward(val)); - } - - template - array_iterator add(const_array_iterator pos, T&& val) - { - return insert(pos, std::forward(val)); - } - - template - std::pair set(const string_view_type& name, T&& val) - { - return insert_or_assign(name, std::forward(val)); - } - - // set - - template - object_iterator set(object_iterator hint, const string_view_type& name, T&& val) - { - return insert_or_assign(hint, name, std::forward(val)); - } - - static basic_json parse_file(const std::basic_string& filename) - { - parse_error_handler_type err_handler; - return parse_file(filename,err_handler); - } - - static basic_json parse_file(const std::basic_string& filename, - parse_error_handler& err_handler) - { - std::basic_ifstream is(filename); - return parse(is,err_handler); - } - - static basic_json parse_stream(std::basic_istream& is) - { - return parse(is); - } - static basic_json parse_stream(std::basic_istream& is, parse_error_handler& err_handler) - { - return parse(is,err_handler); - } - - static basic_json parse_string(const string_type& s) - { - return parse(s); - } - - static basic_json parse_string(const string_type& s, parse_error_handler& err_handler) - { - return parse(s,err_handler); - } - - void resize_array(size_t n) - { - resize(n); - } - - template - void resize_array(size_t n, T val) - { - resize(n,val); - } - - object_iterator begin_members() - { - return object_range().begin(); - } - - const_object_iterator begin_members() const - { - return object_range().begin(); - } - - object_iterator end_members() - { - return object_range().end(); - } - - const_object_iterator end_members() const - { - return object_range().end(); - } - - array_iterator begin_elements() - { - return array_range().begin(); - } - - const_array_iterator begin_elements() const - { - return array_range().begin(); - } - - array_iterator end_elements() - { - return array_range().end(); - } - - const_array_iterator end_elements() const - { - return array_range().end(); - } - - template - basic_json get(const string_view_type& name, T&& default_val) const - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - { - return basic_json(std::forward(default_val)); - } - case storage_type::object_val: - { - const_object_iterator it = object_value().find(name); - if (it != object_range().end()) - { - return it->value(); - } - else - { - return basic_json(std::forward(default_val)); - } - } - default: - { - JSONCONS_THROW(not_an_object(name.data(),name.length())); - } - } - } - - const basic_json& get(const string_view_type& name) const - { - static const basic_json a_null = null_type(); - - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - return a_null; - case storage_type::object_val: - { - const_object_iterator it = object_value().find(name); - return it != object_range().end() ? it->value() : a_null; - } - default: - { - JSONCONS_THROW(not_an_object(name.data(),name.length())); - } - } - } - - bool is_longlong() const noexcept - { - return var_.get_storage_type() == storage_type::int64_val; - } - - bool is_ulonglong() const noexcept - { - return var_.get_storage_type() == storage_type::uint64_val; - } - - long long as_longlong() const - { - return as_integer(); - } - - unsigned long long as_ulonglong() const - { - return as_integer(); - } - - int as_int() const - { - switch (var_.get_storage_type()) - { - case storage_type::double_val: - return static_cast(var_.double_data_cast()->value()); - case storage_type::int64_val: - return static_cast(var_.int64_data_cast()->value()); - case storage_type::uint64_val: - return static_cast(var_.uint64_data_cast()->value()); - case storage_type::bool_val: - return var_.bool_data_cast()->value() ? 1 : 0; - default: - JSONCONS_THROW(json_runtime_error("Not an int")); - } - } - - unsigned int as_uint() const - { - switch (var_.get_storage_type()) - { - case storage_type::double_val: - return static_cast(var_.double_data_cast()->value()); - case storage_type::int64_val: - return static_cast(var_.int64_data_cast()->value()); - case storage_type::uint64_val: - return static_cast(var_.uint64_data_cast()->value()); - case storage_type::bool_val: - return var_.bool_data_cast()->value() ? 1 : 0; - default: - JSONCONS_THROW(json_runtime_error("Not an unsigned int")); - } - } - - long as_long() const - { - switch (var_.get_storage_type()) - { - case storage_type::double_val: - return static_cast(var_.double_data_cast()->value()); - case storage_type::int64_val: - return static_cast(var_.int64_data_cast()->value()); - case storage_type::uint64_val: - return static_cast(var_.uint64_data_cast()->value()); - case storage_type::bool_val: - return var_.bool_data_cast()->value() ? 1 : 0; - default: - JSONCONS_THROW(json_runtime_error("Not a long")); - } - } - - unsigned long as_ulong() const - { - switch (var_.get_storage_type()) - { - case storage_type::double_val: - return static_cast(var_.double_data_cast()->value()); - case storage_type::int64_val: - return static_cast(var_.int64_data_cast()->value()); - case storage_type::uint64_val: - return static_cast(var_.uint64_data_cast()->value()); - case storage_type::bool_val: - return var_.bool_data_cast()->value() ? 1 : 0; - default: - JSONCONS_THROW(json_runtime_error("Not an unsigned long")); - } - } - - bool has_member(const string_type& name) const - { - switch (var_.get_storage_type()) - { - case storage_type::object_val: - { - const_object_iterator it = object_value().find(name); - return it != object_range().end(); - } - break; - default: - return false; - } - } - - void remove_range(size_t from_index, size_t to_index) - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - array_value().remove_range(from_index, to_index); - break; - default: - break; - } - } - // Removes all elements from an array value whose index is between from_index, inclusive, and to_index, exclusive. - - void remove(const string_view_type& name) - { - erase(name); - } - void remove_member(const string_view_type& name) - { - erase(name); - } - // Removes a member from an object value - - bool is_empty() const noexcept - { - return empty(); - } - bool is_numeric() const noexcept - { - return is_number(); - } - - template - static typename std::enable_if::type make_multi_array() - { - return make_array(); - } - template - static typename std::enable_if::type make_multi_array(size_t n) - { - return make_array(n); - } - template - static typename std::enable_if::type make_multi_array(size_t n, T val) - { - return make_array(n,val); - } - template - static typename std::enable_if::type make_multi_array(size_t m, size_t n) - { - return make_array<2>(m, n); - } - template - static typename std::enable_if::type make_multi_array(size_t m, size_t n, T val) - { - return make_array<2>(m, n, val); - } - template - static typename std::enable_if::type make_multi_array(size_t m, size_t n, size_t k) - { - return make_array<3>(m, n, k); - } - template - static typename std::enable_if::type make_multi_array(size_t m, size_t n, size_t k, T val) - { - return make_array<3>(m, n, k, val); - } - range members() - { - return object_range(); - } - - range members() const - { - return object_range(); - } - - range elements() - { - return array_range(); - } - - range elements() const - { - return array_range(); - } -#endif - - range object_range() - { - static basic_json empty_object = object(); - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - return range(empty_object.object_range().begin(), empty_object.object_range().end()); - case storage_type::object_val: - return range(object_value().begin(),object_value().end()); - default: - JSONCONS_THROW(json_runtime_error("Not an object")); - } - } - - range object_range() const - { - static const basic_json empty_object = object(); - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - return range(empty_object.object_range().begin(), empty_object.object_range().end()); - case storage_type::object_val: - return range(object_value().begin(),object_value().end()); - default: - JSONCONS_THROW(json_runtime_error("Not an object")); - } - } - - range array_range() - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - return range(array_value().begin(),array_value().end()); - default: - JSONCONS_THROW(json_runtime_error("Not an array")); - } - } - - range array_range() const - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - return range(array_value().begin(),array_value().end()); - default: - JSONCONS_THROW(json_runtime_error("Not an array")); - } - } - - array& array_value() - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - return var_.array_data_cast()->value(); - default: - JSONCONS_THROW(json_runtime_error("Bad array cast")); - break; - } - } - - const array& array_value() const - { - switch (var_.get_storage_type()) - { - case storage_type::array_val: - return var_.array_data_cast()->value(); - default: - JSONCONS_THROW(json_runtime_error("Bad array cast")); - break; - } - } - - object& object_value() - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - create_object_implicitly(); - JSONCONS_FALLTHROUGH; - case storage_type::object_val: - return var_.object_data_cast()->value(); - default: - JSONCONS_THROW(json_runtime_error("Bad object cast")); - break; - } - } - - const object& object_value() const - { - switch (var_.get_storage_type()) - { - case storage_type::empty_object_val: - const_cast(this)->create_object_implicitly(); // HERE - JSONCONS_FALLTHROUGH; - case storage_type::object_val: - return var_.object_data_cast()->value(); - default: - JSONCONS_THROW(json_runtime_error("Bad object cast")); - break; - } - } - -private: - - void dump_noflush(basic_json_content_handler& handler) const - { - switch (var_.get_storage_type()) - { - case storage_type::short_string_val: - case storage_type::long_string_val: - handler.string_value(as_string_view(), var_.get_semantic_tag()); - break; - case storage_type::byte_string_val: - handler.byte_string_value(var_.byte_string_data_cast()->data(), var_.byte_string_data_cast()->length(), - var_.get_semantic_tag()); - break; - case storage_type::double_val: - handler.double_value(var_.double_data_cast()->value(), - var_.get_semantic_tag()); - break; - case storage_type::int64_val: - handler.int64_value(var_.int64_data_cast()->value(), var_.get_semantic_tag()); - break; - case storage_type::uint64_val: - handler.uint64_value(var_.uint64_data_cast()->value(), var_.get_semantic_tag()); - break; - case storage_type::bool_val: - handler.bool_value(var_.bool_data_cast()->value(), var_.get_semantic_tag()); - break; - case storage_type::null_val: - handler.null_value(var_.get_semantic_tag()); - break; - case storage_type::empty_object_val: - handler.begin_object(0, var_.get_semantic_tag()); - handler.end_object(); - break; - case storage_type::object_val: - { - handler.begin_object(size(), var_.get_semantic_tag()); - const object& o = object_value(); - for (const_object_iterator it = o.begin(); it != o.end(); ++it) - { - handler.name(string_view_type((it->key()).data(),it->key().length())); - it->value().dump_noflush(handler); - } - handler.end_object(); - } - break; - case storage_type::array_val: - { - handler.begin_array(size(), var_.get_semantic_tag()); - const array& o = array_value(); - for (const_array_iterator it = o.begin(); it != o.end(); ++it) - { - it->dump_noflush(handler); - } - handler.end_array(); - } - break; - default: - break; - } - } - - friend std::basic_ostream& operator<<(std::basic_ostream& os, const basic_json& o) - { - o.dump(os); - return os; - } - - friend std::basic_istream& operator<<(std::basic_istream& is, basic_json& o) - { - json_decoder handler; - basic_json_reader> reader(is, handler); - reader.read_next(); - reader.check_done(); - if (!handler.is_valid()) - { - JSONCONS_THROW(json_runtime_error("Failed to parse json stream")); - } - o = handler.get_result(); - return is; - } -}; - -template -void swap(typename Json::key_value_type& a, typename Json::key_value_type& b) -{ - a.swap(b); -} - -template -std::basic_istream& operator>>(std::basic_istream& is, Json& o) -{ - typedef typename Json::char_type char_type; - - json_decoder handler; - basic_json_reader> reader(is, handler); - reader.read_next(); - reader.check_done(); - if (!handler.is_valid()) - { - JSONCONS_THROW(json_runtime_error("Failed to parse json stream")); - } - o = handler.get_result(); - return is; -} - -typedef basic_json> json; -typedef basic_json> wjson; -typedef basic_json> ojson; -typedef basic_json> wojson; - -#if !defined(JSONCONS_NO_DEPRECATED) -typedef basic_json> owjson; -typedef json_decoder json_deserializer; -typedef json_decoder wjson_deserializer; -typedef json_decoder ojson_deserializer; -typedef json_decoder wojson_deserializer; -#endif - -inline namespace literals { - -inline -jsoncons::json operator "" _json(const char* s, std::size_t n) -{ - return jsoncons::json::parse(jsoncons::json::string_view_type(s, n)); -} - -inline -jsoncons::wjson operator "" _json(const wchar_t* s, std::size_t n) -{ - return jsoncons::wjson::parse(jsoncons::wjson::string_view_type(s, n)); -} - -inline -jsoncons::ojson operator "" _ojson(const char* s, std::size_t n) -{ - return jsoncons::ojson::parse(jsoncons::ojson::string_view_type(s, n)); -} - -inline -jsoncons::wojson operator "" _ojson(const wchar_t* s, std::size_t n) -{ - return jsoncons::wojson::parse(jsoncons::wojson::string_view_type(s, n)); -} - -} - -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/bignum.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/bignum.hpp deleted file mode 100644 index f0a650ff94..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/bignum.hpp +++ /dev/null @@ -1,1604 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_BIGNUM_HPP -#define JSONCONS_BIGNUM_HPP - -#include -#include // std::vector -#include -#include -#include // assert -#include // std::numeric_limits -#include // std::max, std::min, std::reverse -#include // std::string -#include // std::memcpy -#include // std::fmod -#include // std::allocator -#include // std::initializer_list -#include // std::enable_if - -namespace jsoncons { - -/* -This implementation is based on Chapter 2 and Appendix A of -Ammeraal, L. (1996) Algorithms and Data Structures in C++, -Chichester: John Wiley. - -*/ - -template -class basic_bignum_base -{ -public: - typedef Allocator allocator_type; - typedef typename std::allocator_traits:: template rebind_alloc basic_type_allocator_type; - -private: - basic_type_allocator_type allocator_; - -public: - basic_bignum_base() - : allocator_() - { - } - explicit basic_bignum_base(const allocator_type& allocator) - : allocator_(basic_type_allocator_type(allocator)) - { - } - - basic_type_allocator_type allocator() const - { - return allocator_; - } -}; - -template > -class basic_bignum : protected basic_bignum_base -{ -private: - using basic_bignum_base::allocator; - - static constexpr uint64_t max_basic_type = (std::numeric_limits::max)(); - static constexpr uint64_t basic_type_bits = sizeof(uint64_t) * 8; // Number of bits - static constexpr uint64_t basic_type_halfBits = basic_type_bits/2; - - static constexpr uint16_t word_length = 4; // Use multiples of word_length words - static constexpr uint64_t r_mask = (uint64_t(1) << basic_type_halfBits) - 1; - static constexpr uint64_t l_mask = max_basic_type - r_mask; - static constexpr uint64_t l_bit = max_basic_type - (max_basic_type >> 1); - - union - { - size_t capacity_; - uint64_t values_[2]; - }; - uint64_t* data_; - bool neg_; - bool dynamic_; - size_t length_; -public: -// Constructors and Destructor - basic_bignum() - : values_{0,0}, data_(values_), neg_(false), dynamic_(false), length_(0) - { - } - - explicit basic_bignum(const Allocator& allocator) - : basic_bignum_base(allocator), values_{0,0}, data_(values_), neg_(false), dynamic_(false), length_(0) - { - } - - basic_bignum(const basic_bignum& n) - : basic_bignum_base(n.allocator()), neg_(n.neg_), length_(n.length_) - { - if (!n.dynamic_) - { - values_[0] = n.values_[0]; - values_[1] = n.values_[1]; - data_ = values_; - dynamic_ = false; - } - else - { - capacity_ = n.capacity_; - data_ = allocator().allocate(capacity_); - dynamic_ = true; - std::memcpy( data_, n.data_, n.length_*sizeof(uint64_t) ); - } - } - - basic_bignum(basic_bignum&& other) - : basic_bignum_base(other.allocator()), neg_(other.neg_), dynamic_(false), length_(other.length_) - { - initialize(std::move(other)); - } - - template - explicit basic_bignum(const CharT* str) - : values_{0,0} - { - initialize(str, strlen(str)); - } - - template - basic_bignum(const std::basic_string& s) - : values_{0,0} - { - initialize(s.data(), s.length()); - } - - template - basic_bignum(const CharT* data, size_t length) - : values_{0,0} - { - initialize(data, length); - } - - template - basic_bignum(const std::basic_string& s, uint8_t base) - : values_{0,0} - { - initialize(s.data(), s.length(), base); - } - - template - basic_bignum(const CharT* data, size_t length, uint8_t base) - : values_{0,0} - { - initialize(data, length, base); - } - - basic_bignum(int signum, std::initializer_list l) - : values_{0,0} - { - if (l.size() > 0) - { - basic_bignum v = 0; - for (auto c: l) - { - v = (v * 256) + (uint64_t)(c); - } - - if (signum == -1) - { - v = -1 - v; - } - - initialize(v); - } - else - { - neg_ = false; - initialize_from_integer(0u); - } - } - - basic_bignum(int signum, const uint8_t* str, size_t n) - : values_{0,0} - { - if (n > 0) - { - basic_bignum v = 0; - for (size_t i = 0; i < n; i++) - { - v = (v * 256) + (uint64_t)(str[i]); - } - - if (signum == -1) - { - v = -1 - v; - } - - initialize(v); - } - else - { - neg_ = false; - initialize_from_integer(0u); - } - } - - basic_bignum(short i) - : values_{0,0} - { - neg_ = i < 0; - uint64_t u = neg_ ? -i : i; - initialize_from_integer( u ); - } - - basic_bignum(unsigned short u) - : values_{0,0} - { - neg_ = false; - initialize_from_integer( u ); - } - - basic_bignum(int i) - : values_{0,0} - { - neg_ = i < 0; - uint64_t u = neg_ ? -i : i; - initialize_from_integer( u ); - } - - basic_bignum(unsigned int u) - : values_{0,0} - { - neg_ = false; - initialize_from_integer( u ); - } - - basic_bignum(long i) - : values_{0,0} - { - neg_ = i < 0; - uint64_t u = neg_ ? -i : i; - initialize_from_integer( u ); - } - - basic_bignum(unsigned long u) - : values_{0,0} - { - neg_ = false; - initialize_from_integer( u ); - } - - basic_bignum(long long i) - : values_{0,0} - { - neg_ = i < 0; - uint64_t u = neg_ ? -i : i; - initialize_from_integer( u ); - } - - basic_bignum(unsigned long long u) - : values_{0,0} - { - neg_ = false; - initialize_from_integer( u ); - } - - explicit basic_bignum( double x ) - : values_{0,0} - { - bool neg = false; - - if ( x < 0 ) - { - neg = true; - x = -x; - } - basic_bignum v = 0; - - double values = (double)max_basic_type + 1.0; - basic_bignum factor = 1; - - while ( x >= 1 ) - { - uint64_t u = (uint64_t) std::fmod( x, values ); - v = v + factor* basic_bignum(u); - x /= values; - factor = factor*(basic_bignum( max_basic_type ) + basic_bignum(1)); - } - - if ( neg ) - { - v.neg_ = true; - } - initialize( v ); - } - - explicit basic_bignum(long double x) - : values_{0,0} - { - bool neg = false; - - if ( x < 0 ) - { - neg = true; - x = -x; - } - - basic_bignum v = 0; - - long double values = (long double)max_basic_type + 1.0; - basic_bignum factor = 1; - - while ( x >= 1.0 ) - { - uint64_t u = (uint64_t) std::fmod( x, values ); - v = v + factor* basic_bignum(u); - x /= values; - factor = factor*(basic_bignum( max_basic_type ) + basic_bignum(1)); - } - - if ( neg ) - { - v.neg_ = true; - } - initialize( v ); - } - - ~basic_bignum() - { - if ( dynamic_ ) - { - allocator().deallocate(data_, capacity_); - } - } - - size_t capacity() const { return dynamic_ ? capacity_ : 2; } - -// Operators - bool operator!() const - { - return length() == 0 ? true : false; - } - - basic_bignum operator-() const - { - basic_bignum v = *this; - v.neg_ = !v.neg_; - return v; - } - - basic_bignum& operator=( const basic_bignum& y ) - { - if ( this != &y ) - { - set_length( y.length() ); - neg_ = y.neg_; - if ( y.length() > 0 ) - { - std::memcpy( data_, y.data_, y.length()*sizeof(uint64_t) ); - } - } - return *this; - } - - basic_bignum& operator+=( const basic_bignum& y ) - { - if ( neg_ != y.neg_ ) - return *this -= -y; - uint64_t d; - uint64_t carry = 0; - - incr_length( (std::max)(y.length(), length()) + 1 ); - - for (size_t i = 0; i < length(); i++ ) - { - if ( i >= y.length() && carry == 0 ) - break; - d = data_[i] + carry; - carry = d < carry; - if ( i < y.length() ) - { - data_[i] = d + y.data_[i]; - if ( data_[i] < d ) - carry = 1; - } - else - data_[i] = d; - } - reduce(); - return *this; - } - - basic_bignum& operator-=( const basic_bignum& y ) - { - if ( neg_ != y.neg_ ) - return *this += -y; - if ( (!neg_ && y > *this) || (neg_ && y < *this) ) - return *this = -(y - *this); - uint64_t borrow = 0; - uint64_t d; - for (size_t i = 0; i < length(); i++ ) - { - if ( i >= y.length() && borrow == 0 ) - break; - d = data_[i] - borrow; - borrow = d > data_[i]; - if ( i < y.length()) - { - data_[i] = d - y.data_[i]; - if ( data_[i] > d ) - borrow = 1; - } - else - data_[i] = d; - } - reduce(); - return *this; - } - - basic_bignum& operator*=( int64_t y ) - { - *this *= uint64_t(y < 0 ? -y : y); - if ( y < 0 ) - neg_ = !neg_; - return *this; - } - - basic_bignum& operator*=( uint64_t y ) - { - size_t len0 = length(); - uint64_t hi; - uint64_t lo; - uint64_t dig = data_[0]; - uint64_t carry = 0; - - incr_length( length() + 1 ); - - size_t i = 0; - for (i = 0; i < len0; i++ ) - { - DDproduct( dig, y, hi, lo ); - data_[i] = lo + carry; - dig = data_[i+1]; - carry = hi + (data_[i] < lo); - } - data_[i] = carry; - reduce(); - return *this; - } - - basic_bignum& operator*=( basic_bignum y ) - { - if ( length() == 0 || y.length() == 0 ) - return *this = 0; - bool difSigns = neg_ != y.neg_; - if ( length() + y.length() == 2 ) // length() = y.length() = 1 - { - uint64_t a = data_[0], b = y.data_[0]; - data_[0] = a * b; - if ( data_[0] / a != b ) - { - set_length( 2 ); - DDproduct( a, b, data_[1], data_[0] ); - } - neg_ = difSigns; - return *this; - } - if ( length() == 1 ) // && y.length() > 1 - { - uint64_t digit = data_[0]; - *this = y; - *this *= digit; - } - else - { - if ( y.length() == 1 ) - *this *= y.data_[0]; - else - { - size_t lenProd = length() + y.length(), jA, jB; - uint64_t sumHi = 0, sumLo, hi, lo, - sumLo_old, sumHi_old, carry=0; - basic_bignum x = *this; - set_length( lenProd ); // Give *this length lenProd - - for (size_t i = 0; i < lenProd; i++ ) - { - sumLo = sumHi; - sumHi = carry; - carry = 0; - for ( jA=0; jA < x.length(); jA++ ) - { - jB = i - jA; - if ( jB >= 0 && jB < y.length() ) - { - DDproduct( x.data_[jA], y.data_[jB], hi, lo ); - sumLo_old = sumLo; - sumHi_old = sumHi; - sumLo += lo; - if ( sumLo < sumLo_old ) - sumHi++; - sumHi += hi; - carry += (sumHi < sumHi_old); - } - } - data_[i] = sumLo; - } - } - } - reduce(); - neg_ = difSigns; - return *this; - } - - basic_bignum& operator/=( const basic_bignum& divisor ) - { - basic_bignum r; - divide( divisor, *this, r, false ); - return *this; - } - - basic_bignum& operator%=( const basic_bignum& divisor ) - { - basic_bignum q; - divide( divisor, q, *this, true ); - return *this; - } - - basic_bignum& operator<<=( uint64_t k ) - { - size_t q = (size_t)(k / basic_type_bits); - if ( q ) // Increase length_ by q: - { - incr_length(length() + q); - for (size_t i = length(); i-- > 0; ) - data_[i] = ( i < q ? 0 : data_[i - q]); - k %= basic_type_bits; - } - if ( k ) // 0 < k < basic_type_bits: - { - uint64_t k1 = basic_type_bits - k; - uint64_t mask = (1 << k) - 1; - incr_length( length() + 1 ); - for (size_t i = length(); i-- > 0; ) - { - data_[i] <<= k; - if ( i > 0 ) - data_[i] |= (data_[i-1] >> k1) & mask; - } - } - reduce(); - return *this; - } - - basic_bignum& operator>>=(uint64_t k) - { - size_t q = (size_t)(k / basic_type_bits); - if ( q >= length() ) - { - set_length( 0 ); - return *this; - } - if (q > 0) - { - memmove( data_, data_+q, (size_t)((length() - q)*sizeof(uint64_t)) ); - set_length( length() - q ); - k %= basic_type_bits; - if ( k == 0 ) - { - reduce(); - return *this; - } - } - - size_t n = (size_t)(length() - 1); - int64_t k1 = basic_type_bits - k; - uint64_t mask = (1 << k) - 1; - for (size_t i = 0; i <= n; i++) - { - data_[i] >>= k; - if ( i < n ) - data_[i] |= ((data_[i+1] & mask) << k1); - } - reduce(); - return *this; - } - - basic_bignum& operator++() - { - *this += 1; - return *this; - } - - basic_bignum operator++(int) - { - basic_bignum old = *this; - ++(*this); - return old; - } - - basic_bignum& operator--() - { - *this -= 1; - return *this; - } - - basic_bignum operator--(int) - { - basic_bignum old = *this; - --(*this); - return old; - } - - basic_bignum& operator|=( const basic_bignum& a ) - { - if ( length() < a.length() ) - { - incr_length( a.length() ); - } - - const uint64_t* qBegin = a.begin(); - const uint64_t* q = a.end() - 1; - uint64_t* p = begin() + a.length() - 1; - - while ( q >= qBegin ) - { - *p-- |= *q--; - } - - reduce(); - - return *this; - } - - basic_bignum& operator^=( const basic_bignum& a ) - { - if ( length() < a.length() ) - { - incr_length( a.length() ); - } - - const uint64_t* qBegin = a.begin(); - const uint64_t* q = a.end() - 1; - uint64_t* p = begin() + a.length() - 1; - - while ( q >= qBegin ) - { - *p-- ^= *q--; - } - - reduce(); - - return *this; - } - - basic_bignum& operator&=( const basic_bignum& a ) - { - size_t old_length = length(); - - set_length( (std::min)( length(), a.length() ) ); - - const uint64_t* pBegin = begin(); - uint64_t* p = end() - 1; - const uint64_t* q = a.begin() + length() - 1; - - while ( p >= pBegin ) - { - *p-- &= *q--; - } - - if ( old_length > length() ) - { - memset( data_ + length(), 0, old_length - length() ); - } - - reduce(); - - return *this; - } - - explicit operator bool() const - { - return length() != 0 ? true : false; - } - - explicit operator int64_t() const - { - int64_t x = 0; - if ( length() > 0 ) - { - x = data_ [0]; - } - - return neg_ ? -x : x; - } - - explicit operator uint64_t() const - { - uint64_t u = 0; - if ( length() > 0 ) - { - u = data_ [0]; - } - - return u; - } - - explicit operator double() const - { - double x = 0.0; - double factor = 1.0; - double values = (double)max_basic_type + 1.0; - - const uint64_t* p = begin(); - const uint64_t* pEnd = end(); - while ( p < pEnd ) - { - x += *p*factor; - factor *= values; - ++p; - } - - return neg_ ? -x : x; - } - - explicit operator long double() const - { - long double x = 0.0; - long double factor = 1.0; - long double values = (long double)max_basic_type + 1.0; - - const uint64_t* p = begin(); - const uint64_t* pEnd = end(); - while ( p < pEnd ) - { - x += *p*factor; - factor *= values; - ++p; - } - - return neg_ ? -x : x; - } - - template - void dump(int& signum, std::vector& data) const - { - basic_bignum n(*this); - if (neg_) - { - signum = -1; - n = - n -1; - } - else - { - signum = 1; - } - basic_bignum divisor(256); - - while (n >= 256) - { - basic_bignum q; - basic_bignum r; - n.divide(divisor, q, r, true); - n = q; - data.push_back((uint8_t)(uint64_t)r); - } - if (n >= 0) - { - data.push_back((uint8_t)(uint64_t)n); - } - std::reverse(data.begin(),data.end()); - } - - template - void dump(std::basic_string& data) const - { - basic_bignum v(*this); - - int len = int(uint32_t(v.length()) * basic_bignum::basic_type_bits / 3) + 2; - data.resize(len); - - int n = len; - int i = 0; - // 1/3 > ln(2)/ln(10) - static uint64_t p10 = 1; - static uint64_t ip10 = 0; - - if ( v.length() == 0 ) - { - data[0] = '0'; - i = 1; - } - else - { - uint64_t r; - if ( p10 == 1 ) - { - while ( p10 <= (std::numeric_limits::max)()/10 ) - { - p10 *= 10; - ip10++; - } - } // p10 is max unsigned power of 10 - basic_bignum R; - basic_bignum LP10 = p10; // LP10 = p10 = ::pow(10, ip10) - if ( v.neg_ ) - { - data[0] = '-'; - i = 1; - } - do - { - v.divide( LP10, v, R, true ); - r = (R.length() ? R.data_[0] : 0); - for ( size_t j=0; j < ip10; j++ ) - { - data[--n] = char(r % 10 + '0'); - r /= 10; - if ( r + v.length() == 0 ) - break; - } - } while ( v.length() ); - while ( n < len ) - data[i++] = data[n++]; - } - data.resize(i); - } - - template - void dump_hex_string(std::basic_string& data) const - { - basic_bignum v(*this); - - int len = int(uint32_t(v.length()) * basic_bignum::basic_type_bits / 3) + 2; - data.resize(len); - - int n = len; - int i = 0; - // 1/3 > ln(2)/ln(10) - static uint64_t p10 = 1; - static uint64_t ip10 = 0; - - if ( v.length() == 0 ) - { - data[0] = '0'; - i = 1; - } - else - { - uint64_t r; - if ( p10 == 1 ) - { - while ( p10 <= (std::numeric_limits::max)()/16 ) - { - p10 *= 16; - ip10++; - } - } // p10 is max unsigned power of 16 - basic_bignum R; - basic_bignum LP10 = p10; // LP10 = p10 = ::pow(16, ip10) - if ( v.neg_ ) - { - data[0] = '-'; - i = 1; - } - do - { - v.divide( LP10, v, R, true ); - r = (R.length() ? R.data_[0] : 0); - for ( size_t j=0; j < ip10; j++ ) - { - uint8_t c = r % 16; - data[--n] = (c < 10) ? ('0' + c) : ('A' - 10 + c); - r /= 16; - if ( r + v.length() == 0 ) - break; - } - } while ( v.length() ); - while ( n < len ) - data[i++] = data[n++]; - } - data.resize(i); - } - -// Global Operators - - friend bool operator==( const basic_bignum& x, const basic_bignum& y ) - { - return x.compare(y) == 0 ? true : false; - } - - friend bool operator==( const basic_bignum& x, int y ) - { - return x.compare(y) == 0 ? true : false; - } - - friend bool operator!=( const basic_bignum& x, const basic_bignum& y ) - { - return x.compare(y) != 0 ? true : false; - } - - friend bool operator!=( const basic_bignum& x, int y ) - { - return x.compare(basic_bignum(y)) != 0 ? true : false; - } - - friend bool operator<( const basic_bignum& x, const basic_bignum& y ) - { - return x.compare(y) < 0 ? true : false; - } - - friend bool operator<( const basic_bignum& x, int64_t y ) - { - return x.compare(y) < 0 ? true : false; - } - - friend bool operator>( const basic_bignum& x, const basic_bignum& y ) - { - return x.compare(y) > 0 ? true : false; - } - - friend bool operator>( const basic_bignum& x, int y ) - { - return x.compare(basic_bignum(y)) > 0 ? true : false; - } - - friend bool operator<=( const basic_bignum& x, const basic_bignum& y ) - { - return x.compare(y) <= 0 ? true : false; - } - - friend bool operator<=( const basic_bignum& x, int y ) - { - return x.compare(y) <= 0 ? true : false; - } - - friend bool operator>=( const basic_bignum& x, const basic_bignum& y ) - { - return x.compare(y) >= 0 ? true : false; - } - - friend bool operator>=( const basic_bignum& x, int y ) - { - return x.compare(y) >= 0 ? true : false; - } - - friend basic_bignum operator+( basic_bignum x, const basic_bignum& y ) - { - return x += y; - } - - friend basic_bignum operator+( basic_bignum x, int64_t y ) - { - return x += y; - } - - friend basic_bignum operator-( basic_bignum x, const basic_bignum& y ) - { - return x -= y; - } - - friend basic_bignum operator-( basic_bignum x, int64_t y ) - { - return x -= y; - } - - friend basic_bignum operator*( int64_t x, const basic_bignum& y ) - { - return basic_bignum(y) *= x; - } - - friend basic_bignum operator*( basic_bignum x, const basic_bignum& y ) - { - return x *= y; - } - - friend basic_bignum operator*( basic_bignum x, int64_t y ) - { - return x *= y; - } - - friend basic_bignum operator/( basic_bignum x, const basic_bignum& y ) - { - return x /= y; - } - - friend basic_bignum operator/( basic_bignum x, int y ) - { - return x /= y; - } - - friend basic_bignum operator%( basic_bignum x, const basic_bignum& y ) - { - return x %= y; - } - - friend basic_bignum operator<<( basic_bignum u, unsigned k ) - { - return u <<= k; - } - - friend basic_bignum operator<<( basic_bignum u, int k ) - { - return u <<= k; - } - - friend basic_bignum operator>>( basic_bignum u, unsigned k ) - { - return u >>= k; - } - - friend basic_bignum operator>>( basic_bignum u, int k ) - { - return u >>= k; - } - - friend basic_bignum operator|( basic_bignum x, const basic_bignum& y ) - { - return x |= y; - } - - friend basic_bignum operator|( basic_bignum x, int y ) - { - return x |= y; - } - - friend basic_bignum operator|( basic_bignum x, unsigned y ) - { - return x |= y; - } - - friend basic_bignum operator^( basic_bignum x, const basic_bignum& y ) - { - return x ^= y; - } - - friend basic_bignum operator^( basic_bignum x, int y ) - { - return x ^= y; - } - - friend basic_bignum operator^( basic_bignum x, unsigned y ) - { - return x ^= y; - } - - friend basic_bignum operator&( basic_bignum x, const basic_bignum& y ) - { - return x &= y; - } - - friend basic_bignum operator&( basic_bignum x, int y ) - { - return x &= y; - } - - friend basic_bignum operator&( basic_bignum x, unsigned y ) - { - return x &= y; - } - - friend basic_bignum abs( const basic_bignum& a ) - { - if ( a.neg_ ) - { - return -a; - } - return a; - } - - friend basic_bignum power( basic_bignum x, unsigned n ) - { - basic_bignum y = 1; - - while ( n ) - { - if ( n & 1 ) - { - y *= x; - } - x *= x; - n >>= 1; - } - - return y; - } - - friend basic_bignum sqrt( const basic_bignum& a ) - { - basic_bignum x = a; - basic_bignum b = a; - basic_bignum q; - - b <<= 1; - while ( b >>= 2, b > 0 ) - { - x >>= 1; - } - while ( x > (q = a/x) + 1 || x < q - 1 ) - { - x += q; - x >>= 1; - } - return x < q ? x : q; - } - - template - friend std::basic_ostream& operator<<(std::basic_ostream& os, const basic_bignum& v) - { - std::basic_string s; - v.dump(s); - os << s; - - return os; - } - - int compare( const basic_bignum& y ) const - { - if ( neg_ != y.neg_ ) - return y.neg_ - neg_; - int code = 0; - if ( length() == 0 && y.length() == 0 ) - code = 0; - else if ( length() < y.length() ) - code = -1; - else if ( length() > y.length() ) - code = +1; - else - { - for (size_t i = length(); i-- > 0; ) - { - if (data_[i] > y.data_[i]) - { - code = 1; - break; - } - else if (data_[i] < y.data_[i]) - { - code = -1; - break; - } - } - } - return neg_ ? -code : code; - } - -private: - void DDproduct( uint64_t A, uint64_t B, - uint64_t& hi, uint64_t& lo ) const - // Multiplying two digits: (hi, lo) = A * B - { - uint64_t hiA = A >> basic_type_halfBits, loA = A & r_mask, - hiB = B >> basic_type_halfBits, loB = B & r_mask, - mid1, mid2, old; - - lo = loA * loB; - hi = hiA * hiB; - mid1 = loA * hiB; - mid2 = hiA * loB; - old = lo; - lo += mid1 << basic_type_halfBits; - hi += (lo < old) + (mid1 >> basic_type_halfBits); - old = lo; - lo += mid2 << basic_type_halfBits; - hi += (lo < old) + (mid2 >> basic_type_halfBits); - } - - uint64_t DDquotient( uint64_t A, uint64_t B, uint64_t d ) const - // Divide double word (A, B) by d. Quotient = (qHi, qLo) - { - uint64_t left, middle, right, qHi, qLo, x, dLo1, - dHi = d >> basic_type_halfBits, dLo = d & r_mask; - qHi = A/(dHi + 1); - // This initial guess of qHi may be too small. - middle = qHi * dLo; - left = qHi * dHi; - x = B - (middle << basic_type_halfBits); - A -= (middle >> basic_type_halfBits) + left + (x > B); - B = x; - dLo1 = dLo << basic_type_halfBits; - // Increase qHi if necessary: - while ( A > dHi || (A == dHi && B >= dLo1) ) - { - x = B - dLo1; - A -= dHi + (x > B); - B = x; - qHi++; - } - qLo = ((A << basic_type_halfBits) | (B >> basic_type_halfBits))/(dHi + 1); - // This initial guess of qLo may be too small. - right = qLo * dLo; - middle = qLo * dHi; - x = B - right; - A -= (x > B); - B = x; - x = B - (middle << basic_type_halfBits); - A -= (middle >> basic_type_halfBits) + (x > B); - B = x; - // Increase qLo if necessary: - while ( A || B >= d ) - { - x = B - d; - A -= (x > B); - B = x; - qLo++; - } - return (qHi << basic_type_halfBits) + qLo; - } - - void subtractmul( uint64_t* a, uint64_t* b, size_t n, uint64_t& q ) const - // a -= q * b: b in n positions; correct q if necessary - { - uint64_t hi, lo, d, carry = 0; - size_t i; - for ( i = 0; i < n; i++ ) - { - DDproduct( b[i], q, hi, lo ); - d = a[i]; - a[i] -= lo; - if ( a[i] > d ) - carry++; - d = a[i + 1]; - a[i + 1] -= hi + carry; - carry = a[i + 1] > d; - } - if ( carry ) // q was too large - { - q--; - carry = 0; - for ( i = 0; i < n; i++ ) - { - d = a[i] + carry; - carry = d < carry; - a[i] = d + b[i]; - if ( a[i] < d ) - carry = 1; - } - a[n] = 0; - } - } - - int normalize( basic_bignum& denom, basic_bignum& num, int& x ) const - { - size_t r = denom.length() - 1; - uint64_t y = denom.data_[r]; - - x = 0; - while ( (y & l_bit) == 0 ) - { - y <<= 1; - x++; - } - denom <<= x; - num <<= x; - if ( r > 0 && denom.data_[r] < denom.data_[r-1] ) - { - denom *= max_basic_type; - num *= max_basic_type; - return 1; - } - return 0; - } - - void unnormalize( basic_bignum& rem, int x, int secondDone ) const - { - if ( secondDone ) - { - rem /= max_basic_type; - } - if ( x > 0 ) - { - rem >>= x; - } - else - { - rem.reduce(); - } - } - - void divide( basic_bignum denom, basic_bignum& quot, basic_bignum& rem, bool remDesired ) const - { - if ( denom.length() == 0 ) - { - throw std::runtime_error( "Zero divide." ); - } - bool quot_neg = neg_ ^ denom.neg_; - bool rem_neg = neg_; - int x = 0; - basic_bignum num = *this; - num.neg_ = denom.neg_ = false; - if ( num < denom ) - { - quot = uint64_t(0); - rem = num; - rem.neg_ = rem_neg; - return; - } - if ( denom.length() == 1 && num.length() == 1 ) - { - quot = uint64_t( num.data_[0]/denom.data_[0] ); - rem = uint64_t( num.data_[0]%denom.data_[0] ); - quot.neg_ = quot_neg; - rem.neg_ = rem_neg; - return; - } - else if (denom.length() == 1 && (denom.data_[0] & l_mask) == 0 ) - { - // Denominator fits into a half word - uint64_t divisor = denom.data_[0], dHi = 0, - q1, r, q2, dividend; - quot.set_length(length()); - for (size_t i=length(); i-- > 0; ) - { - dividend = (dHi << basic_type_halfBits) | (data_[i] >> basic_type_halfBits); - q1 = dividend/divisor; - r = dividend % divisor; - dividend = (r << basic_type_halfBits) | (data_[i] & r_mask); - q2 = dividend/divisor; - dHi = dividend % divisor; - quot.data_[i] = (q1 << basic_type_halfBits) | q2; - } - quot.reduce(); - rem = dHi; - quot.neg_ = quot_neg; - rem.neg_ = rem_neg; - return; - } - basic_bignum num0 = num, denom0 = denom; - int second_done = normalize(denom, num, x); - size_t l = denom.length() - 1; - size_t n = num.length() - 1; - quot.set_length(n - l); - for (size_t i=quot.length(); i-- > 0; ) - quot.data_[i] = 0; - rem = num; - if ( rem.data_[n] >= denom.data_[l] ) - { - rem.incr_length(rem.length() + 1); - n++; - quot.incr_length(quot.length() + 1); - } - uint64_t d = denom.data_[l]; - for ( size_t k = n; k > l; k-- ) - { - uint64_t q = DDquotient(rem.data_[k], rem.data_[k-1], d); - subtractmul( rem.data_ + k - l - 1, denom.data_, l + 1, q ); - quot.data_[k - l - 1] = q; - } - quot.reduce(); - quot.neg_ = quot_neg; - if ( remDesired ) - { - unnormalize(rem, x, second_done); - rem.neg_ = rem_neg; - } - } - - size_t length() const { return length_; } - uint64_t* begin() { return data_; } - const uint64_t* begin() const { return data_; } - uint64_t* end() { return data_ + length_; } - const uint64_t* end() const { return data_ + length_; } - - void set_length(size_t n) - { - length_ = n; - if ( length_ > capacity() ) - { - if ( dynamic_ ) - { - delete[] data_; - } - capacity_ = round_up(length_); - data_ = allocator().allocate(capacity_); - dynamic_ = true; - } - } - - size_t round_up(size_t i) const // Find suitable new block size - { - return (i/word_length + 1) * word_length; - } - - template - typename std::enable_if::value && - !std::is_signed::value && - sizeof(T) <= sizeof(int64_t),void>::type - initialize_from_integer(T u) - { - data_ = values_; - dynamic_ = false; - length_ = u != 0 ? 1 : 0; - - data_ [0] = u; - } - - template - typename std::enable_if::value && - !std::is_signed::value && - sizeof(int64_t) < sizeof(T) && - sizeof(T) <= sizeof(int64_t)*2, void>::type - initialize_from_integer(T u) - { - data_ = values_; - dynamic_ = false; - length_ = u != 0 ? 2 : 0; - - data_[0] = uint64_t(u & max_basic_type); - u >>= basic_type_bits; - data_[1] = uint64_t(u & max_basic_type); - } - - void initialize( const basic_bignum& n ) - { - neg_ = n.neg_; - length_ = n.length_; - - if ( length_ <= 2 ) - { - data_ = values_; - dynamic_ = false; - values_ [0] = n.data_ [0]; - values_ [1] = n.data_ [1]; - } - else - { - capacity_ = round_up( length_ ); - data_ = allocator().allocate(capacity_); - dynamic_ = true; - if ( length_ > 0 ) - { - std::memcpy( data_, n.data_, length_*sizeof(uint64_t) ); - } - reduce(); - } - } - - void initialize(basic_bignum&& other) - { - neg_ = other.neg_; - length_ = other.length_; - dynamic_ = other.dynamic_; - - if (other.dynamic_) - { - capacity_ = other.capacity_; - data_ = other.data_; - - other.data_ = other.values_; - other.dynamic_ = false; - other.length_ = 0; - other.neg_ = false; - } - else - { - values_[0] = other.data_[0]; - values_[1] = other.data_[1]; - data_ = values_; - } - } - - void reduce() - { - uint64_t* p = end() - 1; - uint64_t* pBegin = begin(); - while ( p >= pBegin ) - { - if ( *p ) - { - break; - } - --length_; - --p; - } - if ( length_ == 0 ) - { - neg_ = false; - } - } - - void incr_length( size_t len_new ) - { - size_t len_old = length_; - length_ = len_new; // length_ > len_old - - if ( length_ > capacity() ) - { - size_t capacity_new = round_up( length_ ); - - uint64_t* data_old = data_; - - data_ = allocator().allocate(capacity_new); - - if ( len_old > 0 ) - { - std::memcpy( data_, data_old, len_old*sizeof(uint64_t) ); - } - if ( dynamic_ ) - { - allocator().deallocate(data_old,capacity_); - } - capacity_ = capacity_new; - dynamic_ = true; - } - - if ( length_ > len_old ) - { - memset( data_+len_old, 0, (length_ - len_old)*sizeof(uint64_t) ); - } - } - - template - void initialize(const CharT* data, size_t length) - { - bool neg; - if (*data == '-') - { - neg = true; - data++; - --length; - } - else - { - neg = false; - } - - basic_bignum v = 0; - for (size_t i = 0; i < length; i++) - { - CharT c = data[i]; - switch (c) - { - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - v = (v * 10) + (uint64_t)(c - '0'); - break; - default: - throw std::runtime_error(std::string("Invalid digit ") + "\'" + (char)c + "\'"); - } - } - - if ( neg ) - { - v.neg_ = true; - } - initialize(std::move(v)); - } - - template - void initialize(const CharT* data, size_t length, uint8_t base) - { - if (!(base >= 2 && base <= 16)) - { - throw std::runtime_error("Unsupported base"); - } - - bool neg; - if (*data == '-') - { - neg = true; - data++; - --length; - } - else - { - neg = false; - } - - basic_bignum v = 0; - for (size_t i = 0; i < length; i++) - { - CharT c = data[i]; - uint64_t d; - switch (c) - { - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - d = (uint64_t)(c - '0'); - break; - case 'a':case 'b':case 'c':case 'd':case 'e':case 'f': - d = (uint64_t)(c - ('a' - 10)); - break; - case 'A':case 'B':case 'C':case 'D':case 'E':case 'F': - d = (uint64_t)(c - ('A' - 10)); - break; - default: - throw std::runtime_error(std::string("Invalid digit in base ") + std::to_string(base) + ": \'" + (char)c + "\'"); - } - if (d >= base) - { - throw std::runtime_error(std::string("Invalid digit in base ") + std::to_string(base) + ": \'" + (char)c + "\'"); - } - v = (v * base) + d; - } - - if ( neg ) - { - v.neg_ = true; - } - initialize(std::move(v)); - } -}; - -typedef basic_bignum> bignum; - -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/byte_string.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/byte_string.hpp deleted file mode 100644 index 8d638e813a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/byte_string.hpp +++ /dev/null @@ -1,710 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_BYTE_STRING_HPP -#define JSONCONS_BYTE_STRING_HPP - -#include -#include -#include -#include -#include // std::memcmp -#include // std::allocator -#include -#include -#include -#include // std::move -#include -#include - -namespace jsoncons { - -// Algorithms - -namespace detail { -template -typename std::enable_if::value_type,uint8_t>::value,size_t>::type -encode_base64_generic(InputIt first, InputIt last, const char alphabet[65], Container& result) -{ - size_t count = 0; - unsigned char a3[3]; - unsigned char a4[4]; - unsigned char fill = alphabet[64]; - int i = 0; - int j = 0; - - while (first != last) - { - a3[i++] = *first++; - if (i == 3) - { - a4[0] = (a3[0] & 0xfc) >> 2; - a4[1] = ((a3[0] & 0x03) << 4) + ((a3[1] & 0xf0) >> 4); - a4[2] = ((a3[1] & 0x0f) << 2) + ((a3[2] & 0xc0) >> 6); - a4[3] = a3[2] & 0x3f; - - for (i = 0; i < 4; i++) - { - result.push_back(alphabet[a4[i]]); - ++count; - } - i = 0; - } - } - - if (i > 0) - { - for (j = i; j < 3; ++j) - { - a3[j] = 0; - } - - a4[0] = (a3[0] & 0xfc) >> 2; - a4[1] = ((a3[0] & 0x03) << 4) + ((a3[1] & 0xf0) >> 4); - a4[2] = ((a3[1] & 0x0f) << 2) + ((a3[2] & 0xc0) >> 6); - - for (j = 0; j < i + 1; ++j) - { - result.push_back(alphabet[a4[j]]); - ++count; - } - - if (fill != 0) - { - while (i++ < 3) - { - result.push_back(fill); - ++count; - } - } - } - - return count; -} - -// Hack to support vs2015 -template -typename std::enable_if::type -decode_base64_generic(InputIt first, InputIt last, - const uint8_t reverse_alphabet[256], - F f, - Container& result) -{ - uint8_t a4[4], a3[3]; - uint8_t i = 0; - uint8_t j = 0; - - while (first != last && *first != '=') - { - if (!f(*first)) - { - JSONCONS_THROW(json_runtime_error("Cannot decode encoded byte string")); - } - - a4[i++] = *first++; - if (i == 4) - { - for (i = 0; i < 4; ++i) - { - a4[i] = reverse_alphabet[a4[i]]; - } - - a3[0] = (a4[0] << 2) + ((a4[1] & 0x30) >> 4); - a3[1] = ((a4[1] & 0xf) << 4) + ((a4[2] & 0x3c) >> 2); - a3[2] = ((a4[2] & 0x3) << 6) + a4[3]; - - for (i = 0; i < 3; i++) - { - result.push_back(a3[i]); - } - i = 0; - } - } - - if (i > 0) - { - for (j = 0; j < i; ++j) - { - a4[j] = reverse_alphabet[a4[j]]; - } - - a3[0] = (a4[0] << 2) + ((a4[1] & 0x30) >> 4); - a3[1] = ((a4[1] & 0xf) << 4) + ((a4[2] & 0x3c) >> 2); - - for (j = 0; j < i - 1; ++j) - { - result.push_back(a3[j]); - } - } -} - -} - -template -typename std::enable_if::value_type,uint8_t>::value,size_t>::type -encode_base16(InputIt first, InputIt last, Container& result) -{ - static constexpr char characters[] = "0123456789ABCDEF"; - - for (auto it = first; it != last; ++it) - { - uint8_t c = *it; - result.push_back(characters[c >> 4]); - result.push_back(characters[c & 0xf]); - } - return (last-first)*2; -} - -template -typename std::enable_if::value_type,uint8_t>::value,size_t>::type -encode_base64url(InputIt first, InputIt last, Container& result) -{ - static constexpr char alphabet[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789-_" - "\0"; - return detail::encode_base64_generic(first, last, alphabet, result); -} - -template -typename std::enable_if::value_type,uint8_t>::value,size_t>::type -encode_base64(InputIt first, InputIt last, Container& result) -{ - static constexpr char alphabet[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789+/" - "="; - return detail::encode_base64_generic(first, last, alphabet, result); -} - -template -bool is_base64(Char c) -{ - return (c >= 0 && c < 128) && (isalnum((int)c) || c == '+' || c == '/'); -} - -template -bool is_base64url(Char c) -{ - return (c >= 0 && c < 128) && (isalnum((int)c) || c == '-' || c == '_'); -} - -inline -static bool is_base64url(int c) -{ - return isalnum(c) || c == '-' || c == '_'; -} - -// decode - -// Hack to support vs2015 -template -typename std::enable_if::type -decode_base64url(InputIt first, InputIt last, Container& result) -{ - static constexpr uint8_t reverse_alphabet[256] = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 62, 0xff, 0xff, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0xff, 0xff, 0xff, 0xff, 63, - 0xff, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff - }; - - - jsoncons::detail::decode_base64_generic(first, last, reverse_alphabet, - is_base64url::value_type>, - result); -} - -// Hack to support vs2015 -template -typename std::enable_if::type -decode_base64(InputIt first, InputIt last, Container& result) -{ - static constexpr uint8_t reverse_alphabet[256] = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 62, 0xff, 0xff, 0xff, 63, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff - }; - jsoncons::detail::decode_base64_generic(first, last, reverse_alphabet, - is_base64::value_type>, - result); -} - -// Hack to support vs2015 -template -typename std::enable_if::type -decode_base16(InputIt first, InputIt last, Container& result) -{ - size_t len = std::distance(first,last); - if (len & 1) - { - JSONCONS_THROW(json_runtime_error("Cannot decode encoded base16 string - odd length")); - } - - InputIt it = first; - while (it != last) - { - uint8_t val; - auto a = *it++; - if (a >= '0' && a <= '9') - { - val = (a - '0') << 4; - } - else if ((a | 0x20) >= 'a' && (a | 0x20) <= 'f') - { - val = ((a | 0x20) - 'a' + 10) << 4; - } - else - { - JSONCONS_THROW(json_runtime_error("Not a hex digit. Cannot decode encoded base16 string")); - } - - auto b = *it++; - if (b >= '0' && b <= '9') - { - val |= (b - '0'); - } - else if ((b | 0x20) >= 'a' && (b | 0x20) <= 'f') - { - val |= ((b | 0x20) - 'a' + 10); - } - else - { - JSONCONS_THROW(json_runtime_error("Not a hex digit. Cannot decode encoded base16 string")); - } - - result.push_back(val); - } -} - -struct byte_traits -{ - typedef uint8_t char_type; - - static constexpr int eof() - { - return std::char_traits::eof(); - } - - static int compare(const char_type* s1, const char_type* s2, std::size_t count) - { - return std::memcmp(s1,s2,count); - } -}; - -// basic_byte_string - -template -class basic_byte_string; - -// byte_string_view -class byte_string_view -{ - const uint8_t* data_; - size_t length_; -public: - typedef byte_traits traits_type; - - typedef const uint8_t* const_iterator; - typedef const uint8_t* iterator; - typedef std::size_t size_type; - typedef uint8_t value_type; - - byte_string_view() - : data_(nullptr), length_(0) - { - } - - byte_string_view(const uint8_t* data, size_t length) - : data_(data), length_(length) - { - } - - byte_string_view(const byte_string_view&) = default; - - byte_string_view(byte_string_view&&) = default; - - byte_string_view& operator=(const byte_string_view&) = default; - - byte_string_view& operator=(byte_string_view&&) = default; - - const uint8_t* data() const - { - return data_; - } - - size_t length() const - { - return length_; - } - - size_t size() const - { - return length_; - } - - // iterator support - const_iterator begin() const noexcept - { - return data_; - } - const_iterator end() const noexcept - { - return data_ + length_; - } - - uint8_t operator[](size_type pos) const - { - return data_[pos]; - } - - int compare(const byte_string_view& s) const - { - const int rc = traits_type::compare(data_, s.data(), (std::min)(length_, s.length())); - return rc != 0 ? rc : (length_ == s.length() ? 0 : length_ < s.length() ? -1 : 1); - } - - template - int compare(const basic_byte_string& s) const - { - const int rc = traits_type::compare(data_, s.data(), (std::min)(length_, s.length())); - return rc != 0 ? rc : (length_ == s.length() ? 0 : length_ < s.length() ? -1 : 1); - } - - template - friend std::basic_ostream& operator<<(std::basic_ostream& os, const byte_string_view& o) - { - std::basic_ostringstream ss; - ss.flags(std::ios::hex); - ss.precision(2); - ss.width(2); - ss.fill('0'); - - for (auto b : o) - { - ss << (int)b; - } - os << ss.str(); - return os; - } -}; - -// basic_byte_string -template > -class basic_byte_string -{ - std::vector data_; -public: - typedef byte_traits traits_type; - - typedef typename std::vector::const_iterator const_iterator; - typedef typename std::vector::const_iterator iterator; - typedef std::size_t size_type; - typedef uint8_t value_type; - - basic_byte_string() = default; - - explicit basic_byte_string(const Allocator& alloc) - : data_(alloc) - { - } - - basic_byte_string(std::initializer_list init) - : data_(std::move(init)) - { - } - - basic_byte_string(std::initializer_list init, const Allocator& alloc) - : data_(std::move(init), alloc) - { - } - - explicit basic_byte_string(const byte_string_view& v) - : data_(v.begin(),v.end()) - { - } - - basic_byte_string(const basic_byte_string& v) - : data_(v.data_) - { - } - - basic_byte_string(basic_byte_string&& v) - { - data_.swap(v.data_); - } - - basic_byte_string(const byte_string_view& v, const Allocator& alloc) - : data_(v.begin(),v.end(),alloc) - { - } - - basic_byte_string(const char* s) - { - while (*s) - { - data_.push_back(*s++); - } - } - - basic_byte_string(const uint8_t* data, size_t length) - : data_(data, data+length) - { - } - - basic_byte_string& operator=(const basic_byte_string& s) = default; - - basic_byte_string& operator=(basic_byte_string&& s) = default; - - operator byte_string_view() const noexcept - { - return byte_string_view(data(),length()); - } - - void reserve(size_t new_cap) - { - data_.reserve(new_cap); - } - - void push_back(uint8_t b) - { - data_.push_back(b); - } - - void assign(const uint8_t* s, size_t count) - { - data_.clear(); - data_.insert(s, s+count); - } - - void append(const uint8_t* s, size_t count) - { - data_.insert(s, s+count); - } - - void clear() - { - data_.clear(); - } - - uint8_t operator[](size_type pos) const - { - return data_[pos]; - } - - // iterator support - const_iterator begin() const noexcept - { - return data_.begin(); - } - const_iterator end() const noexcept - { - return data_.end(); - } - - const uint8_t* data() const - { - return data_.data(); - } - - size_t size() const - { - return data_.size(); - } - - size_t length() const - { - return data_.size(); - } - - int compare(const byte_string_view& s) const - { - const int rc = traits_type::compare(data(), s.data(), (std::min)(length(), s.length())); - return rc != 0 ? rc : (length() == s.length() ? 0 : length() < s.length() ? -1 : 1); - } - - int compare(const basic_byte_string& s) const - { - const int rc = traits_type::compare(data(), s.data(), (std::min)(length(), s.length())); - return rc != 0 ? rc : (length() == s.length() ? 0 : length() < s.length() ? -1 : 1); - } - - template - friend std::basic_ostream& operator<<(std::basic_ostream& os, const basic_byte_string& o) - { - os << byte_string_view(o); - return os; - } -}; - -// == -inline -bool operator==(const byte_string_view& lhs, const byte_string_view& rhs) -{ - return lhs.compare(rhs) == 0; -} -template -bool operator==(const byte_string_view& lhs, const basic_byte_string& rhs) -{ - return lhs.compare(rhs) == 0; -} -template -bool operator==(const basic_byte_string& lhs, const byte_string_view& rhs) -{ - return rhs.compare(lhs) == 0; -} -template -bool operator==(const basic_byte_string& lhs, const basic_byte_string& rhs) -{ - return rhs.compare(lhs) == 0; -} - -// != - -inline -bool operator!=(const byte_string_view& lhs, const byte_string_view& rhs) -{ - return lhs.compare(rhs) != 0; -} -template -bool operator!=(const byte_string_view& lhs, const basic_byte_string& rhs) -{ - return lhs.compare(rhs) != 0; -} -template -bool operator!=(const basic_byte_string& lhs, const byte_string_view& rhs) -{ - return rhs.compare(lhs) != 0; -} -template -bool operator!=(const basic_byte_string& lhs, const basic_byte_string& rhs) -{ - return rhs.compare(lhs) != 0; -} - -// <= - -inline -bool operator<=(const byte_string_view& lhs, const byte_string_view& rhs) -{ - return lhs.compare(rhs) <= 0; -} -template -bool operator<=(const byte_string_view& lhs, const basic_byte_string& rhs) -{ - return lhs.compare(rhs) <= 0; -} -template -bool operator<=(const basic_byte_string& lhs, const byte_string_view& rhs) -{ - return rhs.compare(lhs) >= 0; -} -template -bool operator<=(const basic_byte_string& lhs, const basic_byte_string& rhs) -{ - return rhs.compare(lhs) >= 0; -} - -// < - -inline -bool operator<(const byte_string_view& lhs, const byte_string_view& rhs) -{ - return lhs.compare(rhs) < 0; -} -template -bool operator<(const byte_string_view& lhs, const basic_byte_string& rhs) -{ - return lhs.compare(rhs) < 0; -} -template -bool operator<(const basic_byte_string& lhs, const byte_string_view& rhs) -{ - return rhs.compare(lhs) > 0; -} -template -bool operator<(const basic_byte_string& lhs, const basic_byte_string& rhs) -{ - return rhs.compare(lhs) > 0; -} - -// >= - -inline -bool operator>=(const byte_string_view& lhs, const byte_string_view& rhs) -{ - return lhs.compare(rhs) >= 0; -} -template -bool operator>=(const byte_string_view& lhs, const basic_byte_string& rhs) -{ - return lhs.compare(rhs) >= 0; -} -template -bool operator>=(const basic_byte_string& lhs, const byte_string_view& rhs) -{ - return rhs.compare(lhs) <= 0; -} -template -bool operator>=(const basic_byte_string& lhs, const basic_byte_string& rhs) -{ - return rhs.compare(lhs) <= 0; -} - -// > - -inline -bool operator>(const byte_string_view& lhs, const byte_string_view& rhs) -{ - return lhs.compare(rhs) > 0; -} -template -bool operator>(const byte_string_view& lhs, const basic_byte_string& rhs) -{ - return lhs.compare(rhs) > 0; -} -template -bool operator>(const basic_byte_string& lhs, const byte_string_view& rhs) -{ - return rhs.compare(lhs) < 0; -} -template -bool operator>(const basic_byte_string& lhs, const basic_byte_string& rhs) -{ - return rhs.compare(lhs) < 0; -} - -typedef basic_byte_string> byte_string; - - -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/config/binary_detail.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/config/binary_detail.hpp deleted file mode 100644 index 9b94906982..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/config/binary_detail.hpp +++ /dev/null @@ -1,547 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_CONFIG_BINARY_DETAIL_HPP -#define JSONCONS_CONFIG_BINARY_DETAIL_HPP - -#include -#include -#include -#include // std::memcpy -#include -#include // std::enable_if - -#if defined(__apple_build_version__) && ((__clang_major__ < 8) || ((__clang_major__ == 8) && (__clang_minor__ < 1))) -#define APPLE_MISSING_INTRINSICS 1 -#endif - -// The definitions below follow the definitions in compiler_support_p.h, https://github.com/01org/tinycbor -// MIT license - -#ifdef __F16C__ -# include -#endif - -#ifndef __has_builtin -# define __has_builtin(x) 0 -#endif - -#if (defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) || \ - (__has_builtin(__builtin_bswap64) && __has_builtin(__builtin_bswap32)) -# if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -# define JSONCONS_BE64_TO_H __builtin_bswap64 -# define JSONCONS_H_TO_BE64 __builtin_bswap64 -# define JSONCONS_BE32_TO_H __builtin_bswap32 -# define JSONCONS_H_TO_BE32 __builtin_bswap32 -# ifdef __INTEL_COMPILER -# define JSONCONS_BE16_TO_H _bswap16 -# define JSONCONS_H_TO_BE16 _bswap16 -# elif (__GNUC__ * 100 + __GNUC_MINOR__ >= 608) || __has_builtin(__builtin_bswap16) -# define JSONCONS_BE16_TO_H __builtin_bswap16 -# define JSONCONS_H_TO_BE16 __builtin_bswap16 -# else -# define JSONCONS_BE16_TO_H(x) (((uint16_t)x >> 8) | ((uint16_t)x << 8)) -# define JSONCONS_H_TO_BE16 JSONCONS_BE16_TO_H -# endif -# define JSONCONS_LE64_TO_H -# define JSONCONS_H_TO_LE64 -# define JSONCONS_LE32_TO_H -# define JSONCONS_H_TO_LE32 -# define JSONCONS_LE16_TO_H -# define JSONCONS_H_TO_LE16 -# else -# define JSONCONS_LE64_TO_H __builtin_bswap64 -# define JSONCONS_H_TO_LE64 __builtin_bswap64 -# define JSONCONS_LE32_TO_H __builtin_bswap32 -# define JSONCONS_H_TO_LE32 __builtin_bswap32 -# ifdef __INTEL_COMPILER -# define JSONCONS_LE16_TO_H _bswap16 -# define JSONCONS_H_TO_LE16 _bswap16 -# elif (__GNUC__ * 100 + __GNUC_MINOR__ >= 608) || __has_builtin(__builtin_bswap16) -# define JSONCONS_LE16_TO_H __builtin_bswap16 -# define JSONCONS_H_TO_LE16 __builtin_bswap16 -# else -# define JSONCONS_LE16_TO_H(x) (((uint16_t)x >> 8) | ((uint16_t)x << 8)) -# define JSONCONS_H_TO_LE16 JSONCONS_LE16_TO_H -# endif -# define JSONCONS_BE64_TO_H -# define JSONCONS_H_TO_BE64 -# define JSONCONS_BE32_TO_H -# define JSONCONS_H_TO_BE32 -# define JSONCONS_BE16_TO_H -# define JSONCONS_H_TO_BE16 -# endif -#elif defined(__sun) -# include -#elif defined(_MSC_VER) -/* MSVC, which implies Windows, which implies little-endian and sizeof(long) == 4 */ -# define JSONCONS_BE64_TO_H _byteswap_uint64 -# define JSONCONS_H_TO_BE64 _byteswap_uint64 -# define JSONCONS_BE32_TO_H _byteswap_ulong -# define JSONCONS_H_TO_BE32 _byteswap_ulong -# define JSONCONS_BE16_TO_H _byteswap_ushort -# define JSONCONS_H_TO_BE16 _byteswap_ushort -# define JSONCONS_LE64_TO_H -# define JSONCONS_H_TO_LE64 -# define JSONCONS_LE32_TO_H -# define JSONCONS_H_TO_LE32 -# define JSONCONS_LE16_TO_H -# define JSONCONS_H_TO_LE16 -#endif -#ifndef JSONCONS_BE16_TO_H -# include -# define JSONCONS_BE16_TO_H ntohs -# define JSONCONS_H_TO_BE16 htons -#endif -#ifndef JSONCONS_BE32_TO_H -# include -# define JSONCONS_BE32_TO_H ntohl -# define JSONCONS_H_TO_BE32 htonl -#endif -#ifndef JSONCONS_BE64_TO_H -# define JSONCONS_BE64_TO_H ntohll -# define JSONCONS_H_TO_BE64 htonll -/* ntohll isn't usually defined */ -# ifndef ntohll -# if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -# define ntohll -# define htonll -# elif defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -# define ntohll(x) ((ntohl((uint32_t)(x)) * UINT64_C(0x100000000)) + (ntohl((x) >> 32))) -# define htonll ntohll -# else -# error "Unable to determine byte order!" -# endif -# endif -#endif - -namespace jsoncons { namespace detail { - -class read_nbytes_failed : public std::invalid_argument, public virtual json_exception -{ -public: - explicit read_nbytes_failed(size_t count) noexcept - : std::invalid_argument("") - { - buffer_.append("Failed attempting to read "); - buffer_.append(std::to_string(count)); - buffer_.append(" bytes from vector"); - } - ~read_nbytes_failed() noexcept - { - } - const char* what() const noexcept override - { - return buffer_.c_str(); - } -private: - std::string buffer_; -}; - -namespace detail { - -static inline bool add_check_overflow(size_t v1, size_t v2, size_t *r) -{ -#if ((defined(__GNUC__) && (__GNUC__ >= 5)) && !defined(__INTEL_COMPILER)) || __has_builtin(__builtin_add_overflow) - return __builtin_add_overflow(v1, v2, r); -#else - // unsigned additions are well-defined - *r = v1 + v2; - return v1 > v1 + v2; -#endif -} - -} - -inline -uint16_t encode_half(double val) -{ -#if defined(__F16C__) && !defined(APPLE_MISSING_INTRINSICS) - return _cvtss_sh((float)val, 3); -#else - uint64_t v; - std::memcpy(&v, &val, sizeof(v)); - int sign = v >> 63 << 15; - int exp = (v >> 52) & 0x7ff; - int mant = v << 12 >> 12 >> (53-11); /* keep only the 11 most significant bits of the mantissa */ - exp -= 1023; - if (exp == 1024) { - /* infinity or NaN */ - exp = 16; - mant >>= 1; - } else if (exp >= 16) { - /* overflow, as largest number */ - exp = 15; - mant = 1023; - } else if (exp >= -14) { - /* regular normal */ - } else if (exp >= -24) { - /* subnormal */ - mant |= 1024; - mant >>= -(exp + 14); - exp = -15; - } else { - /* underflow, make zero */ - return 0; - } - - /* safe cast here as bit operations above guarantee not to overflow */ - return (uint16_t)(sign | ((exp + 15) << 10) | mant); -#endif -} - -/* this function was copied & adapted from RFC 7049 Appendix D */ -inline -double decode_half(uint16_t half) -{ -#if defined(__F16C__) && !defined(APPLE_MISSING_INTRINSICS) - return _cvtsh_ss(half); -#else - int exp = (half >> 10) & 0x1f; - int mant = half & 0x3ff; - double val; - if (exp == 0) - { - val = ldexp((double)mant, -24); - } - else if (exp != 31) - { - val = ldexp(mant + 1024.0, exp - 25); - } - else - { - val = mant == 0 ? INFINITY : NAN; - } - return half & 0x8000 ? -val : val; -#endif -} - -// to_big_endian - - -template -typename std::enable_if::value && sizeof(T) == sizeof(uint8_t),void>::type -to_big_endian(T val, OutputIt d_first) -{ - *d_first = static_cast(val); -} - - -template -typename std::enable_if::value && -sizeof(T) == sizeof(uint16_t),void>::type -to_big_endian(T val, OutputIt d_first) -{ - T x = JSONCONS_H_TO_BE16(val); - - uint8_t where[sizeof(T)]; - std::memcpy(where, &x, sizeof(T)); - - *d_first++ = where[0]; - *d_first++ = where[1]; -} - -template -typename std::enable_if::value && -sizeof(T) == sizeof(uint32_t),void>::type -to_big_endian(T val, OutputIt d_first) -{ - T x = JSONCONS_H_TO_BE32(val); - - uint8_t where[sizeof(T)]; - std::memcpy(where, &x, sizeof(T)); - - *d_first++ = where[0]; - *d_first++ = where[1]; - *d_first++ = where[2]; - *d_first++ = where[3]; -} - -template -typename std::enable_if::value && -sizeof(T) == sizeof(uint64_t),void>::type -to_big_endian(T val, OutputIt d_first) -{ - T x = JSONCONS_H_TO_BE64(val); - - uint8_t where[sizeof(T)]; - std::memcpy(where, &x, sizeof(T)); - - *d_first++ = where[0]; - *d_first++ = where[1]; - *d_first++ = where[2]; - *d_first++ = where[3]; - *d_first++ = where[4]; - *d_first++ = where[5]; - *d_first++ = where[6]; - *d_first++ = where[7]; -} - -template -void to_big_endian(float val, OutputIt d_first) -{ - uint32_t where; - std::memcpy(&where,&val,sizeof(val)); - to_big_endian(where, d_first); -} - -template -void to_big_endian(double val, OutputIt d_first) -{ - uint64_t where; - std::memcpy(&where,&val,sizeof(val)); - to_big_endian(where, d_first); -} - -// to_little_endian - -template -typename std::enable_if::value && -sizeof(T) == sizeof(uint32_t),void>::type -to_little_endian(T val, OutputIt d_first) -{ - T x = JSONCONS_H_TO_LE32(val); - - uint8_t where[sizeof(T)]; - std::memcpy(where, &x, sizeof(T)); - - *d_first++ = where[0]; - *d_first++ = where[1]; - *d_first++ = where[2]; - *d_first++ = where[3]; -} - -template -typename std::enable_if::value && -sizeof(T) == sizeof(uint64_t),void>::type -to_little_endian(T val, OutputIt d_first) -{ - T x = JSONCONS_H_TO_LE64(val); - - uint8_t where[sizeof(T)]; - std::memcpy(where, &x, sizeof(T)); - - *d_first++ = where[0]; - *d_first++ = where[1]; - *d_first++ = where[2]; - *d_first++ = where[3]; - *d_first++ = where[4]; - *d_first++ = where[5]; - *d_first++ = where[6]; - *d_first++ = where[7]; -} - -template -void to_little_endian(float val, OutputIt d_first) -{ - uint32_t where; - std::memcpy(&where,&val,sizeof(val)); - to_little_endian(where, d_first); -} - -template -void to_little_endian(double val, OutputIt d_first) -{ - uint64_t where; - std::memcpy(&where,&val,sizeof(val)); - to_little_endian(where, d_first); -} - -// from_big_endian - -template -typename std::enable_if::value && -sizeof(T) == sizeof(uint8_t),T>::type -from_big_endian(const uint8_t* first, const uint8_t* last, const uint8_t** endp) -{ - if (first + sizeof(T) > last) - { - *endp = first; - return 0; - } - else - { - *endp = first + sizeof(T); - return static_cast(*(first)); - } -} - -template -typename std::enable_if::value && -sizeof(T) == sizeof(uint16_t),T>::type -from_big_endian(const uint8_t* first, const uint8_t* last, const uint8_t** endp) -{ - if (first + sizeof(T) > last) - { - *endp = first; - return 0; - } - else - { - *endp = first + sizeof(T); - T val; - std::memcpy(&val,first,sizeof(T)); - return JSONCONS_BE16_TO_H(val); - } -} - -template -typename std::enable_if::value && sizeof(T) == sizeof(uint32_t),T>::type -from_big_endian(const uint8_t* first, const uint8_t* last, const uint8_t** endp) -{ - if (first + sizeof(T) > last) - { - *endp = first; - return 0; - } - else - { - *endp = first + sizeof(T); - T val; - std::memcpy(&val,first,sizeof(T)); - return JSONCONS_BE32_TO_H(val); - } -} - -template -typename std::enable_if::value && sizeof(T) == sizeof(uint64_t),T>::type -from_big_endian(const uint8_t* first, const uint8_t* last, const uint8_t** endp) -{ - if (first + sizeof(T) > last) - { - *endp = first; - return 0; - } - else - { - *endp = first + sizeof(T); - T val; - std::memcpy(&val,first,sizeof(T)); - return JSONCONS_BE64_TO_H(val); - } -} - -template -typename std::enable_if::value && -sizeof(T) == sizeof(uint32_t),T>::type -from_big_endian(const uint8_t* first, const uint8_t* last, const uint8_t** endp) -{ - uint32_t data = from_big_endian(first,last,endp); - T val; - std::memcpy(&val,&data,sizeof(T)); - return val; -} - -template -typename std::enable_if::value && -sizeof(T) == sizeof(uint64_t),T>::type -from_big_endian(const uint8_t* first, const uint8_t* last, const uint8_t** endp) -{ - uint64_t data = from_big_endian(first,last,endp); - T val; - std::memcpy(&val,&data,sizeof(T)); - return val; -} - -// from_little_endian - -template -typename std::enable_if::value && -sizeof(T) == sizeof(uint8_t),T>::type -from_little_endian(const uint8_t* first, const uint8_t* last, const uint8_t** endp) -{ - if (first + sizeof(T) > last) - { - *endp = first; - return 0; - } - else - { - *endp = first + sizeof(T); - return static_cast(*(first)); - } -} - -template -typename std::enable_if::value && -sizeof(T) == sizeof(uint16_t),T>::type -from_little_endian(const uint8_t* first, const uint8_t* last, const uint8_t** endp) -{ - if (first + sizeof(T) > last) - { - *endp = first; - return 0; - } - else - { - *endp = first + sizeof(T); - T val; - std::memcpy(&val,first,sizeof(T)); - return JSONCONS_LE16_TO_H(val); - } -} - -template -typename std::enable_if::value && sizeof(T) == sizeof(uint32_t),T>::type -from_little_endian(const uint8_t* first, const uint8_t* last, const uint8_t** endp) -{ - if (first + sizeof(T) > last) - { - *endp = first; - return 0; - } - else - { - *endp = first + sizeof(T); - T val; - std::memcpy(&val,first,sizeof(T)); - return JSONCONS_LE32_TO_H(val); - } -} - -template -typename std::enable_if::value && sizeof(T) == sizeof(uint64_t),T>::type -from_little_endian(const uint8_t* first, const uint8_t* last, const uint8_t** endp) -{ - if (first + sizeof(T) > last) - { - *endp = first; - return 0; - } - else - { - *endp = first + sizeof(T); - T val; - std::memcpy(&val,first,sizeof(T)); - return JSONCONS_LE64_TO_H(val); - } -} - -template -typename std::enable_if::value && -sizeof(T) == sizeof(uint32_t),T>::type -from_little_endian(const uint8_t* first, const uint8_t* last, const uint8_t** endp) -{ - uint32_t data = from_little_endian(first,last,endp); - T val; - std::memcpy(&val,&data,sizeof(T)); - return val; -} - -template -typename std::enable_if::value && -sizeof(T) == sizeof(uint64_t),T>::type -from_little_endian(const uint8_t* first, const uint8_t* last, const uint8_t** endp) -{ - uint64_t data = from_little_endian(first,last,endp); - T val; - std::memcpy(&val,&data,sizeof(T)); - return val; -} - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/config/jsoncons_config.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/config/jsoncons_config.hpp deleted file mode 100644 index f61c18b14c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/config/jsoncons_config.hpp +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSONCONS_CONFIG_HPP -#define JSONCONS_JSONCONS_CONFIG_HPP - -#include -#include -#include -#include - -// Uncomment the following line to suppress deprecated names (recommended for new code) -//#define JSONCONS_NO_DEPRECATED - -// The definitions below follow the definitions in compiler_support_p.h, https://github.com/01org/tinycbor -// MIT license - -// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54577 -#if defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ < 9 -#define JSONCONS_NO_ERASE_TAKING_CONST_ITERATOR 1 -#endif - -#if defined(__clang__) -# define JSONCONS_FALLTHROUGH [[clang::fallthrough]] -#elif defined(__GNUC__) && ((__GNUC__ >= 7)) -# define JSONCONS_FALLTHROUGH __attribute__((fallthrough)) -#elif defined (__GNUC__) -# define JSONCONS_FALLTHROUGH // FALLTHRU -#else -# define JSONCONS_FALLTHROUGH -#endif - -#if defined(__GNUC__) || defined(__clang__) -#define JSONCONS_LIKELY(x) __builtin_expect(!!(x), 1) -#define JSONCONS_UNLIKELY(x) __builtin_expect(!!(x), 0) -#define JSONCONS_UNREACHABLE() __builtin_unreachable() -#elif defined(_MSC_VER) -#define JSONCONS_LIKELY(x) x -#define JSONCONS_UNLIKELY(x) x -#define JSONCONS_UNREACHABLE() __assume(0) -#else -#define JSONCONS_LIKELY(x) x -#define JSONCONS_UNLIKELY(x) x -#define JSONCONS_UNREACHABLE() do {} while (0) -#endif - -// Follows boost 1_68 -#if !defined(JSONCONS_HAS_STRING_VIEW) -# if defined(__clang__) -# if (__cplusplus >= 201703) -# if __has_include() -# define JSONCONS_HAS_STRING_VIEW 1 -# endif // __has_include() -# endif // (__cplusplus >= 201703) -# endif // defined(__clang__) -# if defined(__GNUC__) -# if (__GNUC__ >= 7) -# if (__cplusplus >= 201703) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) -# define JSONCONS_HAS_STRING_VIEW 1 -# endif // (__cplusplus >= 201703) -# endif // (__GNUC__ >= 7) -# endif // defined(__GNUC__) -# if defined(_MSC_VER) -# if (_MSC_VER >= 1910 && _HAS_CXX17) -# define JSONCONS_HAS_STRING_VIEW -# endif // (_MSC_VER >= 1910 && _HAS_CXX17) -# endif // defined(_MSC_VER) -#endif // !defined(JSONCONS_HAS_STRING_VIEW) - -#define JSONCONS_NO_TO_CHARS - -#if defined(ANDROID) || defined(__ANDROID__) -#define JSONCONS_HAS_STRTOLD_L -#if __ANDROID_API__ >= 21 -#else -#define JSONCONS_NO_LOCALECONV -#endif -#endif - -#if defined(_MSC_VER) -#define JSONCONS_HAS_MSC__STRTOD_L -#define JSONCONS_HAS_FOPEN_S -#endif - -#if !defined(JSONCONS_HAS_STRING_VIEW) -#include -namespace jsoncons { -template > -using basic_string_view = jsoncons::detail::basic_string_view; -using string_view = basic_string_view>; -using wstring_view = basic_string_view>; -} -#else -#include -namespace jsoncons { -template > -using basic_string_view = std::basic_string_view; -using string_view = std::string_view; -using wstring_view = std::wstring_view; -} -#endif - -#define JSONCONS_STRING_LITERAL(name, ...) \ - template \ - const std::basic_string& name##_literal() {\ - static const CharT s[] = { __VA_ARGS__};\ - static const std::basic_string sv(s, sizeof(s) / sizeof(CharT));\ - return sv;\ - } - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/config/version.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/config/version.hpp deleted file mode 100644 index 7c012b5536..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/config/version.hpp +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_VERSION_HPP -#define JSONCONS_VERSION_HPP - -#include - -#define JSONCONS_VERSION_MAJOR 0 -#define JSONCONS_VERSION_MINOR 125 -#define JSONCONS_VERSION_PATCH 0 - -namespace jsoncons { - -struct versioning_info -{ - unsigned int const major; - unsigned int const minor; - unsigned int const patch; - - friend std::ostream& operator<<(std::ostream& os, const versioning_info& ver) - { - os << ver.major << '.' - << ver.minor << '.' - << ver.patch; - return os; - } -}; - -constexpr versioning_info version() -{ - return versioning_info{JSONCONS_VERSION_MAJOR, JSONCONS_VERSION_MINOR, JSONCONS_VERSION_PATCH}; -} - -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/grisu3.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/grisu3.hpp deleted file mode 100644 index d8bd81107f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/grisu3.hpp +++ /dev/null @@ -1,405 +0,0 @@ -/* -Implements the Grisu3 algorithm for printing floating-point numbers. - -Follows Florian Loitsch's grisu3_59_56 implementation, available at -http://florian.loitsch.com/publications, in bench.tar.gz, with -minor modifications. -*/ - -/* - Copyright (c) 2009 Florian Loitsch - - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following - conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef JSONCONS_GRISU3_HPP -#define JSONCONS_GRISU3_HPP - -#pragma once -#include -#include -#include -#include -#include -#include // std::memmove - -namespace jsoncons { namespace detail { - -// diy_fp - -typedef struct diy_fp_t { - uint64_t f; - int e; -} diy_fp_t; - -inline -diy_fp_t minus(diy_fp_t x, diy_fp_t y) -{ - assert(x.e == y.e); - assert(x.f >= y.f); - diy_fp_t r = { x.f = x.f - y.f, x.e = x.e }; - return r; -} - -inline -diy_fp_t multiply(diy_fp_t x, diy_fp_t y) -{ - uint64_t a, b, c, d, ac, bc, ad, bd, tmp; - diy_fp_t r; uint64_t M32 = 0xFFFFFFFF; - a = x.f >> 32; b = x.f & M32; - c = y.f >> 32; d = y.f & M32; - ac = a * c; bc = b * c; ad = a * d; bd = b * d; - tmp = (bd >> 32) + (ad & M32) + (bc & M32); - tmp += 1U << 31; /// mult_round - r.f = ac + (ad >> 32) + (bc >> 32) + (tmp >> 32); - r.e = x.e + y.e + 64; - return r; -} - -// k_comp - -inline -int k_comp(int e, int alpha, int /*gamma*/) -{ - constexpr double d_1_log2_10 = 0.30102999566398114; // 1 / lg(10) - return static_cast(std::ceil((alpha - e + 63) * d_1_log2_10)); -} - -// powers_ten_round64 - -constexpr int diy_significand_size = 64; - -static const uint64_t powers_ten[] = { 0xbf29dcaba82fdeae, 0xeef453d6923bd65a, 0x9558b4661b6565f8, 0xbaaee17fa23ebf76, 0xe95a99df8ace6f54, 0x91d8a02bb6c10594, 0xb64ec836a47146fa, 0xe3e27a444d8d98b8, 0x8e6d8c6ab0787f73, 0xb208ef855c969f50, 0xde8b2b66b3bc4724, 0x8b16fb203055ac76, 0xaddcb9e83c6b1794, 0xd953e8624b85dd79, 0x87d4713d6f33aa6c, 0xa9c98d8ccb009506, 0xd43bf0effdc0ba48, 0x84a57695fe98746d, 0xa5ced43b7e3e9188, 0xcf42894a5dce35ea, 0x818995ce7aa0e1b2, 0xa1ebfb4219491a1f, 0xca66fa129f9b60a7, 0xfd00b897478238d1, 0x9e20735e8cb16382, 0xc5a890362fddbc63, 0xf712b443bbd52b7c, 0x9a6bb0aa55653b2d, 0xc1069cd4eabe89f9, 0xf148440a256e2c77, 0x96cd2a865764dbca, 0xbc807527ed3e12bd, 0xeba09271e88d976c, 0x93445b8731587ea3, 0xb8157268fdae9e4c, 0xe61acf033d1a45df, 0x8fd0c16206306bac, 0xb3c4f1ba87bc8697, 0xe0b62e2929aba83c, 0x8c71dcd9ba0b4926, 0xaf8e5410288e1b6f, 0xdb71e91432b1a24b, 0x892731ac9faf056f, 0xab70fe17c79ac6ca, 0xd64d3d9db981787d, 0x85f0468293f0eb4e, 0xa76c582338ed2622, 0xd1476e2c07286faa, 0x82cca4db847945ca, 0xa37fce126597973d, 0xcc5fc196fefd7d0c, 0xff77b1fcbebcdc4f, 0x9faacf3df73609b1, 0xc795830d75038c1e, 0xf97ae3d0d2446f25, 0x9becce62836ac577, 0xc2e801fb244576d5, 0xf3a20279ed56d48a, 0x9845418c345644d7, 0xbe5691ef416bd60c, 0xedec366b11c6cb8f, 0x94b3a202eb1c3f39, 0xb9e08a83a5e34f08, 0xe858ad248f5c22ca, 0x91376c36d99995be, 0xb58547448ffffb2e, 0xe2e69915b3fff9f9, 0x8dd01fad907ffc3c, 0xb1442798f49ffb4b, 0xdd95317f31c7fa1d, 0x8a7d3eef7f1cfc52, 0xad1c8eab5ee43b67, 0xd863b256369d4a41, 0x873e4f75e2224e68, 0xa90de3535aaae202, 0xd3515c2831559a83, 0x8412d9991ed58092, 0xa5178fff668ae0b6, 0xce5d73ff402d98e4, 0x80fa687f881c7f8e, 0xa139029f6a239f72, 0xc987434744ac874f, 0xfbe9141915d7a922, 0x9d71ac8fada6c9b5, 0xc4ce17b399107c23, 0xf6019da07f549b2b, 0x99c102844f94e0fb, 0xc0314325637a193a, 0xf03d93eebc589f88, 0x96267c7535b763b5, 0xbbb01b9283253ca3, 0xea9c227723ee8bcb, 0x92a1958a7675175f, 0xb749faed14125d37, 0xe51c79a85916f485, 0x8f31cc0937ae58d3, 0xb2fe3f0b8599ef08, 0xdfbdcece67006ac9, 0x8bd6a141006042be, 0xaecc49914078536d, 0xda7f5bf590966849, 0x888f99797a5e012d, 0xaab37fd7d8f58179, 0xd5605fcdcf32e1d7, 0x855c3be0a17fcd26, 0xa6b34ad8c9dfc070, 0xd0601d8efc57b08c, 0x823c12795db6ce57, 0xa2cb1717b52481ed, 0xcb7ddcdda26da269, 0xfe5d54150b090b03, 0x9efa548d26e5a6e2, 0xc6b8e9b0709f109a, 0xf867241c8cc6d4c1, 0x9b407691d7fc44f8, 0xc21094364dfb5637, 0xf294b943e17a2bc4, 0x979cf3ca6cec5b5b, 0xbd8430bd08277231, 0xece53cec4a314ebe, 0x940f4613ae5ed137, 0xb913179899f68584, 0xe757dd7ec07426e5, 0x9096ea6f3848984f, 0xb4bca50b065abe63, 0xe1ebce4dc7f16dfc, 0x8d3360f09cf6e4bd, 0xb080392cc4349ded, 0xdca04777f541c568, 0x89e42caaf9491b61, 0xac5d37d5b79b6239, 0xd77485cb25823ac7, 0x86a8d39ef77164bd, 0xa8530886b54dbdec, 0xd267caa862a12d67, 0x8380dea93da4bc60, 0xa46116538d0deb78, 0xcd795be870516656, 0x806bd9714632dff6, 0xa086cfcd97bf97f4, 0xc8a883c0fdaf7df0, 0xfad2a4b13d1b5d6c, 0x9cc3a6eec6311a64, 0xc3f490aa77bd60fd, 0xf4f1b4d515acb93c, 0x991711052d8bf3c5, 0xbf5cd54678eef0b7, 0xef340a98172aace5, 0x9580869f0e7aac0f, 0xbae0a846d2195713, 0xe998d258869facd7, 0x91ff83775423cc06, 0xb67f6455292cbf08, 0xe41f3d6a7377eeca, 0x8e938662882af53e, 0xb23867fb2a35b28e, 0xdec681f9f4c31f31, 0x8b3c113c38f9f37f, 0xae0b158b4738705f, 0xd98ddaee19068c76, 0x87f8a8d4cfa417ca, 0xa9f6d30a038d1dbc, 0xd47487cc8470652b, 0x84c8d4dfd2c63f3b, 0xa5fb0a17c777cf0a, 0xcf79cc9db955c2cc, 0x81ac1fe293d599c0, 0xa21727db38cb0030, 0xca9cf1d206fdc03c, 0xfd442e4688bd304b, 0x9e4a9cec15763e2f, 0xc5dd44271ad3cdba, 0xf7549530e188c129, 0x9a94dd3e8cf578ba, 0xc13a148e3032d6e8, 0xf18899b1bc3f8ca2, 0x96f5600f15a7b7e5, 0xbcb2b812db11a5de, 0xebdf661791d60f56, 0x936b9fcebb25c996, 0xb84687c269ef3bfb, 0xe65829b3046b0afa, 0x8ff71a0fe2c2e6dc, 0xb3f4e093db73a093, 0xe0f218b8d25088b8, 0x8c974f7383725573, 0xafbd2350644eead0, 0xdbac6c247d62a584, 0x894bc396ce5da772, 0xab9eb47c81f5114f, 0xd686619ba27255a3, 0x8613fd0145877586, 0xa798fc4196e952e7, 0xd17f3b51fca3a7a1, 0x82ef85133de648c5, 0xa3ab66580d5fdaf6, 0xcc963fee10b7d1b3, 0xffbbcfe994e5c620, 0x9fd561f1fd0f9bd4, 0xc7caba6e7c5382c9, 0xf9bd690a1b68637b, 0x9c1661a651213e2d, 0xc31bfa0fe5698db8, 0xf3e2f893dec3f126, 0x986ddb5c6b3a76b8, 0xbe89523386091466, 0xee2ba6c0678b597f, 0x94db483840b717f0, 0xba121a4650e4ddec, 0xe896a0d7e51e1566, 0x915e2486ef32cd60, 0xb5b5ada8aaff80b8, 0xe3231912d5bf60e6, 0x8df5efabc5979c90, 0xb1736b96b6fd83b4, 0xddd0467c64bce4a1, 0x8aa22c0dbef60ee4, 0xad4ab7112eb3929e, 0xd89d64d57a607745, 0x87625f056c7c4a8b, 0xa93af6c6c79b5d2e, 0xd389b47879823479, 0x843610cb4bf160cc, 0xa54394fe1eedb8ff, 0xce947a3da6a9273e, 0x811ccc668829b887, 0xa163ff802a3426a9, 0xc9bcff6034c13053, 0xfc2c3f3841f17c68, 0x9d9ba7832936edc1, 0xc5029163f384a931, 0xf64335bcf065d37d, 0x99ea0196163fa42e, 0xc06481fb9bcf8d3a, 0xf07da27a82c37088, 0x964e858c91ba2655, 0xbbe226efb628afeb, 0xeadab0aba3b2dbe5, 0x92c8ae6b464fc96f, 0xb77ada0617e3bbcb, 0xe55990879ddcaabe, 0x8f57fa54c2a9eab7, 0xb32df8e9f3546564, 0xdff9772470297ebd, 0x8bfbea76c619ef36, 0xaefae51477a06b04, 0xdab99e59958885c5, 0x88b402f7fd75539b, 0xaae103b5fcd2a882, 0xd59944a37c0752a2, 0x857fcae62d8493a5, 0xa6dfbd9fb8e5b88f, 0xd097ad07a71f26b2, 0x825ecc24c8737830, 0xa2f67f2dfa90563b, 0xcbb41ef979346bca, 0xfea126b7d78186bd, 0x9f24b832e6b0f436, 0xc6ede63fa05d3144, 0xf8a95fcf88747d94, 0x9b69dbe1b548ce7d, 0xc24452da229b021c, 0xf2d56790ab41c2a3, 0x97c560ba6b0919a6, 0xbdb6b8e905cb600f, 0xed246723473e3813, 0x9436c0760c86e30c, 0xb94470938fa89bcf, 0xe7958cb87392c2c3, 0x90bd77f3483bb9ba, 0xb4ecd5f01a4aa828, 0xe2280b6c20dd5232, 0x8d590723948a535f, 0xb0af48ec79ace837, 0xdcdb1b2798182245, 0x8a08f0f8bf0f156b, 0xac8b2d36eed2dac6, 0xd7adf884aa879177, 0x86ccbb52ea94baeb, 0xa87fea27a539e9a5, 0xd29fe4b18e88640f, 0x83a3eeeef9153e89, 0xa48ceaaab75a8e2b, 0xcdb02555653131b6, 0x808e17555f3ebf12, 0xa0b19d2ab70e6ed6, 0xc8de047564d20a8c, 0xfb158592be068d2f, 0x9ced737bb6c4183d, 0xc428d05aa4751e4d, 0xf53304714d9265e0, 0x993fe2c6d07b7fac, 0xbf8fdb78849a5f97, 0xef73d256a5c0f77d, 0x95a8637627989aae, 0xbb127c53b17ec159, 0xe9d71b689dde71b0, 0x9226712162ab070e, 0xb6b00d69bb55c8d1, 0xe45c10c42a2b3b06, 0x8eb98a7a9a5b04e3, 0xb267ed1940f1c61c, 0xdf01e85f912e37a3, 0x8b61313bbabce2c6, 0xae397d8aa96c1b78, 0xd9c7dced53c72256, 0x881cea14545c7575, 0xaa242499697392d3, 0xd4ad2dbfc3d07788, 0x84ec3c97da624ab5, 0xa6274bbdd0fadd62, 0xcfb11ead453994ba, 0x81ceb32c4b43fcf5, 0xa2425ff75e14fc32, 0xcad2f7f5359a3b3e, 0xfd87b5f28300ca0e, 0x9e74d1b791e07e48, 0xc612062576589ddb, 0xf79687aed3eec551, 0x9abe14cd44753b53, 0xc16d9a0095928a27, 0xf1c90080baf72cb1, 0x971da05074da7bef, 0xbce5086492111aeb, 0xec1e4a7db69561a5, 0x9392ee8e921d5d07, 0xb877aa3236a4b449, 0xe69594bec44de15b, 0x901d7cf73ab0acd9, 0xb424dc35095cd80f, 0xe12e13424bb40e13, 0x8cbccc096f5088cc, 0xafebff0bcb24aaff, 0xdbe6fecebdedd5bf, 0x89705f4136b4a597, 0xabcc77118461cefd, 0xd6bf94d5e57a42bc, 0x8637bd05af6c69b6, 0xa7c5ac471b478423, 0xd1b71758e219652c, 0x83126e978d4fdf3b, 0xa3d70a3d70a3d70a, 0xcccccccccccccccd, 0x8000000000000000, 0xa000000000000000, 0xc800000000000000, 0xfa00000000000000, 0x9c40000000000000, 0xc350000000000000, 0xf424000000000000, 0x9896800000000000, 0xbebc200000000000, 0xee6b280000000000, 0x9502f90000000000, 0xba43b74000000000, 0xe8d4a51000000000, 0x9184e72a00000000, 0xb5e620f480000000, 0xe35fa931a0000000, 0x8e1bc9bf04000000, 0xb1a2bc2ec5000000, 0xde0b6b3a76400000, 0x8ac7230489e80000, 0xad78ebc5ac620000, 0xd8d726b7177a8000, 0x878678326eac9000, 0xa968163f0a57b400, 0xd3c21bcecceda100, 0x84595161401484a0, 0xa56fa5b99019a5c8, 0xcecb8f27f4200f3a, 0x813f3978f8940984, 0xa18f07d736b90be5, 0xc9f2c9cd04674edf, 0xfc6f7c4045812296, 0x9dc5ada82b70b59e, 0xc5371912364ce305, 0xf684df56c3e01bc7, 0x9a130b963a6c115c, 0xc097ce7bc90715b3, 0xf0bdc21abb48db20, 0x96769950b50d88f4, 0xbc143fa4e250eb31, 0xeb194f8e1ae525fd, 0x92efd1b8d0cf37be, 0xb7abc627050305ae, 0xe596b7b0c643c719, 0x8f7e32ce7bea5c70, 0xb35dbf821ae4f38c, 0xe0352f62a19e306f, 0x8c213d9da502de45, 0xaf298d050e4395d7, 0xdaf3f04651d47b4c, 0x88d8762bf324cd10, 0xab0e93b6efee0054, 0xd5d238a4abe98068, 0x85a36366eb71f041, 0xa70c3c40a64e6c52, 0xd0cf4b50cfe20766, 0x82818f1281ed44a0, 0xa321f2d7226895c8, 0xcbea6f8ceb02bb3a, 0xfee50b7025c36a08, 0x9f4f2726179a2245, 0xc722f0ef9d80aad6, 0xf8ebad2b84e0d58c, 0x9b934c3b330c8577, 0xc2781f49ffcfa6d5, 0xf316271c7fc3908b, 0x97edd871cfda3a57, 0xbde94e8e43d0c8ec, 0xed63a231d4c4fb27, 0x945e455f24fb1cf9, 0xb975d6b6ee39e437, 0xe7d34c64a9c85d44, 0x90e40fbeea1d3a4b, 0xb51d13aea4a488dd, 0xe264589a4dcdab15, 0x8d7eb76070a08aed, 0xb0de65388cc8ada8, 0xdd15fe86affad912, 0x8a2dbf142dfcc7ab, 0xacb92ed9397bf996, 0xd7e77a8f87daf7fc, 0x86f0ac99b4e8dafd, 0xa8acd7c0222311bd, 0xd2d80db02aabd62c, 0x83c7088e1aab65db, 0xa4b8cab1a1563f52, 0xcde6fd5e09abcf27, 0x80b05e5ac60b6178, 0xa0dc75f1778e39d6, 0xc913936dd571c84c, 0xfb5878494ace3a5f, 0x9d174b2dcec0e47b, 0xc45d1df942711d9a, 0xf5746577930d6501, 0x9968bf6abbe85f20, 0xbfc2ef456ae276e9, 0xefb3ab16c59b14a3, 0x95d04aee3b80ece6, 0xbb445da9ca61281f, 0xea1575143cf97227, 0x924d692ca61be758, 0xb6e0c377cfa2e12e, 0xe498f455c38b997a, 0x8edf98b59a373fec, 0xb2977ee300c50fe7, 0xdf3d5e9bc0f653e1, 0x8b865b215899f46d, 0xae67f1e9aec07188, 0xda01ee641a708dea, 0x884134fe908658b2, 0xaa51823e34a7eedf, 0xd4e5e2cdc1d1ea96, 0x850fadc09923329e, 0xa6539930bf6bff46, 0xcfe87f7cef46ff17, 0x81f14fae158c5f6e, 0xa26da3999aef774a, 0xcb090c8001ab551c, 0xfdcb4fa002162a63, 0x9e9f11c4014dda7e, 0xc646d63501a1511e, 0xf7d88bc24209a565, 0x9ae757596946075f, 0xc1a12d2fc3978937, 0xf209787bb47d6b85, 0x9745eb4d50ce6333, 0xbd176620a501fc00, 0xec5d3fa8ce427b00, 0x93ba47c980e98ce0, 0xb8a8d9bbe123f018, 0xe6d3102ad96cec1e, 0x9043ea1ac7e41393, 0xb454e4a179dd1877, 0xe16a1dc9d8545e95, 0x8ce2529e2734bb1d, 0xb01ae745b101e9e4, 0xdc21a1171d42645d, 0x899504ae72497eba, 0xabfa45da0edbde69, 0xd6f8d7509292d603, 0x865b86925b9bc5c2, 0xa7f26836f282b733, 0xd1ef0244af2364ff, 0x8335616aed761f1f, 0xa402b9c5a8d3a6e7, 0xcd036837130890a1, 0x802221226be55a65, 0xa02aa96b06deb0fe, 0xc83553c5c8965d3d, 0xfa42a8b73abbf48d, 0x9c69a97284b578d8, 0xc38413cf25e2d70e, 0xf46518c2ef5b8cd1, 0x98bf2f79d5993803, 0xbeeefb584aff8604, 0xeeaaba2e5dbf6785, 0x952ab45cfa97a0b3, 0xba756174393d88e0, 0xe912b9d1478ceb17, 0x91abb422ccb812ef, 0xb616a12b7fe617aa, 0xe39c49765fdf9d95, 0x8e41ade9fbebc27d, 0xb1d219647ae6b31c, 0xde469fbd99a05fe3, 0x8aec23d680043bee, 0xada72ccc20054aea, 0xd910f7ff28069da4, 0x87aa9aff79042287, 0xa99541bf57452b28, 0xd3fa922f2d1675f2, 0x847c9b5d7c2e09b7, 0xa59bc234db398c25, 0xcf02b2c21207ef2f, 0x8161afb94b44f57d, 0xa1ba1ba79e1632dc, 0xca28a291859bbf93, 0xfcb2cb35e702af78, 0x9defbf01b061adab, 0xc56baec21c7a1916, 0xf6c69a72a3989f5c, 0x9a3c2087a63f6399, 0xc0cb28a98fcf3c80, 0xf0fdf2d3f3c30b9f, 0x969eb7c47859e744, 0xbc4665b596706115, 0xeb57ff22fc0c795a, 0x9316ff75dd87cbd8, 0xb7dcbf5354e9bece, 0xe5d3ef282a242e82, 0x8fa475791a569d11, 0xb38d92d760ec4455, 0xe070f78d3927556b, 0x8c469ab843b89563, 0xaf58416654a6babb, 0xdb2e51bfe9d0696a, 0x88fcf317f22241e2, 0xab3c2fddeeaad25b, 0xd60b3bd56a5586f2, 0x85c7056562757457, 0xa738c6bebb12d16d, 0xd106f86e69d785c8, 0x82a45b450226b39d, 0xa34d721642b06084, 0xcc20ce9bd35c78a5, 0xff290242c83396ce, 0x9f79a169bd203e41, 0xc75809c42c684dd1, 0xf92e0c3537826146, 0x9bbcc7a142b17ccc, 0xc2abf989935ddbfe, 0xf356f7ebf83552fe, 0x98165af37b2153df, 0xbe1bf1b059e9a8d6, 0xeda2ee1c7064130c, 0x9485d4d1c63e8be8, 0xb9a74a0637ce2ee1, 0xe8111c87c5c1ba9a, 0x910ab1d4db9914a0, 0xb54d5e4a127f59c8, 0xe2a0b5dc971f303a, 0x8da471a9de737e24, 0xb10d8e1456105dad, 0xdd50f1996b947519, 0x8a5296ffe33cc930, 0xace73cbfdc0bfb7b, 0xd8210befd30efa5a, 0x8714a775e3e95c78, 0xa8d9d1535ce3b396, 0xd31045a8341ca07c, 0x83ea2b892091e44e, 0xa4e4b66b68b65d61, 0xce1de40642e3f4b9, 0x80d2ae83e9ce78f4, 0xa1075a24e4421731, 0xc94930ae1d529cfd, 0xfb9b7cd9a4a7443c, 0x9d412e0806e88aa6, 0xc491798a08a2ad4f, 0xf5b5d7ec8acb58a3, 0x9991a6f3d6bf1766, 0xbff610b0cc6edd3f, 0xeff394dcff8a948f, 0x95f83d0a1fb69cd9, 0xbb764c4ca7a44410, 0xea53df5fd18d5514, 0x92746b9be2f8552c, 0xb7118682dbb66a77, 0xe4d5e82392a40515, 0x8f05b1163ba6832d, 0xb2c71d5bca9023f8, 0xdf78e4b2bd342cf7, 0x8bab8eefb6409c1a, 0xae9672aba3d0c321, 0xda3c0f568cc4f3e9, 0x8865899617fb1871, 0xaa7eebfb9df9de8e, 0xd51ea6fa85785631, 0x8533285c936b35df, 0xa67ff273b8460357, 0xd01fef10a657842c, 0x8213f56a67f6b29c, 0xa298f2c501f45f43, 0xcb3f2f7642717713, 0xfe0efb53d30dd4d8, 0x9ec95d1463e8a507, 0xc67bb4597ce2ce49, 0xf81aa16fdc1b81db, 0x9b10a4e5e9913129, 0xc1d4ce1f63f57d73, 0xf24a01a73cf2dcd0, 0x976e41088617ca02, 0xbd49d14aa79dbc82, 0xec9c459d51852ba3, 0x93e1ab8252f33b46, 0xb8da1662e7b00a17, 0xe7109bfba19c0c9d, 0x906a617d450187e2, 0xb484f9dc9641e9db, 0xe1a63853bbd26451, 0x8d07e33455637eb3, 0xb049dc016abc5e60, 0xdc5c5301c56b75f7, 0x89b9b3e11b6329bb, 0xac2820d9623bf429, 0xd732290fbacaf134, 0x867f59a9d4bed6c0, 0xa81f301449ee8c70, 0xd226fc195c6a2f8c, 0x83585d8fd9c25db8, 0xa42e74f3d032f526, 0xcd3a1230c43fb26f, 0x80444b5e7aa7cf85, 0xa0555e361951c367, 0xc86ab5c39fa63441, 0xfa856334878fc151, 0x9c935e00d4b9d8d2, 0xc3b8358109e84f07, 0xf4a642e14c6262c9, 0x98e7e9cccfbd7dbe, 0xbf21e44003acdd2d, 0xeeea5d5004981478, 0x95527a5202df0ccb, 0xbaa718e68396cffe, 0xe950df20247c83fd, 0x91d28b7416cdd27e, 0xb6472e511c81471e, 0xe3d8f9e563a198e5, 0x8e679c2f5e44ff8f, 0xb201833b35d63f73, 0xde81e40a034bcf50, 0x8b112e86420f6192, 0xadd57a27d29339f6, 0xd94ad8b1c7380874, 0x87cec76f1c830549, 0xa9c2794ae3a3c69b, 0xd433179d9c8cb841, 0x849feec281d7f329, 0xa5c7ea73224deff3, 0xcf39e50feae16bf0, 0x81842f29f2cce376, 0xa1e53af46f801c53, 0xca5e89b18b602368, 0xfcf62c1dee382c42, 0x9e19db92b4e31ba9, 0xc5a05277621be294, 0xf70867153aa2db39, 0x9a65406d44a5c903, 0xc0fe908895cf3b44, 0xf13e34aabb430a15, 0x96c6e0eab509e64d, 0xbc789925624c5fe1, 0xeb96bf6ebadf77d9, 0x933e37a534cbaae8, 0xb80dc58e81fe95a1, 0xe61136f2227e3b0a, 0x8fcac257558ee4e6, 0xb3bd72ed2af29e20, 0xe0accfa875af45a8, 0x8c6c01c9498d8b89, 0xaf87023b9bf0ee6b, 0xdb68c2ca82ed2a06, 0x892179be91d43a44, 0xab69d82e364948d4 }; -static const int powers_ten_e[] = { -1203, -1200, -1196, -1193, -1190, -1186, -1183, -1180, -1176, -1173, -1170, -1166, -1163, -1160, -1156, -1153, -1150, -1146, -1143, -1140, -1136, -1133, -1130, -1127, -1123, -1120, -1117, -1113, -1110, -1107, -1103, -1100, -1097, -1093, -1090, -1087, -1083, -1080, -1077, -1073, -1070, -1067, -1063, -1060, -1057, -1053, -1050, -1047, -1043, -1040, -1037, -1034, -1030, -1027, -1024, -1020, -1017, -1014, -1010, -1007, -1004, -1000, -997, -994, -990, -987, -984, -980, -977, -974, -970, -967, -964, -960, -957, -954, -950, -947, -944, -940, -937, -934, -931, -927, -924, -921, -917, -914, -911, -907, -904, -901, -897, -894, -891, -887, -884, -881, -877, -874, -871, -867, -864, -861, -857, -854, -851, -847, -844, -841, -838, -834, -831, -828, -824, -821, -818, -814, -811, -808, -804, -801, -798, -794, -791, -788, -784, -781, -778, -774, -771, -768, -764, -761, -758, -754, -751, -748, -744, -741, -738, -735, -731, -728, -725, -721, -718, -715, -711, -708, -705, -701, -698, -695, -691, -688, -685, -681, -678, -675, -671, -668, -665, -661, -658, -655, -651, -648, -645, -642, -638, -635, -632, -628, -625, -622, -618, -615, -612, -608, -605, -602, -598, -595, -592, -588, -585, -582, -578, -575, -572, -568, -565, -562, -558, -555, -552, -549, -545, -542, -539, -535, -532, -529, -525, -522, -519, -515, -512, -509, -505, -502, -499, -495, -492, -489, -485, -482, -479, -475, -472, -469, -465, -462, -459, -455, -452, -449, -446, -442, -439, -436, -432, -429, -426, -422, -419, -416, -412, -409, -406, -402, -399, -396, -392, -389, -386, -382, -379, -376, -372, -369, -366, -362, -359, -356, -353, -349, -346, -343, -339, -336, -333, -329, -326, -323, -319, -316, -313, -309, -306, -303, -299, -296, -293, -289, -286, -283, -279, -276, -273, -269, -266, -263, -259, -256, -253, -250, -246, -243, -240, -236, -233, -230, -226, -223, -220, -216, -213, -210, -206, -203, -200, -196, -193, -190, -186, -183, -180, -176, -173, -170, -166, -163, -160, -157, -153, -150, -147, -143, -140, -137, -133, -130, -127, -123, -120, -117, -113, -110, -107, -103, -100, -97, -93, -90, -87, -83, -80, -77, -73, -70, -67, -63, -60, -57, -54, -50, -47, -44, -40, -37, -34, -30, -27, -24, -20, -17, -14, -10, -7, -4, 0, 3, 6, 10, 13, 16, 20, 23, 26, 30, 33, 36, 39, 43, 46, 49, 53, 56, 59, 63, 66, 69, 73, 76, 79, 83, 86, 89, 93, 96, 99, 103, 106, 109, 113, 116, 119, 123, 126, 129, 132, 136, 139, 142, 146, 149, 152, 156, 159, 162, 166, 169, 172, 176, 179, 182, 186, 189, 192, 196, 199, 202, 206, 209, 212, 216, 219, 222, 226, 229, 232, 235, 239, 242, 245, 249, 252, 255, 259, 262, 265, 269, 272, 275, 279, 282, 285, 289, 292, 295, 299, 302, 305, 309, 312, 315, 319, 322, 325, 328, 332, 335, 338, 342, 345, 348, 352, 355, 358, 362, 365, 368, 372, 375, 378, 382, 385, 388, 392, 395, 398, 402, 405, 408, 412, 415, 418, 422, 425, 428, 431, 435, 438, 441, 445, 448, 451, 455, 458, 461, 465, 468, 471, 475, 478, 481, 485, 488, 491, 495, 498, 501, 505, 508, 511, 515, 518, 521, 524, 528, 531, 534, 538, 541, 544, 548, 551, 554, 558, 561, 564, 568, 571, 574, 578, 581, 584, 588, 591, 594, 598, 601, 604, 608, 611, 614, 617, 621, 624, 627, 631, 634, 637, 641, 644, 647, 651, 654, 657, 661, 664, 667, 671, 674, 677, 681, 684, 687, 691, 694, 697, 701, 704, 707, 711, 714, 717, 720, 724, 727, 730, 734, 737, 740, 744, 747, 750, 754, 757, 760, 764, 767, 770, 774, 777, 780, 784, 787, 790, 794, 797, 800, 804, 807, 810, 813, 817, 820, 823, 827, 830, 833, 837, 840, 843, 847, 850, 853, 857, 860, 863, 867, 870, 873, 877, 880, 883, 887, 890, 893, 897, 900, 903, 907, 910, 913, 916, 920, 923, 926, 930, 933, 936, 940, 943, 946, 950, 953, 956, 960, 963, 966, 970, 973, 976, 980, 983, 986, 990, 993, 996, 1000, 1003, 1006, 1009, 1013, 1016, 1019, 1023, 1026, 1029, 1033, 1036, 1039, 1043, 1046, 1049, 1053, 1056, 1059, 1063, 1066, 1069, 1073, 1076 }; - -inline -diy_fp_t cached_power(int k) -{ - diy_fp_t res; - int index = 343 + k; - res.f = powers_ten[index]; - res.e = powers_ten_e[index]; - return res; -} - -// double - -typedef union { - double d; - uint64_t n; -} converter_t; - -inline -uint64_t double_to_uint64(double d) { converter_t tmp; tmp.d = d; return tmp.n; } - -inline -double uint64_to_double(uint64_t d64) { converter_t tmp; tmp.n = d64; return tmp.d; } - -constexpr int dp_significand_size = 52; -constexpr int dp_exponent_bias = (0x3FF + dp_significand_size); -constexpr int dp_min_exponent = (-dp_exponent_bias); -constexpr uint64_t dp_exponent_mask = 0x7FF0000000000000; -constexpr uint64_t dp_significand_mask = 0x000FFFFFFFFFFFFF; -constexpr uint64_t dp_hidden_bit = 0x0010000000000000; - -inline -diy_fp_t normalize_diy_fp(diy_fp_t in) -{ - diy_fp_t res = in; - /* Normalize now */ - /* the original number could have been a denormal. */ - while (!(res.f & dp_hidden_bit)) - { - res.f <<= 1; - res.e--; - } - /* do the final shifts in one go. Don't forget the hidden bit (the '-1') */ - res.f <<= (diy_significand_size - dp_significand_size - 1); - res.e = res.e - (diy_significand_size - dp_significand_size - 1); - return res; -} - -inline -diy_fp_t double2diy_fp(double d) -{ - uint64_t d64 = double_to_uint64(d); - int biased_e = (d64 & dp_exponent_mask) >> dp_significand_size; - uint64_t significand = (d64 & dp_significand_mask); - diy_fp_t res; - if (biased_e != 0) - { - res.f = significand + dp_hidden_bit; - res.e = biased_e - dp_exponent_bias; - } - else - { - res.f = significand; - res.e = dp_min_exponent + 1; - } - return res; -} - -inline -diy_fp_t normalize_boundary(diy_fp_t in) -{ - diy_fp_t res = in; - /* Normalize now */ - /* the original number could have been a denormal. */ - while (!(res.f & (dp_hidden_bit << 1))) - { - res.f <<= 1; - res.e--; - } - /* do the final shifts in one go. Don't forget the hidden bit (the '-1') */ - res.f <<= (diy_significand_size - dp_significand_size - 2); - res.e = res.e - (diy_significand_size - dp_significand_size - 2); - return res; -} - -inline -void normalized_boundaries(double d, diy_fp_t *out_m_minus, diy_fp_t *out_m_plus) -{ - diy_fp_t v = double2diy_fp(d); - diy_fp_t pl, mi; - bool significand_is_zero = v.f == dp_hidden_bit; - pl.f = (v.f << 1) + 1; pl.e = v.e - 1; - pl = normalize_boundary(pl); - if (significand_is_zero) - { - mi.f = (v.f << 2) - 1; - mi.e = v.e - 2; - } else - { - mi.f = (v.f << 1) - 1; - mi.e = v.e - 1; - } - mi.f <<= mi.e - pl.e; - mi.e = pl.e; - *out_m_plus = pl; - *out_m_minus = mi; -} - -inline -double random_double() -{ - uint64_t tmp = 0; - int i; - for (i = 0; i < 8; i++) - { - tmp <<= 8; - tmp += rand() % 256; - } - return uint64_to_double(tmp); -} - -// fast_exponent -template -void fill_exponent(int K, Result& result) -{ - if (K < 0) - { - result.push_back('-'); - K = -K; - } - else - { - result.push_back('+'); // compatibility with sprintf - } - if (K >= 100) - { - result.push_back((char)('0' + K / 100)); K %= 100; - result.push_back((char)('0' + K / 10)); K %= 10; - result.push_back((char)('0' + K)); - } - else if (K >= 10) - { - result.push_back((char)('0' + K / 10)); K %= 10; - result.push_back((char)('0' + K)); - } - else - { - result.push_back('0'); // compatibility with sprintf - result.push_back((char)('0' + K)); - } -} - -template -void prettify_string(const char *buffer, size_t length, int k, int min_exp, int max_exp, Result& result) -{ - int nb_digits = (int)length; - int offset; - /* v = buffer * 10^k - kk is such that 10^(kk-1) <= v < 10^kk - this way kk gives the position of the decimal point. - */ - int kk = nb_digits + k; - - if (nb_digits <= kk && kk <= max_exp) - { - /* the first digits are already in. Add some 0s and call it a day. */ - /* the max_exp is a personal choice. Only 16 digits could possibly be relevant. - * Basically we want to print 12340000000 rather than 1234.0e7 or 1.234e10 */ - for (int i = 0; i < nb_digits; ++i) - { - result.push_back(buffer[i]); - } - for (int i = nb_digits; i < kk; ++i) - { - result.push_back('0'); - } - result.push_back('.'); - result.push_back('0'); - } - else if (0 < kk && kk <= max_exp) - { - /* comma number. Just insert a '.' at the correct location. */ - for (int i = 0; i < kk; ++i) - { - result.push_back(buffer[i]); - } - result.push_back('.'); - for (int i = kk; i < nb_digits; ++i) - { - result.push_back(buffer[i]); - } - } - else if (min_exp < kk && kk <= 0) - { - offset = 2 - kk; - - result.push_back('0'); - result.push_back('.'); - for (int i = 2; i < offset; ++i) - result.push_back('0'); - for (int i = 0; i < nb_digits; ++i) - { - result.push_back(buffer[i]); - } - } - else if (nb_digits == 1) - { - result.push_back(buffer[0]); - result.push_back('e'); - fill_exponent(kk - 1, result); - } - else - { - result.push_back(buffer[0]); - result.push_back('.'); - for (int i = 1; i < nb_digits; ++i) - { - result.push_back(buffer[i]); - } - result.push_back('e'); - fill_exponent(kk - 1, result); - } -} - -// grisu3 - -inline -bool round_weed(char *buffer, int len, - uint64_t wp_W, uint64_t Delta, - uint64_t rest, uint64_t ten_kappa, - uint64_t ulp) -{ - uint64_t wp_Wup = wp_W - ulp; - uint64_t wp_Wdown = wp_W + ulp; - while (rest < wp_Wup && /// round1 - Delta - rest >= ten_kappa && - (rest + ten_kappa < wp_Wup || /// closer - wp_Wup - rest >= rest + ten_kappa - wp_Wup)) - { - buffer[len - 1]--; rest += ten_kappa; - } - if (rest < wp_Wdown && /// round2 - Delta - rest >= ten_kappa && - (rest + ten_kappa < wp_Wdown || - wp_Wdown - rest > rest + ten_kappa - wp_Wdown)) return 0; - return 2 * ulp <= rest && rest <= Delta - 4 * ulp; /// weed -} - -inline -bool digit_gen(diy_fp_t Wm, diy_fp_t W, diy_fp_t Wp, - char *buffer, int *len, int *K) -{ - const uint32_t TEN2 = 100; - - uint32_t div, p1; uint64_t p2, tmp, unit = 1; - int d, kappa; - diy_fp_t one, wp_W, Delta; - Delta = minus(Wp, Wm); Delta.f += 2 * unit; - wp_W = minus(Wp, W); wp_W.f += unit; - one.f = ((uint64_t)1) << -Wp.e; one.e = Wp.e; - p1 = static_cast((Wp.f + 1) >> -one.e); - p2 = (Wp.f + 1) & (one.f - 1); - *len = 0; kappa = 3; div = TEN2; - while (kappa > 0) - { - d = p1 / div; - if (d || *len) buffer[(*len)++] = (char)('0' + d); - p1 %= div; kappa--; - tmp = (((uint64_t)p1) << -one.e) + p2; - if (tmp < Delta.f) - { - *K += kappa; - return round_weed(buffer, *len, wp_W.f, Delta.f, tmp, - ((uint64_t)div) << -one.e, unit); - } - div /= 10; - } - while (1) - { - p2 *= 10; Delta.f *= 10; unit *= 10; - d = static_cast(p2 >> -one.e); - if (d || *len) buffer[(*len)++] = (char)('0' + d); - p2 &= one.f - 1; kappa--; - if (p2 < Delta.f) - { - *K += kappa; - return round_weed(buffer, *len, wp_W.f * unit, Delta.f, p2, - one.f, unit); - } - } -} - -inline -bool grisu3(double v, char *buffer, int *length, int *K) -{ - diy_fp_t w_m, w_p; - int q = 64, alpha = -59, gamma = -56; - normalized_boundaries(v, &w_m, &w_p); - diy_fp_t w = normalize_diy_fp(double2diy_fp(v)); - int mk = k_comp(w_p.e + q, alpha, gamma); - diy_fp_t c_mk = cached_power(mk); - diy_fp_t W = multiply(w, c_mk); - diy_fp_t Wp = multiply(w_p, c_mk); - diy_fp_t Wm = multiply(w_m, c_mk); - *K = -mk; - bool result = digit_gen(Wm, W, Wp, buffer, length, K); - buffer[*length] = 0; - return result; -} - -}} // namespace detail namespace jsoncons - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/heap_only_string.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/heap_only_string.hpp deleted file mode 100644 index a90d863d49..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/heap_only_string.hpp +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_DETAIL_HEAP_ONLY_STRING_HPP -#define JSONCONS_DETAIL_HEAP_ONLY_STRING_HPP - -#include -#include -#include -#include -#include // std::memcpy -#include // std::allocator -#include - -namespace jsoncons { namespace detail { - -template -class heap_only_string_base -{ - Allocator allocator_; -public: - Allocator& get_allocator() - { - return allocator_; - } - - const Allocator& get_allocator() const - { - return allocator_; - } -protected: - heap_only_string_base(const Allocator& allocator) - : allocator_(allocator) - { - } - - ~heap_only_string_base() {} -}; - -template -class heap_only_string_factory; - -template -class heap_only_string : public heap_only_string_base -{ - typedef typename std::allocator_traits::template rebind_alloc allocator_type; - typedef std::allocator_traits allocator_traits_type; - typedef typename allocator_traits_type::pointer pointer; - - friend class heap_only_string_factory; -public: - typedef CharT char_type; - typedef heap_only_string value_type; - - ~heap_only_string() {} - - const char_type* c_str() const { return to_plain_pointer(p_); } - const char_type* data() const { return to_plain_pointer(p_); } - size_t length() const { return length_; } - - using heap_only_string_base::get_allocator; - - - friend std::basic_ostream& operator<<(std::basic_ostream& os, const heap_only_string& s) - { - os.write(s.data(),s.length()); - return os; - } -private: - heap_only_string() - : heap_only_string_base(Allocator()) - { - - } - heap_only_string(const Allocator& allocator) - : heap_only_string_base(allocator) - { - - } - - pointer p_; - size_t length_; - - heap_only_string(const heap_only_string&) = delete; - heap_only_string& operator=(const heap_only_string&) = delete; - -}; - -template -class heap_only_string_factory -{ - typedef CharT char_type; - typedef typename std::allocator_traits::template rebind_alloc byte_allocator_type; - typedef std::allocator_traits byte_allocator_traits_type; - typedef typename byte_allocator_traits_type::pointer byte_pointer; - typedef typename heap_only_string::pointer pointer; -public: - - typedef typename std::allocator_traits::template rebind_alloc> string_allocator_type; - typedef std::allocator_traits string_allocator_traits_type; - typedef typename string_allocator_traits_type::pointer string_pointer; - - typedef heap_only_string* raw_string_pointer_type; - - struct string_storage - { - heap_only_string data; - char_type c[1]; - }; - typedef typename std::aligned_storage::type storage_type; - - static size_t aligned_size(size_t n) - { - return sizeof(storage_type) + n; - } -public: - static string_pointer create(const char_type* s, size_t length) - { - return create(s, length, Allocator()); - } - static string_pointer create(const char_type* s, size_t length, const Allocator& allocator) - { - size_t mem_size = aligned_size(length*sizeof(char_type)); - - byte_allocator_type alloc(allocator); - byte_pointer ptr = alloc.allocate(mem_size); - - char* storage = to_plain_pointer(ptr); - raw_string_pointer_type ps = new(storage)heap_only_string(alloc); - auto psa = reinterpret_cast(storage); - - CharT* p = new(&psa->c)char_type[length + 1]; - std::memcpy(p, s, length*sizeof(char_type)); - p[length] = 0; - ps->p_ = std::pointer_traits::pointer_to(*p); - ps->length_ = length; - return std::pointer_traits::pointer_to(*ps); - } - - static void destroy(string_pointer ptr) - { - raw_string_pointer_type rawp = to_plain_pointer(ptr); - char* p = reinterpret_cast(rawp); - size_t mem_size = aligned_size(ptr->length_*sizeof(char_type)); - byte_allocator_type alloc(ptr->get_allocator()); - alloc.deallocate(p,mem_size); - } -}; - - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/parse_number.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/parse_number.hpp deleted file mode 100644 index 221bc4512c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/parse_number.hpp +++ /dev/null @@ -1,526 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_DETAIL_PARSE_NUMBER_HPP -#define JSONCONS_DETAIL_PARSE_NUMBER_HPP - -#include -#include -#include -#include -#include -#include // std::numeric_limits -#include // std::enable_if -#include -#include -#include - -namespace jsoncons { namespace detail { - -enum class to_integer_errc : uint8_t {ok=0,overflow,invalid_digit}; - -template -struct to_integer_result -{ - T value; - to_integer_errc ec; -}; - -class to_integer_error_category_impl - : public std::error_category -{ -public: - const char* name() const noexcept override - { - return "jsoncons/to_integer"; - } - std::string message(int ev) const override - { - switch (static_cast(ev)) - { - case to_integer_errc::overflow: - return "Integer overflow"; - case to_integer_errc::invalid_digit: - return "Invalid digit"; - default: - return "Unknown to_integer error"; - } - } -}; - -inline -const std::error_category& to_integer_error_category() -{ - static to_integer_error_category_impl instance; - return instance; -} - -inline -std::error_code make_error_code(to_integer_errc e) -{ - return std::error_code(static_cast(e),to_integer_error_category()); -} - -template -bool is_integer(const CharT* s, size_t length) -{ - const CharT* end = s + length; - if (s == end) - { - return false; - } - if (*s == '-') - { - ++s; - } - if (s == end) - { - return false; - } - for (;s < end; ++s) - { - if (!(*s >= '0' && *s <= '9')) - { - return false; - } - } - return true; -} - -template -bool is_uinteger(const CharT* s, size_t length) -{ - const CharT* end = s + length; - if (s == end) - { - return false; - } - for (;s < end; ++s) - { - if (!(*s >= '0' && *s <= '9')) - { - return false; - } - } - return true; -} - -// Precondition: s satisfies - -// digit -// digit1-digits -// - digit -// - digit1-digits - -template -typename std::enable_if::value && std::is_signed::value,to_integer_result>::type -to_integer(const CharT* s, size_t length) -{ - static_assert(std::numeric_limits::is_specialized, "Integer type not specialized"); - JSONCONS_ASSERT(length > 0); - - to_integer_errc ec{}; - - T n = 0; - const CharT* end = s + length; - if (*s == '-') - { - static const T min_value = (std::numeric_limits::lowest)(); - static const T min_value_div_10 = min_value / 10; - ++s; - for (; s < end; ++s) - { - T x = *s - '0'; - if (n < min_value_div_10) - { - ec = to_integer_errc::overflow; - break; - } - n = n * 10; - if (n < min_value + x) - { - ec = to_integer_errc::overflow; - break; - } - - n -= x; - } - } - else - { - static const T max_value = (std::numeric_limits::max)(); - static const T max_value_div_10 = max_value / 10; - for (; s < end; ++s) - { - T x = *s - '0'; - if (n > max_value_div_10) - { - ec = to_integer_errc::overflow; - break; - } - n = n * 10; - if (n > max_value - x) - { - ec = to_integer_errc::overflow; - break; - } - - n += x; - } - } - - return to_integer_result({n,ec}); -} - -// Precondition: s satisfies - -// digit -// digit1-digits -// - digit -// - digit1-digits - -template -typename std::enable_if::value && !std::is_signed::value,to_integer_result>::type -to_integer(const CharT* s, size_t length) -{ - static_assert(std::numeric_limits::is_specialized, "Integer type not specialized"); - JSONCONS_ASSERT(length > 0); - - T n = 0; - to_integer_errc ec{}; - - const CharT* end = s + length; - - static const T max_value = (std::numeric_limits::max)(); - static const T max_value_div_10 = max_value / 10; - for (; s < end; ++s) - { - T x = *s - '0'; - if (n > max_value_div_10) - { - ec = to_integer_errc::overflow; - break; - } - n = n * 10; - if (n > max_value - x) - { - ec = to_integer_errc::overflow; - break; - } - - n += x; - } - - return to_integer_result({ n,ec }); -} - -// base16_to_integer - -template -typename std::enable_if::value && std::is_signed::value,to_integer_result>::type -base16_to_integer(const CharT* s, size_t length) -{ - static_assert(std::numeric_limits::is_specialized, "Integer type not specialized"); - JSONCONS_ASSERT(length > 0); - - T n = 0; - to_integer_errc ec{}; - const CharT* end = s + length; - if (*s == '-') - { - static const T min_value = (std::numeric_limits::lowest)(); - static const T min_value_div_16 = min_value / 16; - ++s; - for (; s < end; ++s) - { - CharT c = *s; - T x = 0; - switch (c) - { - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - x = c - '0'; - break; - case 'a':case 'b':case 'c':case 'd':case 'e':case 'f': - x = c - ('a' - 10); - break; - case 'A':case 'B':case 'C':case 'D':case 'E':case 'F': - x = c - ('A' - 10); - break; - default: - throw std::runtime_error("Invalid hex digit"); - } - if (n < min_value_div_16) - { - ec = to_integer_errc::overflow; - break; - } - n = n * 16; - if (n < min_value + x) - { - ec = to_integer_errc::overflow; - break; - } - n -= x; - } - } - else - { - static const T max_value = (std::numeric_limits::max)(); - static const T max_value_div_16 = max_value / 16; - for (; s < end; ++s) - { - CharT c = *s; - T x = 0; - switch (c) - { - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - x = c - '0'; - break; - case 'a':case 'b':case 'c':case 'd':case 'e':case 'f': - x = c - ('a' - 10); - break; - case 'A':case 'B':case 'C':case 'D':case 'E':case 'F': - x = c - ('A' - 10); - break; - default: - throw std::runtime_error("Invalid hex digit"); - } - if (n > max_value_div_16) - { - ec = to_integer_errc::overflow; - break; - } - n = n * 16; - if (n > max_value - x) - { - ec = to_integer_errc::overflow; - break; - } - - n += x; - } - } - - return to_integer_result({ n,ec }); -} - -template -typename std::enable_if::value && !std::is_signed::value,to_integer_result>::type -base16_to_integer(const CharT* s, size_t length) -{ - static_assert(std::numeric_limits::is_specialized, "Integer type not specialized"); - JSONCONS_ASSERT(length > 0); - - T n = 0; - to_integer_errc ec{}; - const CharT* end = s + length; - - static const T max_value = (std::numeric_limits::max)(); - static const T max_value_div_16 = max_value / 16; - for (; s < end; ++s) - { - CharT c = *s; - T x = *s; - switch (c) - { - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - x = c - '0'; - break; - case 'a':case 'b':case 'c':case 'd':case 'e':case 'f': - x = c - ('a' - 10); - break; - case 'A':case 'B':case 'C':case 'D':case 'E':case 'F': - x = c - ('A' - 10); - break; - default: - throw std::runtime_error("Invalid hex digit"); - } - if (n > max_value_div_16) - { - ec = to_integer_errc::overflow; - break; - } - n = n * 16; - if (n > max_value - x) - { - ec = to_integer_errc::overflow; - break; - } - - n += x; - } - - return to_integer_result({ n,ec }); -} - -#if defined(JSONCONS_HAS_MSC__STRTOD_L) - -class string_to_double -{ -private: - _locale_t locale_; -public: - string_to_double() - { - locale_ = _create_locale(LC_NUMERIC, "C"); - } - ~string_to_double() - { - _free_locale(locale_); - } - - char get_decimal_point() const - { - return '.'; - } - - template - typename std::enable_if::value,double>::type - operator()(const CharT* s, size_t) const - { - CharT *end = nullptr; - double val = _strtod_l(s, &end, locale_); - if (s == end) - { - JSONCONS_THROW(json_runtime_error("Convert string to double failed")); - } - return val; - } - - template - typename std::enable_if::value,double>::type - operator()(const CharT* s, size_t) const - { - CharT *end = nullptr; - double val = _wcstod_l(s, &end, locale_); - if (s == end) - { - JSONCONS_THROW(json_runtime_error("Convert string to double failed")); - } - return val; - } -private: - // noncopyable and nonmoveable - string_to_double(const string_to_double&) = delete; - string_to_double& operator=(const string_to_double&) = delete; -}; - -#elif defined(JSONCONS_HAS_STRTOLD_L) - -class string_to_double -{ -private: - locale_t locale_; -public: - string_to_double() - { - locale_ = newlocale(LC_ALL_MASK, "C", (locale_t) 0); - } - ~string_to_double() - { - freelocale(locale_); - } - - char get_decimal_point() const - { - return '.'; - } - - template - typename std::enable_if::value,double>::type - operator()(const CharT* s, size_t length) const - { - char *end = nullptr; - double val = strtold_l(s, &end, locale_); - if (s == end) - { - JSONCONS_THROW(json_runtime_error("Convert string to double failed")); - } - return val; - } - - template - typename std::enable_if::value,double>::type - operator()(const CharT* s, size_t length) const - { - CharT *end = nullptr; - double val = wcstold_l(s, &end, locale_); - if (s == end) - { - JSONCONS_THROW(json_runtime_error("Convert string to double failed")); - } - return val; - } - -private: - // noncopyable and nonmoveable - string_to_double(const string_to_double& fr) = delete; - string_to_double& operator=(const string_to_double& fr) = delete; -}; - -#else -class string_to_double -{ -private: - std::vector buffer_; - char decimal_point_; -public: - string_to_double() - : buffer_() - { - struct lconv * lc = localeconv(); - if (lc != nullptr && lc->decimal_point[0] != 0) - { - decimal_point_ = lc->decimal_point[0]; - } - else - { - decimal_point_ = '.'; - } - buffer_.reserve(100); - } - - char get_decimal_point() const - { - return decimal_point_; - } - - template - typename std::enable_if::value,double>::type - operator()(const CharT* s, size_t /*length*/) const - { - CharT *end = nullptr; - double val = strtod(s, &end); - if (s == end) - { - JSONCONS_THROW(json_runtime_error("Convert string to double failed")); - } - return val; - } - - template - typename std::enable_if::value,double>::type - operator()(const CharT* s, size_t /*length*/) const - { - CharT *end = nullptr; - double val = wcstod(s, &end); - if (s == end) - { - JSONCONS_THROW(json_runtime_error("Convert string to double failed")); - } - return val; - } - -private: - // noncopyable and nonmoveable - string_to_double(const string_to_double& fr) = delete; - string_to_double& operator=(const string_to_double& fr) = delete; -}; -#endif - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/print_number.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/print_number.hpp deleted file mode 100644 index 8fa870d3f7..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/print_number.hpp +++ /dev/null @@ -1,344 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_DETAIL_PRINT_NUMBER_HPP -#define JSONCONS_DETAIL_PRINT_NUMBER_HPP - -#include -#include -#include -#include -#include // std::numeric_limits -#include -#include -#include -#include -#include - -namespace jsoncons { namespace detail { - -// print_integer - -template -size_t print_integer(int64_t value, Result& result) -{ - typedef typename Result::value_type char_type; - - size_t count = 0; - - char_type buf[255]; - uint64_t u = (value < 0) ? static_cast(-value) : static_cast(value); - char_type* p = buf; - do - { - *p++ = static_cast(48 + u%10); - } - while (u /= 10); - count += (p-buf); - if (value < 0) - { - result.push_back('-'); - ++count; - } - while (--p >= buf) - { - result.push_back(*p); - } - - return count; -} - -// print_uinteger - -template -size_t print_uinteger(uint64_t value, Result& result) -{ - typedef typename Result::value_type char_type; - - size_t count = 0; - - char_type buf[255]; - char_type* p = buf; - do - { - *p++ = static_cast(48 + value % 10); - } - while (value /= 10); - count += (p-buf); - while (--p >= buf) - { - result.push_back(*p); - } - return count; -} - -// print_integer - -template -size_t integer_to_hex_string(int64_t value, Result& result) -{ - typedef typename Result::value_type char_type; - - size_t count = 0; - - char_type buf[255]; - uint64_t u = (value < 0) ? static_cast(-value) : static_cast(value); - char_type* p = buf; - do - { - *p++ = to_hex_character(u%16); - } - while (u /= 16); - count += (p-buf); - if (value < 0) - { - result.push_back('-'); - ++count; - } - while (--p >= buf) - { - result.push_back(*p); - } - - return count; -} - -// print_uinteger - -template -size_t uinteger_to_hex_string(uint64_t value, Result& result) -{ - typedef typename Result::value_type char_type; - - size_t count = 0; - - char_type buf[255]; - char_type* p = buf; - do - { - *p++ = to_hex_character(value % 16); - } - while (value /= 16); - count += (p-buf); - while (--p >= buf) - { - result.push_back(*p); - } - return count; -} - -// print_double - -template -void dump_buffer(const char* buffer, size_t length, char decimal_point, Result& result) -{ - const char* sbeg = buffer; - const char* send = sbeg + length; - - if (sbeg != send) - { - bool needs_dot = true; - for (const char* q = sbeg; q < send; ++q) - { - switch (*q) - { - case '-':case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8':case '9':case '+': - result.push_back(*q); - break; - case 'e': - case 'E': - result.push_back('e'); - needs_dot = false; - break; - default: - if (*q == decimal_point) - { - needs_dot = false; - result.push_back('.'); - } - break; - } - } - if (needs_dot) - { - result.push_back('.'); - result.push_back('0'); - needs_dot = true; - } - } -} - -template -bool dtoa(double val, char decimal_point, Result& result, std::false_type) -{ - if (val == 0) - { - result.push_back('0'); - result.push_back('.'); - result.push_back('0'); - return true; - } - - jsoncons::detail::string_to_double to_double_; - - char buffer[100]; - int precision = std::numeric_limits::digits10; - int length = snprintf(buffer, sizeof(buffer), "%1.*g", precision, val); - if (length < 0) - { - return false; - } - if (to_double_(buffer,sizeof(buffer)) != val) - { - const int precision2 = std::numeric_limits::max_digits10; - length = snprintf(buffer, sizeof(buffer), "%1.*g", precision2, val); - if (length < 0) - { - return false; - } - } - dump_buffer(buffer, length, decimal_point, result); - return true; -} - -template -bool dtoa(double v, char decimal_point, Result& result, std::true_type) -{ - if (v == 0) - { - result.push_back('0'); - result.push_back('.'); - result.push_back('0'); - return true; - } - - int length = 0; - int k; - - char buffer[100]; - - double u = std::signbit(v) ? -v : v; - if (jsoncons::detail::grisu3(u, buffer, &length, &k)) - { - if (std::signbit(v)) - { - result.push_back('-'); - } - // min exp: -4 is consistent with sprintf - // max exp: std::numeric_limits::max_digits10 - jsoncons::detail::prettify_string(buffer, length, k, -4, std::numeric_limits::max_digits10, result); - return true; - } - else - { - return dtoa(v, decimal_point, result, std::false_type()); - } -} - -template -bool dtoa(double v, char decimal_point, Result& result) -{ - return dtoa(v, decimal_point, result, std::integral_constant::is_iec559>()); -} - -class print_double -{ -private: - string_to_double to_double_; - floating_point_options override_; - char decimal_point_; -public: - print_double(const floating_point_options& options) - : override_(options) - { -#if !defined(JSONCONS_NO_LOCALECONV) - struct lconv * lc = localeconv(); - if (lc != nullptr && lc->decimal_point[0] != 0) - { - decimal_point_ = lc->decimal_point[0]; - } - else -#endif - { - decimal_point_ = '.'; - } - } - - template - size_t operator()(double val, Result& result) - { - size_t count = 0; - - chars_format format = override_.format() != chars_format() ? override_.format() : chars_format::general; - - int decimal_places; - if (override_.decimal_places() != 0) - { - decimal_places = override_.decimal_places(); - } - else - { - format = chars_format::general; - decimal_places = 0; - } - - char number_buffer[200]; - int length = 0; - - switch (format) - { - case chars_format::fixed: - { - length = snprintf(number_buffer, sizeof(number_buffer), "%1.*f", decimal_places, val); - if (length < 0) - { - JSONCONS_THROW(json_runtime_error("print_double failed.")); - } - dump_buffer(number_buffer, length, decimal_point_, result); - } - break; - case chars_format::scientific: - { - length = snprintf(number_buffer, sizeof(number_buffer), "%1.*e", decimal_places, val); - if (length < 0) - { - JSONCONS_THROW(json_runtime_error("print_double failed.")); - } - dump_buffer(number_buffer, length, decimal_point_, result); - } - break; - case chars_format::general: - { - if (override_.precision() != 0) - { - int precision = override_.precision(); - length = snprintf(number_buffer, sizeof(number_buffer), "%1.*g", precision, val); - if (length < 0) - { - JSONCONS_THROW(json_runtime_error("print_double failed.")); - } - dump_buffer(number_buffer, length, decimal_point_, result); - } - else - { - if (!dtoa(val, decimal_point_, result)) - { - JSONCONS_THROW(json_runtime_error("print_double failed.")); - } - } - break; - } - default: - JSONCONS_THROW(json_runtime_error("print_double failed.")); - break; - } - return count; - } -}; - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/string_view.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/string_view.hpp deleted file mode 100644 index e85abff6b8..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/string_view.hpp +++ /dev/null @@ -1,525 +0,0 @@ -// Copyright 2019 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_STRING_VIEW_HPP -#define JSONCONS_STRING_VIEW_HPP - -#include -#include -#include -#include -#include -#include // std::find, std::min, std::reverse -#include -#include -#include -#include -#include // std::basic_istream - -namespace jsoncons { namespace detail { - -template > -class basic_string_view -{ -private: - const CharT* data_; - size_t length_; -public: - typedef CharT value_type; - typedef const CharT& const_reference; - typedef Traits traits_type; - typedef std::size_t size_type; - static const size_type npos = size_type(-1); - typedef const CharT* const_iterator; - typedef const CharT* iterator; - typedef std::reverse_iterator const_reverse_iterator; - - basic_string_view() - : data_(nullptr), length_(0) - { - } - basic_string_view(const CharT* data, size_t length) - : data_(data), length_(length) - { - } - basic_string_view(const CharT* data) - : data_(data), length_(Traits::length(data)) - { - } - basic_string_view(const basic_string_view& other) = default; - - template - basic_string_view(const std::basic_string& s) - : data_(s.data()), length_(s.length()) - { - } - - template - explicit operator std::basic_string() const - { - return std::basic_string(data_,length_); - } - - // iterator support - const_iterator begin() const noexcept - { - return data_; - } - const_iterator end() const noexcept - { - return data_ + length_; - } - const_iterator cbegin() const noexcept - { - return data_; - } - const_iterator cend() const noexcept - { - return data_ + length_; - } - const_reverse_iterator rbegin() const noexcept - { - return const_reverse_iterator(end()); - } - const_reverse_iterator rend() const noexcept - { - return const_reverse_iterator(begin()); - } - const_reverse_iterator crbegin() const noexcept - { - return const_reverse_iterator(end()); - } - const_reverse_iterator crend() const noexcept - { - return const_reverse_iterator(begin()); - } - - // capacity - - size_t size() const - { - return length_; - } - - size_t length() const - { - return length_; - } - size_type max_size() const noexcept - { - return length_; - } - bool empty() const noexcept - { - return length_ == 0; - } - - // element access - - const_reference operator[](size_type pos) const - { - return data_[pos]; - } - - const_reference at(size_t pos) const - { - if (pos >= length_) - { - throw std::out_of_range("pos exceeds length"); - } - return data_[pos]; - } - - const_reference front() const - { - return data_[0]; - } - const_reference back() const - { - return data_[length_-1]; - } - - const CharT* data() const - { - return data_; - } - - // string operations - - basic_string_view substr(size_type pos, size_type n=npos) const - { - if (pos > length_) - { - throw std::out_of_range("pos exceeds size"); - } - if (n == npos || pos + n > length_) - { - n = length_ - pos; - } - return basic_string_view(data_ + pos, n); - } - - int compare(const basic_string_view& s) const - { - const int rc = Traits::compare(data_, s.data_, (std::min)(length_, s.length_)); - return rc != 0 ? rc : (length_ == s.length_ ? 0 : length_ < s.length_ ? -1 : 1); - } - - int compare(const CharT* data) const - { - const size_t length = Traits::length(data); - const int rc = Traits::compare(data_, data, (std::min)(length_, length)); - return rc != 0 ? rc : (length_ == length? 0 : length_ < length? -1 : 1); - } - - template - int compare(const std::basic_string& s) const - { - const int rc = Traits::compare(data_, s.data(), (std::min)(length_, s.length())); - return rc != 0 ? rc : (length_ == s.length() ? 0 : length_ < s.length() ? -1 : 1); - } - - size_type find(basic_string_view s, size_type pos = 0) const noexcept - { - if (pos > length_) - { - return npos; - } - if (s.length_ == 0) - { - return pos; - } - const_iterator it = std::search(cbegin() + pos, cend(), - s.cbegin(), s.cend(), Traits::eq); - return it == cend() ? npos : std::distance(cbegin(), it); - } - size_type find(CharT ch, size_type pos = 0) const noexcept - { - return find(basic_string_view(&ch, 1), pos); - } - size_type find(const CharT* s, size_type pos, size_type n) const noexcept - { - return find(basic_string_view(s, n), pos); - } - size_type find(const CharT* s, size_type pos = 0) const noexcept - { - return find(basic_string_view(s), pos); - } - - size_type rfind(basic_string_view s, size_type pos = npos) const noexcept - { - if (length_ < s.length_) - { - return npos; - } - if (pos > length_ - s.length_) - { - pos = length_ - s.length_; - } - if (s.length_ == 0) - { - return pos; - } - for (const CharT* p = data_ + pos; true; --p) - { - if (Traits::compare(p, s.data_, s.length_) == 0) - { - return p - data_; - } - if (p == data_) - { - return npos; - } - }; - } - size_type rfind(CharT ch, size_type pos = npos) const noexcept - { - return rfind(basic_string_view(&ch, 1), pos); - } - size_type rfind(const CharT* s, size_type pos, size_type n) const noexcept - { - return rfind(basic_string_view(s, n), pos); - } - size_type rfind(const CharT* s, size_type pos = npos) const noexcept - { - return rfind(basic_string_view(s), pos); - } - - size_type find_first_of(basic_string_view s, size_type pos = 0) const noexcept - { - if (pos >= length_ || s.length_ == 0) - { - return npos; - } - const_iterator it = std::find_first_of - (cbegin() + pos, cend(), s.cbegin(), s.cend(), Traits::eq); - return it == cend() ? npos : std::distance (cbegin(), it); - } - size_type find_first_of(CharT ch, size_type pos = 0) const noexcept - { - return find_first_of(basic_string_view(&ch, 1), pos); - } - size_type find_first_of(const CharT* s, size_type pos, size_type n) const noexcept - { - return find_first_of(basic_string_view(s, n), pos); - } - size_type find_first_of(const CharT* s, size_type pos = 0) const noexcept - { - return find_first_of(basic_string_view(s), pos); - } - - size_type find_last_of(basic_string_view s, size_type pos = npos) const noexcept - { - if (s.length_ == 0) - { - return npos; - } - if (pos >= length_) - { - pos = 0; - } - else - { - pos = length_ - (pos+1); - } - const_reverse_iterator it = std::find_first_of - (crbegin() + pos, crend(), s.cbegin(), s.cend(), Traits::eq); - return it == crend() ? npos : (length_ - 1 - std::distance(crbegin(), it)); - } - size_type find_last_of(CharT ch, size_type pos = npos) const noexcept - { - return find_last_of(basic_string_view(&ch, 1), pos); - } - size_type find_last_of(const CharT* s, size_type pos, size_type n) const noexcept - { - return find_last_of(basic_string_view(s, n), pos); - } - size_type find_last_of(const CharT* s, size_type pos = npos) const noexcept - { - return find_last_of(basic_string_view(s), pos); - } - - size_type find_first_not_of(basic_string_view s, size_type pos = 0) const noexcept - { - if (pos >= length_) - return npos; - if (s.length_ == 0) - return pos; - - const_iterator it = cend(); - for (auto p = cbegin()+pos; p != cend(); ++p) - { - if (Traits::find(s.data_, s.length_, *p) == 0) - { - it = p; - break; - } - } - return it == cend() ? npos : std::distance (cbegin(), it); - } - size_type find_first_not_of(CharT ch, size_type pos = 0) const noexcept - { - return find_first_not_of(basic_string_view(&ch, 1), pos); - } - size_type find_first_not_of(const CharT* s, size_type pos, size_type n) const noexcept - { - return find_first_not_of(basic_string_view(s, n), pos); - } - size_type find_first_not_of(const CharT* s, size_type pos = 0) const noexcept - { - return find_first_not_of(basic_string_view(s), pos); - } - - size_type find_last_not_of(basic_string_view s, size_type pos = npos) const noexcept - { - if (pos >= length_) - { - pos = length_ - 1; - } - if (s.length_ == 0) - { - return pos; - } - pos = length_ - (pos+1); - - const_iterator it = crend(); - for (auto p = crbegin()+pos; p != crend(); ++p) - { - if (Traits::find(s.data_, s.length_, *p) == 0) - { - it = p; - break; - } - } - return it == crend() ? npos : (length_ - 1 - std::distance(crbegin(), it)); - } - size_type find_last_not_of(CharT ch, size_type pos = npos) const noexcept - { - return find_last_not_of(basic_string_view(&ch, 1), pos); - } - size_type find_last_not_of(const CharT* s, size_type pos, size_type n) const noexcept - { - return find_last_not_of(basic_string_view(s, n), pos); - } - size_type find_last_not_of(const CharT* s, size_type pos = npos) const noexcept - { - return find_last_not_of(basic_string_view(s), pos); - } - - friend std::basic_ostream& operator<<(std::basic_ostream& os, const basic_string_view& sv) - { - os.write(sv.data_,sv.length_); - return os; - } -}; - -// == -template -bool operator==(const basic_string_view& lhs, - const basic_string_view& rhs) -{ - return lhs.compare(rhs) == 0; -} -template -bool operator==(const basic_string_view& lhs, - const std::basic_string& rhs) -{ - return lhs.compare(rhs) == 0; -} -template -bool operator==(const std::basic_string& lhs, - const basic_string_view& rhs) -{ - return rhs.compare(lhs) == 0; -} -template -bool operator==(const basic_string_view& lhs, - const CharT* rhs) -{ - return lhs.compare(rhs) == 0; -} -template -bool operator==(const CharT* lhs, - const basic_string_view& rhs) -{ - return rhs.compare(lhs) == 0; -} - -// != -template -bool operator!=(const basic_string_view& lhs, - const basic_string_view& rhs) -{ - return lhs.compare(rhs) != 0; -} -template -bool operator!=(const basic_string_view& lhs, - const std::basic_string& rhs) -{ - return lhs.compare(rhs) != 0; -} -template -bool operator!=(const std::basic_string& lhs, - const basic_string_view& rhs) -{ - return rhs.compare(lhs) != 0; -} -template -bool operator!=(const basic_string_view& lhs, - const CharT* rhs) -{ - return lhs.compare(rhs) != 0; -} -template -bool operator!=(const CharT* lhs, - const basic_string_view& rhs) -{ - return rhs.compare(lhs) != 0; -} - -// <= -template -bool operator<=(const basic_string_view& lhs, - const basic_string_view& rhs) -{ - return lhs.compare(rhs) <= 0; -} -template -bool operator<=(const basic_string_view& lhs, - const std::basic_string& rhs) -{ - return lhs.compare(rhs) <= 0; -} -template -bool operator<=(const std::basic_string& lhs, - const basic_string_view& rhs) -{ - return rhs.compare(lhs) >= 0; -} - -// < -template -bool operator<(const basic_string_view& lhs, - const basic_string_view& rhs) -{ - return lhs.compare(rhs) < 0; -} -template -bool operator<(const basic_string_view& lhs, - const std::basic_string& rhs) -{ - return lhs.compare(rhs) < 0; -} -template -bool operator<(const std::basic_string& lhs, - const basic_string_view& rhs) -{ - return rhs.compare(lhs) > 0; -} - -// >= -template -bool operator>=(const basic_string_view& lhs, - const basic_string_view& rhs) -{ - return lhs.compare(rhs) >= 0; -} -template -bool operator>=(const basic_string_view& lhs, - const std::basic_string& rhs) -{ - return lhs.compare(rhs) >= 0; -} -template -bool operator>=(const std::basic_string& lhs, - const basic_string_view& rhs) -{ - return rhs.compare(lhs) <= 0; -} - -// > -template -bool operator>(const basic_string_view& lhs, - const basic_string_view& rhs) -{ - return lhs.compare(rhs) > 0; -} -template -bool operator>(const basic_string_view& lhs, - const std::basic_string& rhs) -{ - return lhs.compare(rhs) > 0; -} -template -bool operator>(const std::basic_string& lhs, - const basic_string_view& rhs) -{ - return rhs.compare(lhs) < 0; -} - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/type_traits.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/type_traits.hpp deleted file mode 100644 index 53c5514376..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/detail/type_traits.hpp +++ /dev/null @@ -1,228 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_DETAIL_TYPE_TRAITS_HPP -#define JSONCONS_DETAIL_TYPE_TRAITS_HPP - -#include -#include -#include -#include // std::enable_if, std::true_type -#include -#include // std::iterator_traits -#include -#include -#include -#include - -namespace jsoncons -{ -// static_max - -template -struct static_max; - -template -struct static_max -{ - static const size_t value = arg; -}; - -template -struct static_max -{ - static const size_t value = arg1 >= arg2 ? - static_max::value : - static_max::value; -}; - -// type_wrapper - -template -struct type_wrapper -{ - typedef T* pointer_type; - typedef const T* const_pointer_type; - typedef T value_type; - typedef T& reference; - typedef const T& const_reference; -}; - -template -struct type_wrapper -{ - typedef T* pointer_type; - typedef const T* const_pointer_type; - typedef T value_type; - typedef T& reference; - typedef const T& const_reference; -}; - -template -struct type_wrapper -{ - typedef T* pointer_type; - typedef const T* const_pointer_type; - typedef T value_type; - typedef T& reference; - typedef const T& const_reference; -}; - -template -struct type_wrapper -{ - typedef T* pointer_type; - typedef const T* const_pointer_type; - typedef T value_type; - typedef T& reference; - typedef const T& const_reference; -}; - -inline -char to_hex_character(uint8_t c) -{ - JSONCONS_ASSERT(c <= 0xF); - - return (char)((c < 10) ? ('0' + c) : ('A' - 10 + c)); -} - -inline -bool is_control_character(uint32_t c) -{ - return c <= 0x1F || c == 0x7f; -} - -inline -bool is_non_ascii_codepoint(uint32_t cp) -{ - return cp >= 0x80; -} - -template -struct is_stateless - : public std::integral_constant::value && - std::is_empty::value)> -{}; - -// type traits extensions - - -namespace detail { - -// to_plain_pointer - -template inline -typename std::pointer_traits::element_type* to_plain_pointer(Pointer ptr) -{ - return (std::addressof(*ptr)); -} - -template inline -T * to_plain_pointer(T * ptr) -{ - return (ptr); -} - -// has_char_traits_member_type - -template -struct has_char_traits_member_type : std::false_type {}; - -template -struct has_char_traits_member_type::value ->::type> : std::true_type {}; - - -// is_string_like - -template -struct is_string_like : std::false_type {}; - -template -struct is_string_like::value && !std::is_void::value && !std::is_void::value ->::type> : std::true_type {}; - -// is_string_view_like - -template -struct is_string_view_like : std::false_type {}; - -template -struct is_string_view_like::value && !std::is_void::value && !is_string_like::value ->::type> : std::true_type {}; - -// is_integer_like - -template -struct is_integer_like : std::false_type {}; - -template -struct is_integer_like::value && - std::is_signed::value && - !std::is_same::value>::type> : std::true_type {}; - -// is_uinteger_like - -template -struct is_uinteger_like : std::false_type {}; - -template -struct is_uinteger_like::value && - !std::is_signed::value && - !std::is_same::value>::type> : std::true_type {}; - -// is_floating_point_like - -template -struct is_floating_point_like : std::false_type {}; - -template -struct is_floating_point_like::value>::type> : std::true_type {}; - -// is_map_like - -template -struct is_map_like : std::false_type {}; - -template -struct is_map_like::value>::type> - : std::true_type {}; - -// is_array_like -template -struct is_array_like : std::false_type {}; - -template -struct is_array_like> : std::true_type {}; - -// is_vector_like - -template -struct is_vector_like : std::false_type {}; - -template -struct is_vector_like::value && - !std::is_void::value_type>::value && - !is_array_like::value && - !has_char_traits_member_type::value && - !is_map_like::value ->::type> - : std::true_type {}; - -} - -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/encode_decode_json.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/encode_decode_json.hpp deleted file mode 100644 index 196b2244b9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/encode_decode_json.hpp +++ /dev/null @@ -1,280 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_ENCODE_DECODE_JSON_HPP -#define JSONCONS_ENCODE_DECODE_JSON_HPP - -#include -#include -#include -#include -#include // std::basic_istream -#include -#include - -namespace jsoncons { - -// decode_json - -template -typename std::enable_if::value,T>::type -decode_json(const std::basic_string& s, - const basic_json_decode_options& options = basic_json_options::default_options()) -{ - jsoncons::json_decoder decoder; - basic_json_reader> reader(s, decoder, options); - reader.read(); - return decoder.get_result(); -} - -template -typename std::enable_if::value,T>::type -decode_json(const std::basic_string& s, - const basic_json_decode_options& options = basic_json_options::default_options()) -{ - basic_json_cursor reader(s, options); - T val; - read_from(basic_json(), reader, val); - return val; -} - -template -typename std::enable_if::value,T>::type -decode_json(std::basic_istream& is, - const basic_json_decode_options& options = basic_json_options::default_options()) -{ - jsoncons::json_decoder decoder; - basic_json_reader> reader(is, decoder, options); - reader.read(); - return decoder.get_result(); -} - -template -typename std::enable_if::value,T>::type -decode_json(std::basic_istream& is, - const basic_json_decode_options& options = basic_json_options::default_options()) -{ - basic_json_cursor reader(is, options); - T val; - read_from(basic_json(), reader, val); - return val; -} - -template -T decode_json(const basic_json& j, - const std::basic_string& s, - const basic_json_decode_options& options = basic_json_options::default_options()) -{ - basic_json_cursor reader(s, options); - T val; - read_from(j, reader, val); - return val; -} - -template -T decode_json(const basic_json& j, - std::basic_istream& is, - const basic_json_decode_options& options = basic_json_options::default_options()) -{ - basic_json_cursor reader(is, options); - T val; - read_from(j, reader, val); - return val; -} - -// encode_json - -template -void encode_json(const T& val, - basic_json_content_handler& receiver) -{ - write_to(basic_json(), val, receiver); - receiver.flush(); -} - -// to stream - -template -typename std::enable_if::value>::type -encode_json(const T& val, - std::basic_ostream& os, - const basic_json_encode_options& options = basic_json_options::default_options(), - indenting line_indent = indenting::no_indent) -{ - if (line_indent == indenting::indent) - { - basic_json_encoder encoder(os, options); - val.dump(encoder); - } - else - { - basic_json_compressed_encoder encoder(os, options); - val.dump(encoder); - } -} - -template -typename std::enable_if::value>::type -encode_json(const T& val, - std::basic_ostream& os, - const basic_json_encode_options& options = basic_json_options::default_options(), - indenting line_indent = indenting::no_indent) -{ - if (line_indent == indenting::indent) - { - basic_json_encoder encoder(os, options); - encode_json(val, encoder); - } - else - { - basic_json_compressed_encoder encoder(os, options); - encode_json(val, encoder); - } -} - -template -void encode_json(const T& val, - std::basic_ostream& os, - indenting line_indent) -{ - encode_json(val, os, basic_json_options::default_options(), line_indent); -} - -// to string - -template -typename std::enable_if::value>::type -encode_json(const T& val, - std::basic_string& s, - const basic_json_encode_options& options = basic_json_options::default_options(), - indenting line_indent = indenting::no_indent) -{ - if (line_indent == indenting::indent) - { - basic_json_encoder>> encoder(s, options); - val.dump(encoder); - } - else - { - basic_json_compressed_encoder>> encoder(s, options); - val.dump(encoder); - } -} - -template -typename std::enable_if::value>::type -encode_json(const T& val, - std::basic_string& s, - const basic_json_encode_options& options = basic_json_options::default_options(), - indenting line_indent = indenting::no_indent) -{ - if (line_indent == indenting::indent) - { - basic_json_encoder>> encoder(s, options); - encode_json(val, encoder); - } - else - { - basic_json_compressed_encoder>> encoder(s, options); - encode_json(val, encoder); - } -} - -template -void encode_json(const T& val, - std::basic_string& s, - indenting line_indent) -{ - encode_json(val, s, basic_json_options::default_options(), line_indent); -} - -template -void encode_json(const basic_json& j, - const T& val, - basic_json_content_handler& receiver) -{ - write_to(j, val, receiver); - receiver.flush(); -} - -template -void encode_json(const basic_json& j, - const T& val, - std::basic_ostream& os, - const basic_json_encode_options& options = basic_json_options::default_options(), - indenting line_indent = indenting::no_indent) -{ - if (line_indent == indenting::indent) - { - basic_json_encoder encoder(os, options); - encode_json(j, val, encoder); - } - else - { - basic_json_compressed_encoder encoder(os, options); - encode_json(j, val, encoder); - } -} - -template -void encode_json(const basic_json& j, - const T& val, - std::basic_ostream& os, - indenting line_indent) -{ - if (line_indent == indenting::indent) - { - basic_json_encoder encoder(os, basic_json_options::default_options()); - encode_json(j, val, encoder); - } - else - { - basic_json_compressed_encoder encoder(os, basic_json_options::default_options()); - encode_json(j, val, encoder); - } -} - -template -void encode_json(const basic_json& j, - const T& val, - std::basic_string& s, - const basic_json_encode_options& options = basic_json_options::default_options(), - indenting line_indent = indenting::no_indent) -{ - if (line_indent == indenting::indent) - { - basic_json_encoder>> encoder(s, options); - encode_json(j, val, encoder); - } - else - { - basic_json_compressed_encoder>> encoder(s, options); - encode_json(j, val, encoder); - } -} - -template -void encode_json(const basic_json& j, - const T& val, - std::basic_string& s, - indenting line_indent) -{ - if (line_indent == indenting::indent) - { - basic_json_encoder>> encoder(s, basic_json_options::default_options()); - encode_json(j, val, encoder); - } - else - { - basic_json_compressed_encoder>> encoder(s, basic_json_options::default_options()); - encode_json(j, val, encoder); - } -} - -} - -#endif - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json.hpp deleted file mode 100644 index 0dad529e33..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json.hpp +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSON_HPP -#define JSONCONS_JSON_HPP - -#include -#include - -#endif - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_container_types.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_container_types.hpp deleted file mode 100644 index 8bc415a5b1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_container_types.hpp +++ /dev/null @@ -1,1962 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSON_CONTAINER_TYPES_HPP -#define JSONCONS_JSON_CONTAINER_TYPES_HPP - -#include -#include -#include -#include -#include -#include // std::sort, std::stable_sort, std::lower_bound, std::unique -#include -#include -#include // std::iterator_traits -#include // std::allocator -#include // std::move -#include // std::enable_if -#include -#include - -namespace jsoncons { - -// key_value - -template -class key_value -{ -public: - typedef KeyT key_type; - typedef ValueT value_type; - typedef typename KeyT::allocator_type allocator_type; - typedef typename value_type::string_view_type string_view_type; -private: - key_type key_; - value_type value_; -public: - - key_value() - { - } - - key_value(const key_type& name, const value_type& val) - : key_(name), value_(val) - { - } - - key_value(const string_view_type& name) - : key_(name) - { - } - - template - key_value(key_type&& name, T&& val) - : key_(std::forward(name)), - value_(std::forward(val)) - { - } - - template - key_value(key_type&& name, - T&& val, - const allocator_type& allocator) - : key_(std::forward(name)), value_(std::forward(val), allocator) - { - } - - key_value(const key_value& member) - : key_(member.key_), value_(member.value_) - { - } - - key_value(key_value&& member) - : key_(std::move(member.key_)), value_(std::move(member.value_)) - { - } - - string_view_type key() const - { - return string_view_type(key_.data(),key_.size()); - } - - value_type& value() - { - return value_; - } - - const value_type& value() const - { - return value_; - } - - template - void value(T&& value) - { - value_ = std::forward(value); - } - - void swap(key_value& member) - { - key_.swap(member.key_); - value_.swap(member.value_); - } - - key_value& operator=(const key_value& member) - { - if (this != & member) - { - key_ = member.key_; - value_ = member.value_; - } - return *this; - } - - key_value& operator=(key_value&& member) - { - if (this != &member) - { - key_.swap(member.key_); - value_.swap(member.value_); - } - return *this; - } - - void shrink_to_fit() - { - key_.shrink_to_fit(); - value_.shrink_to_fit(); - } -#if !defined(JSONCONS_NO_DEPRECATED) - const key_type& name() const - { - return key_; - } -#endif - - friend bool operator==(const key_value& lhs, const key_value& rhs) - { - return lhs.key_ == rhs.key_ && lhs.value_ == rhs.value_; - } - - friend bool operator!=(const key_value& lhs, const key_value& rhs) - { - return !(lhs == rhs); - } - - friend bool operator<(const key_value& lhs, const key_value& rhs) - { - if (lhs.key_ < rhs.key_) - { - return true; - } - if (lhs.key_ == rhs.key_ && lhs.value_ < rhs.value_) - { - return true; - } - return false; - } - - friend bool operator<=(const key_value& lhs, const key_value& rhs) - { - return !(rhs < lhs); - } - - friend bool operator>(const key_value& lhs, const key_value& rhs) - { - return !(lhs <= rhs); - } - - friend bool operator>=(const key_value& lhs, const key_value& rhs) - { - return !(lhs < rhs); - } -}; - -template -struct get_key_value -{ - typedef key_value key_value_type; - - template - key_value_type operator()(const std::pair& p) - { - return key_value_type(p.first,p.second); - } - template - key_value_type operator()(std::pair&& p) - { - return key_value_type(std::forward(p.first),std::forward(p.second)); - } - template - const key_value_type& operator()(const key_value& p) - { - return p; - } - template - key_value_type operator()(key_value&& p) - { - return std::move(p); - } -}; - -// json_array - -template -class container_base -{ -public: - typedef Allocator allocator_type; -private: - allocator_type allocator_; -public: - container_base() - : allocator_() - { - } - container_base(const allocator_type& allocator) - : allocator_(allocator) - { - } - - allocator_type get_allocator() const - { - return allocator_; - } -}; - -// json_array - -template -class json_array : public container_base -{ -public: - typedef typename Json::allocator_type allocator_type; - typedef Json value_type; -private: - typedef typename Json::implementation_policy implementation_policy; - typedef typename std::allocator_traits:: template rebind_alloc value_allocator_type; - using value_container_type = typename implementation_policy::template sequence_container_type; - value_container_type elements_; -public: - typedef typename value_container_type::iterator iterator; - typedef typename value_container_type::const_iterator const_iterator; - typedef typename std::iterator_traits::reference reference; - typedef typename std::iterator_traits::reference const_reference; - - using container_base::get_allocator; - - json_array() - { - } - - explicit json_array(const allocator_type& allocator) - : container_base(allocator), - elements_(value_allocator_type(allocator)) - { - } - - explicit json_array(size_t n, - const allocator_type& allocator = allocator_type()) - : container_base(allocator), - elements_(n,Json(),value_allocator_type(allocator)) - { - } - - explicit json_array(size_t n, - const Json& value, - const allocator_type& allocator = allocator_type()) - : container_base(allocator), - elements_(n,value,value_allocator_type(allocator)) - { - } - - template - json_array(InputIterator begin, InputIterator end, const allocator_type& allocator = allocator_type()) - : container_base(allocator), - elements_(begin,end,value_allocator_type(allocator)) - { - } - json_array(const json_array& val) - : container_base(val.get_allocator()), - elements_(val.elements_) - { - } - json_array(const json_array& val, const allocator_type& allocator) - : container_base(allocator), - elements_(val.elements_,value_allocator_type(allocator)) - { - } - - json_array(json_array&& val) noexcept - : container_base(val.get_allocator()), - elements_(std::move(val.elements_)) - { - } - json_array(json_array&& val, const allocator_type& allocator) - : container_base(allocator), - elements_(std::move(val.elements_),value_allocator_type(allocator)) - { - } - - json_array(std::initializer_list init) - : container_base(), - elements_(std::move(init)) - { - } - - json_array(std::initializer_list init, - const allocator_type& allocator) - : container_base(allocator), - elements_(std::move(init),value_allocator_type(allocator)) - { - } - ~json_array() - { - } - - void swap(json_array& val) - { - elements_.swap(val.elements_); - } - - size_t size() const {return elements_.size();} - - size_t capacity() const {return elements_.capacity();} - - void clear() {elements_.clear();} - - void shrink_to_fit() - { - for (size_t i = 0; i < elements_.size(); ++i) - { - elements_[i].shrink_to_fit(); - } - elements_.shrink_to_fit(); - } - - void reserve(size_t n) {elements_.reserve(n);} - - void resize(size_t n) {elements_.resize(n);} - - void resize(size_t n, const Json& val) {elements_.resize(n,val);} - -#if !defined(JSONCONS_NO_DEPRECATED) - void remove_range(size_t from_index, size_t to_index) - { - JSONCONS_ASSERT(from_index <= to_index); - JSONCONS_ASSERT(to_index <= elements_.size()); - elements_.erase(elements_.cbegin()+from_index,elements_.cbegin()+to_index); - } -#endif - void erase(const_iterator pos) - { -#if defined(JSONCONS_NO_ERASE_TAKING_CONST_ITERATOR) - iterator it = elements_.begin() + (pos - elements_.begin()); - elements_.erase(it); -#else - elements_.erase(pos); -#endif - } - - void erase(const_iterator first, const_iterator last) - { -#if defined(JSONCONS_NO_ERASE_TAKING_CONST_ITERATOR) - iterator it1 = elements_.begin() + (first - elements_.begin()); - iterator it2 = elements_.begin() + (last - elements_.begin()); - elements_.erase(it1,it2); -#else - elements_.erase(first,last); -#endif - } - - Json& operator[](size_t i) {return elements_[i];} - - const Json& operator[](size_t i) const {return elements_[i];} - - // push_back - - template - typename std::enable_if::value,void>::type - push_back(T&& value) - { - elements_.emplace_back(std::forward(value)); - } - - template - typename std::enable_if::value,void>::type - push_back(T&& value) - { - elements_.emplace_back(std::forward(value),get_allocator()); - } - - template - typename std::enable_if::value,iterator>::type - insert(const_iterator pos, T&& value) - { -#if defined(JSONCONS_NO_ERASE_TAKING_CONST_ITERATOR) - iterator it = elements_.begin() + (pos - elements_.begin()); - return elements_.emplace(it, std::forward(value)); -#else - return elements_.emplace(pos, std::forward(value)); -#endif - } - template - typename std::enable_if::value,iterator>::type - insert(const_iterator pos, T&& value) - { -#if defined(JSONCONS_NO_ERASE_TAKING_CONST_ITERATOR) - iterator it = elements_.begin() + (pos - elements_.begin()); - return elements_.emplace(it, std::forward(value), get_allocator()); -#else - return elements_.emplace(pos, std::forward(value), get_allocator()); -#endif - } - - template - iterator insert(const_iterator pos, InputIt first, InputIt last) - { -#if defined(JSONCONS_NO_ERASE_TAKING_CONST_ITERATOR) - iterator it = elements_.begin() + (pos - elements_.begin()); - elements_.insert(it, first, last); - return first == last ? it : it + 1; -#else - return elements_.insert(pos, first, last); -#endif - } - - template - typename std::enable_if::value,iterator>::type - emplace(const_iterator pos, Args&&... args) - { -#if defined(JSONCONS_NO_ERASE_TAKING_CONST_ITERATOR) - iterator it = elements_.begin() + (pos - elements_.begin()); - return elements_.emplace(it, std::forward(args)...); -#else - return elements_.emplace(pos, std::forward(args)...); -#endif - } - - template - Json& emplace_back(Args&&... args) - { - elements_.emplace_back(std::forward(args)...); - return elements_.back(); - } - - iterator begin() {return elements_.begin();} - - iterator end() {return elements_.end();} - - const_iterator begin() const {return elements_.begin();} - - const_iterator end() const {return elements_.end();} - - bool operator==(const json_array& rhs) const - { - return elements_ == rhs.elements_; - } - - bool operator<(const json_array& rhs) const - { - return elements_ < rhs.elements_; - } -private: - - json_array& operator=(const json_array&) = delete; -}; - -struct sorted_unique_range_tag -{ -}; - -// json_object - -template -class json_object -{ -}; - -// Sort keys -template -class json_object::type> : - public container_base -{ -public: - typedef typename Json::allocator_type allocator_type; - typedef KeyT key_type; - typedef key_value key_value_type; - typedef typename Json::char_type char_type; - typedef typename Json::string_view_type string_view_type; -private: - typedef typename Json::implementation_policy implementation_policy; - typedef typename std::allocator_traits:: template rebind_alloc key_value_allocator_type; - using key_value_container_type = typename implementation_policy::template sequence_container_type; - - key_value_container_type members_; -public: - typedef typename key_value_container_type::iterator iterator; - typedef typename key_value_container_type::const_iterator const_iterator; - - using container_base::get_allocator; - - json_object() - { - } - - explicit json_object(const allocator_type& allocator) - : container_base(allocator), - members_(key_value_allocator_type(allocator)) - { - } - - json_object(const json_object& val) - : container_base(val.get_allocator()), - members_(val.members_) - { - } - - json_object(json_object&& val) - : container_base(val.get_allocator()), - members_(std::move(val.members_)) - { - } - - json_object(const json_object& val, const allocator_type& allocator) - : container_base(allocator), - members_(val.members_,key_value_allocator_type(allocator)) - { - } - - json_object(json_object&& val,const allocator_type& allocator) - : container_base(allocator), members_(std::move(val.members_),key_value_allocator_type(allocator)) - { - } - - template - json_object(InputIt first, InputIt last) - { - size_t count = std::distance(first,last); - members_.reserve(count); - for (auto s = first; s != last; ++s) - { - members_.emplace_back(get_key_value()(*s)); - } - std::stable_sort(members_.begin(),members_.end(), - [](const key_value_type& a, const key_value_type& b){return a.key().compare(b.key()) < 0;}); - auto it = std::unique(members_.begin(), members_.end(), - [](const key_value_type& a, const key_value_type& b){ return !(a.key().compare(b.key()));}); - members_.erase(it, members_.end()); - } - - template - json_object(InputIt first, InputIt last, - const allocator_type& allocator) - : container_base(allocator), - members_(key_value_allocator_type(allocator)) - { - size_t count = std::distance(first,last); - members_.reserve(count); - for (auto s = first; s != last; ++s) - { - members_.emplace_back(get_key_value()(*s)); - } - std::stable_sort(members_.begin(),members_.end(), - [](const key_value_type& a, const key_value_type& b){return a.key().compare(b.key()) < 0;}); - auto it = std::unique(members_.begin(), members_.end(), - [](const key_value_type& a, const key_value_type& b){ return !(a.key().compare(b.key()));}); - members_.erase(it, members_.end()); - } - - json_object(std::initializer_list init) - { - for (const auto& element : init) - { - if (element.size() != 2 || !element[0].is_string()) - { - JSONCONS_THROW(json_runtime_error("Cannot create object from initializer list")); - break; - } - } - for (auto& element : init) - { - insert_or_assign(element[0].as_string_view(), std::move(element[1])); - } - } - - json_object(std::initializer_list init, - const allocator_type& allocator) - : container_base(allocator), - members_(key_value_allocator_type(allocator)) - { - for (const auto& element : init) - { - if (element.size() != 2 || !element[0].is_string()) - { - JSONCONS_THROW(json_runtime_error("Cannot create object from initializer list")); - break; - } - } - for (auto& element : init) - { - insert_or_assign(element[0].as_string_view(), std::move(element[1])); - } - } - - void swap(json_object& val) - { - members_.swap(val.members_); - } - - iterator begin() - { - return members_.begin(); - } - - iterator end() - { - return members_.end(); - } - - const_iterator begin() const - { - return members_.begin(); - } - - const_iterator end() const - { - return members_.end(); - } - - size_t size() const {return members_.size();} - - size_t capacity() const {return members_.capacity();} - - void clear() {members_.clear();} - - void shrink_to_fit() - { - for (size_t i = 0; i < members_.size(); ++i) - { - members_[i].shrink_to_fit(); - } - members_.shrink_to_fit(); - } - - void reserve(size_t n) {members_.reserve(n);} - - Json& at(size_t i) - { - if (i >= members_.size()) - { - JSONCONS_THROW(json_runtime_error("Invalid array subscript")); - } - return members_[i].value(); - } - - const Json& at(size_t i) const - { - if (i >= members_.size()) - { - JSONCONS_THROW(json_runtime_error("Invalid array subscript")); - } - return members_[i].value(); - } - - iterator find(const string_view_type& name) - { - auto it = std::lower_bound(members_.begin(),members_.end(), name, - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - auto result = (it != members_.end() && it->key() == name) ? it : members_.end(); - return result; - } - - const_iterator find(const string_view_type& name) const - { - auto it = std::lower_bound(members_.begin(),members_.end(), - name, - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - auto result = (it != members_.end() && it->key() == name) ? it : members_.end(); - return result; - } - - void erase(const_iterator pos) - { -#if defined(JSONCONS_NO_ERASE_TAKING_CONST_ITERATOR) - iterator it = members_.begin() + (pos - members_.begin()); - members_.erase(it); -#else - members_.erase(pos); -#endif - } - - void erase(const_iterator first, const_iterator last) - { -#if defined(JSONCONS_NO_ERASE_TAKING_CONST_ITERATOR) - iterator it1 = members_.begin() + (first - members_.begin()); - iterator it2 = members_.begin() + (last - members_.begin()); - members_.erase(it1,it2); -#else - members_.erase(first,last); -#endif - } - - void erase(const string_view_type& name) - { - auto it = std::lower_bound(members_.begin(),members_.end(), name, - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - if (it != members_.end() && it->key() == name) - { - members_.erase(it); - } - } - - template - void insert(InputIt first, InputIt last, Convert convert) - { - size_t count = std::distance(first,last); - members_.reserve(members_.size() + count); - for (auto s = first; s != last; ++s) - { - members_.emplace_back(convert(*s)); - } - std::stable_sort(members_.begin(),members_.end(), - [](const key_value_type& a, const key_value_type& b){return a.key().compare(b.key()) < 0;}); - auto it = std::unique(members_.begin(), members_.end(), - [](const key_value_type& a, const key_value_type& b){ return !(a.key().compare(b.key()));}); - members_.erase(it, members_.end()); - } - - template - void insert(sorted_unique_range_tag, InputIt first, InputIt last, Convert convert) - { - if (first != last) - { - size_t count = std::distance(first,last); - members_.reserve(members_.size() + count); - - auto it = find(convert(*first).key()); - if (it != members_.end()) - { - for (auto s = first; s != last; ++s) - { - it = members_.emplace(it, convert(*s)); - } - } - else - { - for (auto s = first; s != last; ++s) - { - members_.emplace_back(convert(*s)); - } - } - } - } - - // insert_or_assign - - template - typename std::enable_if::value,std::pair>::type - insert_or_assign(const string_view_type& name, T&& value) - { - bool inserted; - auto it = std::lower_bound(members_.begin(),members_.end(), name, - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - if (it == members_.end()) - { - members_.emplace_back(key_type(name.begin(),name.end()), - std::forward(value)); - inserted = true; - it = members_.begin() + members_.size() - 1; - } - else if (it->key() == name) - { - it->value(Json(std::forward(value))); - inserted = false; // assigned - } - else - { - it = members_.emplace(it, - key_type(name.begin(),name.end()), - std::forward(value)); - inserted = true; - } - return std::make_pair(it,inserted); - } - - template - typename std::enable_if::value,std::pair>::type - insert_or_assign(const string_view_type& name, T&& value) - { - bool inserted; - auto it = std::lower_bound(members_.begin(),members_.end(), name, - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - if (it == members_.end()) - { - members_.emplace_back(key_type(name.begin(),name.end(), get_allocator()), - std::forward(value),get_allocator()); - inserted = true; - it = members_.begin() + members_.size() - 1; - } - else if (it->key() == name) - { - it->value(Json(std::forward(value), get_allocator())); - inserted = false; // assigned - } - else - { - it = members_.emplace(it, - key_type(name.begin(),name.end(), get_allocator()), - std::forward(value),get_allocator()); - inserted = true; - } - return std::make_pair(it,inserted); - } - - // try_emplace - - template - typename std::enable_if::value,std::pair>::type - try_emplace(const string_view_type& name, Args&&... args) - { - bool inserted; - auto it = std::lower_bound(members_.begin(),members_.end(), name, - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - if (it == members_.end()) - { - members_.emplace_back(key_type(name.begin(),name.end()), - std::forward(args)...); - it = members_.begin() + members_.size() - 1; - inserted = true; - } - else if (it->key() == name) - { - inserted = false; - } - else - { - it = members_.emplace(it, - key_type(name.begin(),name.end()), - std::forward(args)...); - inserted = true; - } - return std::make_pair(it,inserted); - } - - template - typename std::enable_if::value,std::pair>::type - try_emplace(const string_view_type& name, Args&&... args) - { - bool inserted; - auto it = std::lower_bound(members_.begin(),members_.end(), name, - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - if (it == members_.end()) - { - members_.emplace_back(key_type(name.begin(),name.end(), get_allocator()), - std::forward(args)...); - it = members_.begin() + members_.size() - 1; - inserted = true; - } - else if (it->key() == name) - { - inserted = false; - } - else - { - it = members_.emplace(it, - key_type(name.begin(),name.end(), get_allocator()), - std::forward(args)...); - inserted = true; - } - return std::make_pair(it,inserted); - } - - template - typename std::enable_if::value,iterator>::type - try_emplace(iterator hint, const string_view_type& name, Args&&... args) - { - iterator it = hint; - - if (hint != members_.end() && hint->key() <= name) - { - it = std::lower_bound(hint,members_.end(), name, - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - } - else - { - it = std::lower_bound(members_.begin(),members_.end(), name, - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - } - - if (it == members_.end()) - { - members_.emplace_back(key_type(name.begin(),name.end()), - std::forward(args)...); - it = members_.begin() + (members_.size() - 1); - } - else if (it->key() == name) - { - } - else - { - it = members_.emplace(it, - key_type(name.begin(),name.end()), - std::forward(args)...); - } - - return it; - } - - template - typename std::enable_if::value,iterator>::type - try_emplace(iterator hint, const string_view_type& name, Args&&... args) - { - iterator it = hint; - if (hint != members_.end() && hint->key() <= name) - { - it = std::lower_bound(hint,members_.end(), name, - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - } - else - { - it = std::lower_bound(members_.begin(),members_.end(), name, - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - } - - if (it == members_.end()) - { - members_.emplace_back(key_type(name.begin(),name.end(), get_allocator()), - std::forward(args)...); - it = members_.begin() + (members_.size() - 1); - } - else if (it->key() == name) - { - } - else - { - it = members_.emplace(it, - key_type(name.begin(),name.end(), get_allocator()), - std::forward(args)...); - } - return it; - } - - // insert_or_assign - - template - typename std::enable_if::value,iterator>::type - insert_or_assign(iterator hint, const string_view_type& name, T&& value) - { - iterator it; - if (hint != members_.end() && hint->key() <= name) - { - it = std::lower_bound(hint,members_.end(), name, - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - } - else - { - it = std::lower_bound(members_.begin(),members_.end(), name, - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - } - - if (it == members_.end()) - { - members_.emplace_back(key_type(name.begin(),name.end()), - std::forward(value)); - it = members_.begin() + (members_.size() - 1); - } - else if (it->key() == name) - { - it->value(Json(std::forward(value))); - } - else - { - it = members_.emplace(it, - key_type(name.begin(),name.end()), - std::forward(value)); - } - return it; - } - - template - typename std::enable_if::value,iterator>::type - insert_or_assign(iterator hint, const string_view_type& name, T&& value) - { - iterator it; - if (hint != members_.end() && hint->key() <= name) - { - it = std::lower_bound(hint,members_.end(), name, - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - } - else - { - it = std::lower_bound(members_.begin(),members_.end(), name, - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - } - - if (it == members_.end()) - { - members_.emplace_back(key_type(name.begin(),name.end(), get_allocator()), - std::forward(value),get_allocator()); - it = members_.begin() + (members_.size() - 1); - } - else if (it->key() == name) - { - it->value(Json(std::forward(value),get_allocator())); - } - else - { - it = members_.emplace(it, - key_type(name.begin(),name.end(), get_allocator()), - std::forward(value),get_allocator()); - } - return it; - } - - // merge - - void merge(const json_object& source) - { - for (auto it = source.begin(); it != source.end(); ++it) - { - try_emplace(it->key(),it->value()); - } - } - - void merge(json_object&& source) - { - auto it = std::make_move_iterator(source.begin()); - auto end = std::make_move_iterator(source.end()); - for (; it != end; ++it) - { - auto pos = std::lower_bound(members_.begin(),members_.end(), it->key(), - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - if (pos == members_.end() ) - { - members_.emplace_back(*it); - } - else if (it->key() != pos->key()) - { - members_.emplace(pos,*it); - } - } - } - - void merge(iterator hint, const json_object& source) - { - for (auto it = source.begin(); it != source.end(); ++it) - { - hint = try_emplace(hint, it->key(),it->value()); - } - } - - void merge(iterator hint, json_object&& source) - { - auto it = std::make_move_iterator(source.begin()); - auto end = std::make_move_iterator(source.end()); - for (; it != end; ++it) - { - iterator pos; - if (hint != members_.end() && hint->key() <= it->key()) - { - pos = std::lower_bound(hint,members_.end(), it->key(), - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - } - else - { - pos = std::lower_bound(members_.begin(),members_.end(), it->key(), - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - } - if (pos == members_.end() ) - { - members_.emplace_back(*it); - hint = members_.begin() + (members_.size() - 1); - } - else if (it->key() != pos->key()) - { - hint = members_.emplace(pos,*it); - } - } - } - - // merge_or_update - - void merge_or_update(const json_object& source) - { - for (auto it = source.begin(); it != source.end(); ++it) - { - insert_or_assign(it->key(),it->value()); - } - } - - void merge_or_update(json_object&& source) - { - auto it = std::make_move_iterator(source.begin()); - auto end = std::make_move_iterator(source.end()); - for (; it != end; ++it) - { - auto pos = std::lower_bound(members_.begin(),members_.end(), it->key(), - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - if (pos == members_.end() ) - { - members_.emplace_back(*it); - } - else - { - pos->value(it->value()); - } - } - } - - void merge_or_update(iterator hint, const json_object& source) - { - for (auto it = source.begin(); it != source.end(); ++it) - { - hint = insert_or_assign(hint, it->key(),it->value()); - } - } - - void merge_or_update(iterator hint, json_object&& source) - { - auto it = std::make_move_iterator(source.begin()); - auto end = std::make_move_iterator(source.end()); - for (; it != end; ++it) - { - iterator pos; - if (hint != members_.end() && hint->key() <= it->key()) - { - pos = std::lower_bound(hint,members_.end(), it->key(), - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - } - else - { - pos = std::lower_bound(members_.begin(),members_.end(), it->key(), - [](const key_value_type& a, const string_view_type& k){return a.key().compare(k) < 0;}); - } - if (pos == members_.end() ) - { - members_.emplace_back(*it); - hint = members_.begin() + (members_.size() - 1); - } - else - { - pos->value(it->value()); - hint = pos; - } - } - } - - bool operator==(const json_object& rhs) const - { - return members_ == rhs.members_; - } - - bool operator<(const json_object& rhs) const - { - return members_ < rhs.members_; - } -private: - json_object& operator=(const json_object&) = delete; -}; - -// Preserve order -template -class json_object::type> : - public container_base -{ -public: - typedef typename Json::allocator_type allocator_type; - typedef typename Json::char_type char_type; - typedef KeyT key_type; - typedef typename Json::string_view_type string_view_type; - typedef key_value key_value_type; -private: - typedef typename Json::implementation_policy implementation_policy; - typedef typename std::allocator_traits:: template rebind_alloc key_value_allocator_type; - using key_value_container_type = typename implementation_policy::template sequence_container_type; - typedef typename std::allocator_traits:: template rebind_alloc index_allocator_type; - using index_container_type = typename implementation_policy::template sequence_container_type; - - key_value_container_type members_; - index_container_type index_; -public: - typedef typename key_value_container_type::iterator iterator; - typedef typename key_value_container_type::const_iterator const_iterator; - - using container_base::get_allocator; - - json_object() - { - } - json_object(const allocator_type& allocator) - : container_base(allocator), - members_(key_value_allocator_type(allocator)), - index_(index_allocator_type(allocator)) - { - } - - json_object(const json_object& val) - : container_base(val.get_allocator()), - members_(val.members_), - index_(val.index_) - { - } - - json_object(json_object&& val) - : container_base(val.get_allocator()), - members_(std::move(val.members_)), - index_(std::move(val.index_)) - { - } - - json_object(const json_object& val, const allocator_type& allocator) - : container_base(allocator), - members_(val.members_,key_value_allocator_type(allocator)), - index_(val.index_,index_allocator_type(allocator)) - { - } - - json_object(json_object&& val,const allocator_type& allocator) - : container_base(allocator), - members_(std::move(val.members_),key_value_allocator_type(allocator)), - index_(std::move(val.index_),index_allocator_type(allocator)) - { - } - - template - json_object(InputIt first, InputIt last) - { - size_t count = std::distance(first,last); - members_.reserve(count); - for (auto s = first; s != last; ++s) - { - members_.emplace_back(get_key_value()(*s)); - } - - build_index(); - auto last_unique = std::unique(index_.begin(), index_.end(), - [&](size_t a, size_t b) { return !(members_.at(a).key().compare(members_.at(b).key())); }); - - if (last_unique != index_.end()) - { - index_.erase(last_unique, index_.end()); - std::sort(index_.begin(), index_.end()); - - auto result = index_.rbegin(); - if (*result != members_.size()) - { - members_.erase(members_.begin() + (*result + 1), members_.end()); - } - for (auto it = index_.rbegin() + 1; it != index_.rend(); ++it, ++result) - { - if (*result - *it > 1) - { - members_.erase(members_.begin() + (*it + 1), members_.begin() + *result); - } - } - } - build_index(); - } - - template - json_object(InputIt first, InputIt last, - const allocator_type& allocator) - : container_base(allocator), - members_(key_value_allocator_type(allocator)), - index_(index_allocator_type(allocator)) - { - size_t count = std::distance(first,last); - members_.reserve(count); - for (auto s = first; s != last; ++s) - { - members_.emplace_back(get_key_value()(*s)); - } - - build_index(); - auto last_unique = std::unique(index_.begin(), index_.end(), - [&](size_t a, size_t b) { return !(members_.at(a).key().compare(members_.at(b).key())); }); - - if (last_unique != index_.end()) - { - index_.erase(last_unique, index_.end()); - std::sort(index_.begin(), index_.end()); - - auto result = index_.rbegin(); - if (*result != members_.size()) - { - members_.erase(members_.begin() + (*result + 1), members_.end()); - } - for (auto it = index_.rbegin() + 1; it != index_.rend(); ++it, ++result) - { - if (*result - *it > 1) - { - members_.erase(members_.begin() + (*it + 1), members_.begin() + *result); - } - } - } - build_index(); - } - - json_object(std::initializer_list init) - { - for (const auto& element : init) - { - if (element.size() != 2 || !element[0].is_string()) - { - JSONCONS_THROW(json_runtime_error("Cannot create object from initializer list")); - break; - } - } - for (auto& element : init) - { - insert_or_assign(element[0].as_string_view(), std::move(element[1])); - } - } - - json_object(std::initializer_list init, - const allocator_type& allocator) - : container_base(allocator), - members_(key_value_allocator_type(allocator)), - index_(index_allocator_type(allocator)) - { - for (const auto& element : init) - { - if (element.size() != 2 || !element[0].is_string()) - { - JSONCONS_THROW(json_runtime_error("Cannot create object from initializer list")); - break; - } - } - for (auto& element : init) - { - insert_or_assign(element[0].as_string_view(), std::move(element[1])); - } - } - - void swap(json_object& val) - { - members_.swap(val.members_); - } - - iterator begin() - { - return members_.begin(); - } - - iterator end() - { - return members_.end(); - } - - const_iterator begin() const - { - return members_.begin(); - } - - const_iterator end() const - { - return members_.end(); - } - - size_t size() const {return members_.size();} - - size_t capacity() const {return members_.capacity();} - - void clear() - { - members_.clear(); - index_.clear(); - } - - void shrink_to_fit() - { - for (size_t i = 0; i < members_.size(); ++i) - { - members_[i].shrink_to_fit(); - } - members_.shrink_to_fit(); - index_.shrink_to_fit(); - } - - void reserve(size_t n) {members_.reserve(n);} - - Json& at(size_t i) - { - if (i >= members_.size()) - { - JSONCONS_THROW(json_runtime_error("Invalid array subscript")); - } - return members_[i].value(); - } - - const Json& at(size_t i) const - { - if (i >= members_.size()) - { - JSONCONS_THROW(json_runtime_error("Invalid array subscript")); - } - return members_[i].value(); - } - - iterator find(const string_view_type& name) - { - auto it = std::lower_bound(index_.begin(),index_.end(), name, - [&](size_t i, const string_view_type& k){return members_.at(i).key().compare(k) < 0;}); - if (it != index_.end() && members_.at(*it).key() == name) - { - return members_.begin() + *it; - } - else - { - return members_.end(); - } - } - - const_iterator find(const string_view_type& name) const - { - auto it = std::lower_bound(index_.begin(),index_.end(), name, - [&](size_t i, const string_view_type& k){return members_.at(i).key().compare(k) < 0;}); - if (it != index_.end() && members_.at(*it).key() == name) - { - return members_.begin() + *it; - } - else - { - return members_.end(); - } - } - - void erase(const_iterator first, const_iterator last) - { - size_t pos1 = first == members_.end() ? members_.size() : first - members_.begin(); - size_t pos2 = last == members_.end() ? members_.size() : last - members_.begin(); - - if (pos1 < members_.size() && pos2 <= members_.size()) - { - erase_index_entries(pos1,pos2); - -#if defined(JSONCONS_NO_ERASE_TAKING_CONST_ITERATOR) - iterator it1 = members_.begin() + (first - members_.begin()); - iterator it2 = members_.begin() + (last - members_.begin()); - members_.erase(it1,it2); -#else - members_.erase(first,last); -#endif - } - } - - void erase(const string_view_type& name) - { - auto pos = find(name); - if (pos != members_.end()) - { - erase_index_entry(name); -#if defined(JSONCONS_NO_ERASE_TAKING_CONST_ITERATOR) - iterator it = members_.begin() + (pos - members_.begin()); - members_.erase(it); -#else - members_.erase(pos); -#endif - } - } - - template - void insert(InputIt first, InputIt last, Convert convert) - { - size_t count = std::distance(first,last); - members_.reserve(members_.size() + count); - for (auto s = first; s != last; ++s) - { - members_.emplace_back(convert(*s)); - } - - build_index(); - auto last_unique = std::unique(index_.begin(), index_.end(), - [&](size_t a, size_t b) { return !(members_.at(a).key().compare(members_.at(b).key())); }); - - if (last_unique != index_.end()) - { - index_.erase(last_unique, index_.end()); - std::sort(index_.begin(), index_.end()); - - auto result = index_.rbegin(); - if (*result != members_.size()) - { - members_.erase(members_.begin() + (*result + 1), members_.end()); - } - for (auto it = index_.rbegin() + 1; it != index_.rend(); ++it, ++result) - { - if (*result - *it > 1) - { - members_.erase(members_.begin() + (*it + 1), members_.begin() + *result); - } - } - } - build_index(); - } - - template - void insert(sorted_unique_range_tag, InputIt first, InputIt last, Convert convert) - { - size_t count = std::distance(first,last); - - members_.reserve(members_.size() + count); - for (auto s = first; s != last; ++s) - { - members_.emplace_back(convert(*s)); - } - - build_index(); - } - - template - typename std::enable_if::value,std::pair>::type - insert_or_assign(const string_view_type& name, T&& value) - { - auto result = insert_index_entry(name,members_.size()); - if (result.second) - { - members_.emplace_back(key_type(name.begin(), name.end()), std::forward(value)); - auto it = members_.begin() + result.first; - return std::make_pair(it,true); - } - else - { - auto it = members_.begin() + result.first; - it->value(Json(std::forward(value))); - return std::make_pair(it,false); - } - } - - template - typename std::enable_if::value,std::pair>::type - insert_or_assign(const string_view_type& name, T&& value) - { - auto result = insert_index_entry(name,members_.size()); - if (result.second) - { - members_.emplace_back(key_type(name.begin(),name.end(),get_allocator()), - std::forward(value),get_allocator()); - auto it = members_.begin() + result.first; - return std::make_pair(it,true); - } - else - { - auto it = members_.begin() + result.first; - it->value(Json(std::forward(value),get_allocator())); - return std::make_pair(it,false); - } - } - - template - typename std::enable_if::value,iterator>::type - insert_or_assign(iterator hint, const string_view_type& key, T&& value) - { - if (hint == members_.end()) - { - auto result = insert_or_assign(key, std::forward(value)); - return result.first; - } - else - { - size_t pos = hint - members_.begin(); - auto result = insert_index_entry(key,pos); - - if (result.second) - { - auto it = members_.emplace(hint, key_type(key.begin(), key.end()), std::forward(value)); - return it; - } - else - { - auto it = members_.begin() + result.first; - it->value(Json(std::forward(value))); - return it; - } - } - } - - template - typename std::enable_if::value,iterator>::type - insert_or_assign(iterator hint, const string_view_type& key, T&& value) - { - if (hint == members_.end()) - { - auto result = insert_or_assign(key, std::forward(value)); - return result.first; - } - else - { - size_t pos = hint - members_.begin(); - auto result = insert_index_entry(key,pos); - - if (result.second) - { - auto it = members_.emplace(hint, - key_type(key.begin(),key.end(),get_allocator()), - std::forward(value),get_allocator()); - return it; - } - else - { - auto it = members_.begin() + result.first; - it->value(Json(std::forward(value),get_allocator())); - return it; - } - } - } - - // merge - - void merge(const json_object& source) - { - for (auto it = source.begin(); it != source.end(); ++it) - { - try_emplace(it->key(),it->value()); - } - } - - void merge(json_object&& source) - { - auto it = std::make_move_iterator(source.begin()); - auto end = std::make_move_iterator(source.end()); - for (; it != end; ++it) - { - auto pos = find(it->key()); - if (pos == members_.end() ) - { - try_emplace(it->key(),std::move(it->value())); - } - } - } - - void merge(iterator hint, const json_object& source) - { - size_t pos = hint - members_.begin(); - for (auto it = source.begin(); it != source.end(); ++it) - { - hint = try_emplace(hint, it->key(),it->value()); - size_t newpos = hint - members_.begin(); - if (newpos == pos) - { - ++hint; - pos = hint - members_.begin(); - } - else - { - hint = members_.begin() + pos; - } - } - } - - void merge(iterator hint, json_object&& source) - { - size_t pos = hint - members_.begin(); - - auto it = std::make_move_iterator(source.begin()); - auto end = std::make_move_iterator(source.end()); - for (; it != end; ++it) - { - hint = try_emplace(hint, it->key(), std::move(it->value())); - size_t newpos = hint - members_.begin(); - if (newpos == pos) - { - ++hint; - pos = hint - members_.begin(); - } - else - { - hint = members_.begin() + pos; - } - } - } - - // merge_or_update - - void merge_or_update(const json_object& source) - { - for (auto it = source.begin(); it != source.end(); ++it) - { - insert_or_assign(it->key(),it->value()); - } - } - - void merge_or_update(json_object&& source) - { - auto it = std::make_move_iterator(source.begin()); - auto end = std::make_move_iterator(source.end()); - for (; it != end; ++it) - { - auto pos = find(it->key()); - if (pos == members_.end() ) - { - insert_or_assign(it->key(),std::move(it->value())); - } - else - { - pos->value(std::move(it->value())); - } - } - } - - void merge_or_update(iterator hint, const json_object& source) - { - size_t pos = hint - members_.begin(); - for (auto it = source.begin(); it != source.end(); ++it) - { - hint = insert_or_assign(hint, it->key(),it->value()); - size_t newpos = hint - members_.begin(); - if (newpos == pos) - { - ++hint; - pos = hint - members_.begin(); - } - else - { - hint = members_.begin() + pos; - } - } - } - - void merge_or_update(iterator hint, json_object&& source) - { -/* - auto it = std::make_move_iterator(source.begin()); - auto end = std::make_move_iterator(source.end()); - for (; it != end; ++it) - { - auto pos = find(it->key()); - if (pos == members_.end() ) - { - hint = try_emplace(hint,it->key(),std::move(it->value())); - } - else - { - pos->value(std::move(it->value())); - hint = pos; - } - } -*/ - size_t pos = hint - members_.begin(); - auto it = std::make_move_iterator(source.begin()); - auto end = std::make_move_iterator(source.end()); - for (; it != end; ++it) - { - hint = insert_or_assign(hint, it->key(),std::move(it->value())); - size_t newpos = hint - members_.begin(); - if (newpos == pos) - { - ++hint; - pos = hint - members_.begin(); - } - else - { - hint = members_.begin() + pos; - } - } - } - - // try_emplace - - template - typename std::enable_if::value,std::pair>::type - try_emplace(const string_view_type& name, Args&&... args) - { - auto result = insert_index_entry(name,members_.size()); - if (result.second) - { - members_.emplace_back(key_type(name.begin(), name.end()), std::forward(args)...); - auto it = members_.begin() + result.first; - return std::make_pair(it,true); - } - else - { - auto it = members_.begin() + result.first; - return std::make_pair(it,false); - } - } - - template - typename std::enable_if::value,std::pair>::type - try_emplace(const string_view_type& key, Args&&... args) - { - auto result = insert_index_entry(key,members_.size()); - if (result.second) - { - members_.emplace_back(key_type(key.begin(),key.end(), get_allocator()), - std::forward(args)...); - auto it = members_.begin() + result.first; - return std::make_pair(it,true); - } - else - { - auto it = members_.begin() + result.first; - return std::make_pair(it,false); - } - } - - template - typename std::enable_if::value,iterator>::type - try_emplace(iterator hint, const string_view_type& key, Args&&... args) - { - if (hint == members_.end()) - { - auto result = try_emplace(key, std::forward(args)...); - return result.first; - } - else - { - size_t pos = hint - members_.begin(); - auto result = insert_index_entry(key, pos); - - if (result.second) - { - auto it = members_.emplace(hint, key_type(key.begin(), key.end()), std::forward(args)...); - return it; - } - else - { - auto it = members_.begin() + result.first; - return it; - } - } - } - - template - typename std::enable_if::value,iterator>::type - try_emplace(iterator hint, const string_view_type& key, Args&&... args) - { - if (hint == members_.end()) - { - auto result = try_emplace(key, std::forward(args)...); - return result.first; - } - else - { - size_t pos = hint - members_.begin(); - auto result = insert_index_entry(key, pos); - - if (result.second) - { - auto it = members_.emplace(hint, - key_type(key.begin(),key.end(), get_allocator()), - std::forward(args)...); - return it; - } - else - { - auto it = members_.begin() + result.first; - return it; - } - } - } - - bool operator==(const json_object& rhs) const - { - return members_ == rhs.members_; - } - - bool operator<(const json_object& rhs) const - { - return members_ < rhs.members_; - } -private: - - std::pair insert_index_entry(const string_view_type& key, size_t pos) - { - JSONCONS_ASSERT(pos <= index_.size()); - - auto it = std::lower_bound(index_.begin(),index_.end(), key, - [&](size_t i, const string_view_type& k){return members_.at(i).key().compare(k) < 0;}); - - if (it == index_.end()) - { - size_t count = index_.size() - pos; - for (size_t i = 0; count > 0 && i < index_.size(); ++i) - { - if (index_[i] >= pos) - { - ++index_[i]; - --count; - } - } - index_.push_back(pos); - return std::make_pair(pos,true); - } - else if (members_.at(*it).key() != key) - { - size_t count = index_.size() - pos; - for (size_t i = 0; count > 0 && i < index_.size(); ++i) - { - if (index_[i] >= pos) - { - ++index_[i]; - --count; - } - } - auto it2 = index_.insert(it, pos); - return std::make_pair(*it2,true); - } - else - { - return std::make_pair(*it,false); - } - } - - void erase_index_entry(const string_view_type& key) - { - auto it = std::lower_bound(index_.begin(),index_.end(), key, - [&](size_t i, const string_view_type& k){return members_.at(i).key().compare(k) < 0;}); - - if (it != index_.end() && members_.at(*it).key() != key) - { - size_t pos = *it; - size_t count = index_.size() - pos; - for (size_t i = 0; i < index_.size() && count > 0; ++i) - { - if (index_[i] > pos) - { - --index_[i]; - --count; - } - } - index_.erase(it); - } - } - - void erase_index_entries(size_t pos1, size_t pos2) - { - for (size_t i = 0; i < index_.size(); ++i) - { - if (index_[i] >= pos1 && index_[i] < pos2) - { - index_.erase(index_.begin()+index_[i]); - } - else if (index_[i] > pos2) - { - --index_[i]; - } - } - } - - void build_index() - { - index_.clear(); - index_.reserve(members_.size()); - for (size_t i = 0; i < members_.size(); ++i) - { - index_.push_back(i); - } - std::stable_sort(index_.begin(),index_.end(), - [&](size_t a, size_t b){return members_.at(a).key().compare(members_.at(b).key()) < 0;}); - } - - json_object& operator=(const json_object&) = delete; -}; - -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_content_handler.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_content_handler.hpp deleted file mode 100644 index 131e2b6729..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_content_handler.hpp +++ /dev/null @@ -1,431 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSON_CONTENT_HANDLER_HPP -#define JSONCONS_JSON_CONTENT_HANDLER_HPP - -#include -#include -#include -#include -#include - -namespace jsoncons { - -// null_type - -struct null_type -{ -}; - -enum class semantic_tag : uint8_t -{ - none = 0, - undefined = 0x01, - datetime = 0x02, - timestamp = 0x03, - bigint = 0x04, - bigdec = 0x05, - bigfloat = 0x06, - base16 = 0x07, - base64 = 0x08, - base64url = 0x09, - uri = 0x0a -#if !defined(JSONCONS_NO_DEPRECATED) - , big_integer = bigint - , big_decimal = bigdec - , big_float = bigfloat - , date_time = datetime -#endif -}; - -#if !defined(JSONCONS_NO_DEPRECATED) - typedef semantic_tag semantic_tag_type; -#endif - -template -class basic_json_content_handler -{ -#if !defined(JSONCONS_NO_DEPRECATED) - std::basic_string buffer_; -#endif -public: - typedef CharT char_type; - typedef std::char_traits char_traits_type; - - typedef basic_string_view string_view_type; - - basic_json_content_handler(basic_json_content_handler&&) = default; - - basic_json_content_handler& operator=(basic_json_content_handler&&) = default; - - basic_json_content_handler() = default; - - virtual ~basic_json_content_handler() {} - - void flush() - { - do_flush(); - } - - bool begin_object(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()) - { - return do_begin_object(tag, context); - } - - bool begin_object(size_t length, - semantic_tag tag=semantic_tag::none, - const ser_context& context = null_ser_context()) - { - return do_begin_object(length, tag, context); - } - - bool end_object(const ser_context& context = null_ser_context()) - { - return do_end_object(context); - } - - bool begin_array(semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()) - { - return do_begin_array(tag, context); - } - - bool begin_array(size_t length, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()) - { - return do_begin_array(length, tag, context); - } - - bool end_array(const ser_context& context=null_ser_context()) - { - return do_end_array(context); - } - - bool name(const string_view_type& name, const ser_context& context=null_ser_context()) - { - return do_name(name, context); - } - - bool string_value(const string_view_type& value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()) - { - return do_string_value(value, tag, context); - } - - bool byte_string_value(const byte_string_view& b, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()) - { - return do_byte_string_value(b, tag, context); - } - - bool byte_string_value(const uint8_t* p, size_t size, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()) - { - return do_byte_string_value(byte_string(p, size), tag, context); - } -#if !defined(JSONCONS_NO_DEPRECATED) - bool byte_string_value(const byte_string_view& b, - byte_string_chars_format encoding_hint, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()) - { - switch (encoding_hint) - { - case byte_string_chars_format::base16: - tag = semantic_tag::base16; - break; - case byte_string_chars_format::base64: - tag = semantic_tag::base64; - break; - case byte_string_chars_format::base64url: - tag = semantic_tag::base64url; - break; - default: - break; - } - return do_byte_string_value(b, tag, context); - } - - bool byte_string_value(const uint8_t* p, size_t size, - byte_string_chars_format encoding_hint, - semantic_tag tag=semantic_tag::none, - const ser_context& context=null_ser_context()) - { - switch (encoding_hint) - { - case byte_string_chars_format::base16: - tag = semantic_tag::base16; - break; - case byte_string_chars_format::base64: - tag = semantic_tag::base64; - break; - case byte_string_chars_format::base64url: - tag = semantic_tag::base64url; - break; - default: - break; - } - return do_byte_string_value(byte_string(p, size), tag, context); - } - bool big_integer_value(const string_view_type& s, const ser_context& context=null_ser_context()) - { - return do_string_value(s, semantic_tag::bigint, context); - } - - bool big_decimal_value(const string_view_type& s, const ser_context& context=null_ser_context()) - { - return do_string_value(s, semantic_tag::bigdec, context); - } - - bool date_time_value(const string_view_type& s, const ser_context& context=null_ser_context()) - { - return do_string_value(s, semantic_tag::datetime, context); - } - - bool timestamp_value(int64_t val, const ser_context& context=null_ser_context()) - { - return do_int64_value(val, semantic_tag::timestamp, context); - } -#endif - - bool int64_value(int64_t value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()) - { - return do_int64_value(value, tag, context); - } - - bool uint64_value(uint64_t value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()) - { - return do_uint64_value(value, tag, context); - } - - bool double_value(double value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()) - { - return do_double_value(value, tag, context); - } - - bool bool_value(bool value, - semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()) - { - return do_bool_value(value, tag, context); - } - - bool null_value(semantic_tag tag = semantic_tag::none, - const ser_context& context=null_ser_context()) - { - return do_null_value(tag, context); - } - -#if !defined(JSONCONS_NO_DEPRECATED) - - bool begin_document() - { - return true; - } - - bool end_document() - { - flush(); - return true; - } - - void begin_json() - { - } - - void end_json() - { - end_document(); - } - - void name(const CharT* p, size_t length, const ser_context& context) - { - name(string_view_type(p, length), context); - } - - void integer_value(int64_t value) - { - int64_value(value); - } - - void integer_value(int64_t value, const ser_context& context) - { - int64_value(value,context); - } - - void uinteger_value(uint64_t value) - { - uint64_value(value); - } - - void uinteger_value(uint64_t value, const ser_context& context) - { - uint64_value(value,context); - } - - bool bignum_value(const string_view_type& s, const ser_context& context=null_ser_context()) - { - return do_string_value(s, semantic_tag::bigint, context); - } - - bool decimal_value(const string_view_type& s, const ser_context& context=null_ser_context()) - { - return do_string_value(s, semantic_tag::bigdec, context); - } - - bool epoch_time_value(int64_t val, const ser_context& context=null_ser_context()) - { - return do_int64_value(val, semantic_tag::timestamp, context); - } - -#endif - -private: - virtual void do_flush() = 0; - - virtual bool do_begin_object(semantic_tag, const ser_context& context) = 0; - - virtual bool do_begin_object(size_t, semantic_tag tag, const ser_context& context) - { - return do_begin_object(tag, context); - } - - virtual bool do_end_object(const ser_context& context) = 0; - - virtual bool do_begin_array(semantic_tag, const ser_context& context) = 0; - - virtual bool do_begin_array(size_t, semantic_tag tag, const ser_context& context) - { - return do_begin_array(tag, context); - } - - virtual bool do_end_array(const ser_context& context) = 0; - - virtual bool do_name(const string_view_type& name, const ser_context& context) = 0; - - virtual bool do_null_value(semantic_tag, const ser_context& context) = 0; - - virtual bool do_string_value(const string_view_type& value, semantic_tag tag, const ser_context& context) = 0; - - virtual bool do_byte_string_value(const byte_string_view& b, - semantic_tag tag, - const ser_context& context) = 0; - - virtual bool do_double_value(double value, - semantic_tag tag, - const ser_context& context) = 0; - - virtual bool do_int64_value(int64_t value, - semantic_tag tag, - const ser_context& context) = 0; - - virtual bool do_uint64_value(uint64_t value, - semantic_tag tag, - const ser_context& context) = 0; - - virtual bool do_bool_value(bool value, semantic_tag tag, const ser_context& context) = 0; -}; - -template -class basic_null_json_content_handler final : public basic_json_content_handler -{ -public: - using typename basic_json_content_handler::string_view_type; -private: - void do_flush() override - { - } - - bool do_begin_object(semantic_tag, const ser_context&) override - { - return true; - } - - bool do_end_object(const ser_context&) override - { - return true; - } - - bool do_begin_array(semantic_tag, const ser_context&) override - { - return true; - } - - bool do_end_array(const ser_context&) override - { - return true; - } - - bool do_name(const string_view_type&, const ser_context&) override - { - return true; - } - - bool do_null_value(semantic_tag, const ser_context&) override - { - return true; - } - - bool do_string_value(const string_view_type&, semantic_tag, const ser_context&) override - { - return true; - } - - bool do_byte_string_value(const byte_string_view&, - semantic_tag, - const ser_context&) override - { - return true; - } - - bool do_int64_value(int64_t, - semantic_tag, - const ser_context&) override - { - return true; - } - - bool do_uint64_value(uint64_t, - semantic_tag, - const ser_context&) override - { - return true; - } - - bool do_double_value(double, - semantic_tag, - const ser_context&) override - { - return true; - } - - bool do_bool_value(bool, semantic_tag, const ser_context&) override - { - return true; - } -}; - -typedef basic_json_content_handler json_content_handler; -typedef basic_json_content_handler wjson_content_handler; - -typedef basic_null_json_content_handler null_json_content_handler; -typedef basic_null_json_content_handler wnull_json_content_handler; - -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_conversion_traits.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_conversion_traits.hpp deleted file mode 100644 index 2aee25a233..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_conversion_traits.hpp +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSON_CONVERSION_TRAITS_HPP -#define JSONCONS_JSON_CONVERSION_TRAITS_HPP - -#include -#include -#include -#include -#include // std::enable_if -#include -#include -#include -#include -#include -#include -#include - -namespace jsoncons { - -template -void read_from(const Json& j, basic_staj_reader& reader, T& val, std::error_code& ec); - -template -void read_from(const Json& j, basic_staj_reader& reader, T& val) -{ - std::error_code ec; - read_from(j, reader, val, ec); - if (ec) - { - throw ser_error(ec, reader.context().line(), reader.context().column()); - } -} - -template -void write_to(const T&val, basic_json_content_handler& receiver); - -} // namespace jsoncons - -#include - -namespace jsoncons { - -template -struct json_conversion_traits -{ - template - static T decode(basic_staj_reader& reader, std::error_code& ec) - { - json_decoder decoder; - reader.accept(decoder, ec); - return decoder.get_result().template as(); - } - - template - static void encode(const T& val, basic_json_content_handler& receiver) - { - auto j = json_type_traits::to_json(val); - j.dump(receiver); - } -}; - -// specializations - -// vector like - -template -struct json_conversion_traits::value && jsoncons::detail::is_vector_like::value ->::type> -{ - typedef typename T::value_type value_type; - - template - static T decode(basic_staj_reader& reader, std::error_code& ec) - { - T v; - basic_staj_array_iterator end; - basic_staj_array_iterator it(reader, ec); - - while (it != end && !ec) - { - v.push_back(*it); - it.increment(ec); - } - return v; - } - - template - static void encode(const T& val, basic_json_content_handler& receiver) - { - receiver.begin_array(); - for (auto it = std::begin(val); it != std::end(val); ++it) - { - json_conversion_traits::template encode(*it,receiver); - } - receiver.end_array(); - receiver.flush(); - } -}; -// std::array - -template -struct json_conversion_traits> -{ - typedef typename std::array::value_type value_type; - - template - static std::array decode(basic_staj_reader& reader, std::error_code& ec) - { - std::array v; - v.fill(T{}); - basic_staj_array_iterator end; - basic_staj_array_iterator it(reader, ec); - - for (size_t i = 0; it != end && i < N && !ec; ++i) - { - v[i] = *it; - it.increment(ec); - } - return v; - } - - template - static void encode(const std::array& val, basic_json_content_handler& receiver) - { - receiver.begin_array(); - for (auto it = std::begin(val); it != std::end(val); ++it) - { - json_conversion_traits::template encode(*it,receiver); - } - receiver.end_array(); - receiver.flush(); - } -}; - -// map like - -template -struct json_conversion_traits::value && jsoncons::detail::is_map_like::value ->::type> -{ - typedef typename T::mapped_type mapped_type; - typedef typename T::value_type value_type; - typedef typename T::key_type key_type; - - template - static T decode(basic_staj_reader& reader, std::error_code& ec) - { - T m; - basic_staj_object_iterator end; - basic_staj_object_iterator it(reader, ec); - - while (it != end && !ec) - { - m.emplace(it->first,it->second); - it.increment(ec); - } - return m; - } - - template - static void encode(const T& val, basic_json_content_handler& receiver) - { - receiver.begin_object(); - for (auto it = std::begin(val); it != std::end(val); ++it) - { - receiver.name(it->first); - json_conversion_traits::template encode(it->second,receiver); - } - receiver.end_object(); - receiver.flush(); - } -}; - -template -void read_from(const Json&, basic_staj_reader& reader, T& val, std::error_code& ec) -{ - val = json_conversion_traits::template decode(reader,ec); -} - -template -void write_to(const Json&, const T&val, basic_json_content_handler& receiver) -{ - json_conversion_traits::template encode(val, receiver); -} - -} - -#endif - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_cursor.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_cursor.hpp deleted file mode 100644 index 8659eb4911..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_cursor.hpp +++ /dev/null @@ -1,680 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSON_CURSOR_HPP -#define JSONCONS_JSON_CURSOR_HPP - -#include // std::allocator -#include -#include -#include -#include -#include -#include // std::basic_istream -#include -#include -#include -#include -#include -#include -#include -#include - -namespace jsoncons { - -template -class basic_staj_event_handler final : public basic_json_content_handler -{ -public: - using typename basic_json_content_handler::string_view_type; -private: - basic_staj_event event_; -public: - basic_staj_event_handler() - : event_(staj_event_type::null_value) - { - } - - basic_staj_event_handler(staj_event_type event_type) - : event_(event_type) - { - } - - const basic_staj_event& event() const - { - return event_; - } -private: - - bool do_begin_object(semantic_tag, const ser_context&) override - { - event_ = basic_staj_event(staj_event_type::begin_object); - return false; - } - - bool do_end_object(const ser_context&) override - { - event_ = basic_staj_event(staj_event_type::end_object); - return false; - } - - bool do_begin_array(semantic_tag, const ser_context&) override - { - event_ = basic_staj_event(staj_event_type::begin_array); - return false; - } - - bool do_end_array(const ser_context&) override - { - event_ = basic_staj_event(staj_event_type::end_array); - return false; - } - - bool do_name(const string_view_type& name, const ser_context&) override - { - event_ = basic_staj_event(name.data(), name.length(), staj_event_type::name); - return false; - } - - bool do_null_value(semantic_tag, const ser_context&) override - { - event_ = basic_staj_event(staj_event_type::null_value); - return false; - } - - bool do_bool_value(bool value, semantic_tag, const ser_context&) override - { - event_ = basic_staj_event(value); - return false; - } - - bool do_string_value(const string_view_type& s, semantic_tag tag, const ser_context&) override - { - event_ = basic_staj_event(s.data(), s.length(), staj_event_type::string_value, tag); - return false; - } - - bool do_byte_string_value(const byte_string_view&, - semantic_tag, - const ser_context&) override - { - JSONCONS_UNREACHABLE(); - } - - bool do_int64_value(int64_t value, - semantic_tag tag, - const ser_context&) override - { - event_ = basic_staj_event(value, tag); - return false; - } - - bool do_uint64_value(uint64_t value, - semantic_tag tag, - const ser_context&) override - { - event_ = basic_staj_event(value, tag); - return false; - } - - bool do_double_value(double value, - semantic_tag tag, - const ser_context&) override - { - event_ = basic_staj_event(value, tag); - return false; - } - - void do_flush() override - { - } -}; - -template,class Allocator=std::allocator> -class basic_json_cursor : public basic_staj_reader, private virtual ser_context -{ -public: - typedef Src source_type; - typedef CharT char_type; - typedef Allocator allocator_type; -private: - static const size_t default_max_buffer_length = 16384; - - basic_staj_event_handler event_handler_; - default_parse_error_handler default_err_handler_; - - default_basic_staj_filter default_filter_; - - typedef typename std::allocator_traits:: template rebind_alloc char_allocator_type; - - basic_json_parser parser_; - basic_staj_filter& filter_; - source_type source_; - std::vector buffer_; - size_t buffer_length_; - bool eof_; - bool begin_; - - // Noncopyable and nonmoveable - basic_json_cursor(const basic_json_cursor&) = delete; - basic_json_cursor& operator=(const basic_json_cursor&) = delete; - -public: - typedef basic_string_view string_view_type; - - // Constructors that throw parse exceptions - template - basic_json_cursor(Source&& source) - : basic_json_cursor(std::forward(source), - default_filter_, - basic_json_options::default_options(), - default_err_handler_) - { - } - - template - basic_json_cursor(Source&& source, - basic_staj_filter& filter) - : basic_json_cursor(std::forward(source), - filter,basic_json_options::default_options(), - default_err_handler_) - { - } - - template - basic_json_cursor(Source&& source, - parse_error_handler& err_handler) - : basic_json_cursor(std::forward(source), - default_filter_, - basic_json_options::default_options(), - err_handler) - { - } - - template - basic_json_cursor(Source&& source, - basic_staj_filter& filter, - parse_error_handler& err_handler) - : basic_json_cursor(std::forward(source), - filter, - basic_json_options::default_options(), - err_handler) - { - } - - template - basic_json_cursor(Source&& source, - const basic_json_decode_options& options) - : basic_json_cursor(std::forward(source), - default_filter_, - options, - default_err_handler_) - { - } - - template - basic_json_cursor(Source&& source, - basic_staj_filter& filter, - const basic_json_decode_options& options) - : basic_json_cursor(std::forward(source), - filter, - options, - default_err_handler_) - { - } - - template - basic_json_cursor(Source&& source, - basic_staj_filter& filter, - const basic_json_decode_options& options, - parse_error_handler& err_handler, - typename std::enable_if,Source>::value>::type* = 0) - : parser_(options,err_handler), - filter_(filter), - source_(source), - buffer_length_(default_max_buffer_length), - eof_(false), - begin_(true) - { - buffer_.reserve(buffer_length_); - if (!done()) - { - next(); - } - } - - template - basic_json_cursor(Source&& source, - basic_staj_filter& filter, - const basic_json_decode_options& options, - parse_error_handler& err_handler, - typename std::enable_if,Source>::value>::type* = 0) - : parser_(options,err_handler), - filter_(filter), - buffer_length_(0), - eof_(false), - begin_(false) - { - basic_string_view sv(std::forward(source)); - auto result = unicons::skip_bom(sv.begin(), sv.end()); - if (result.ec != unicons::encoding_errc()) - { - throw ser_error(result.ec,parser_.line(),parser_.column()); - } - size_t offset = result.it - sv.begin(); - parser_.update(sv.data()+offset,sv.size()-offset); - if (!done()) - { - next(); - } - } - - // Constructors that set parse error codes - template - basic_json_cursor(Source&& source, - std::error_code& ec) - : basic_json_cursor(std::forward(source),default_filter_,basic_json_options::default_options(),default_err_handler_,ec) - { - } - - template - basic_json_cursor(Source&& source, - basic_staj_filter& filter, - std::error_code& ec) - : basic_json_cursor(std::forward(source), - filter, - basic_json_options::default_options(), - default_err_handler_, - ec) - { - } - - template - basic_json_cursor(Source&& source, - parse_error_handler& err_handler, - std::error_code& ec) - : basic_json_cursor(std::forward(source), - default_filter_, - basic_json_options::default_options(), - err_handler, - ec) - { - } - - template - basic_json_cursor(Source&& source, - basic_staj_filter& filter, - parse_error_handler& err_handler, - std::error_code& ec) - : basic_json_cursor(std::forward(source), - filter, - basic_json_options::default_options(), - err_handler, - ec) - { - } - - template - basic_json_cursor(Source&& source, - const basic_json_decode_options& options, - std::error_code& ec) - : basic_json_cursor(std::forward(source), - default_filter_, - options, - default_err_handler_, - ec) - { - } - - template - basic_json_cursor(Source&& source, - basic_staj_filter& filter, - const basic_json_decode_options& options, - std::error_code& ec) - : basic_json_cursor(std::forward(source), - filter,options, - default_err_handler_, - ec) - { - } - - template - basic_json_cursor(Source&& source, - basic_staj_filter& filter, - const basic_json_decode_options& options, - parse_error_handler& err_handler, - std::error_code& ec, - typename std::enable_if,Source>::value>::type* = 0) - : parser_(options,err_handler), - filter_(filter), - source_(source), - eof_(false), - buffer_length_(default_max_buffer_length), - begin_(true) - { - buffer_.reserve(buffer_length_); - if (!done()) - { - next(ec); - } - } - - template - basic_json_cursor(Source&& source, - basic_staj_filter& filter, - const basic_json_decode_options& options, - parse_error_handler& err_handler, - std::error_code& ec, - typename std::enable_if,Source>::value>::type* = 0) - : parser_(options,err_handler), - filter_(filter), - eof_(false), - buffer_length_(0), - begin_(false) - { - basic_string_view sv(std::forward(source)); - auto result = unicons::skip_bom(sv.begin(), sv.end()); - if (result.ec != unicons::encoding_errc()) - { - ec = result.ec; - return; - } - size_t offset = result.it - sv.begin(); - parser_.update(sv.data()+offset,sv.size()-offset); - if (!done()) - { - next(ec); - } - } - - size_t buffer_length() const - { - return buffer_length_; - } - - void buffer_length(size_t length) - { - buffer_length_ = length; - buffer_.reserve(buffer_length_); - } - - bool done() const override - { - return parser_.done(); - } - - const basic_staj_event& current() const override - { - return event_handler_.event(); - } - - void accept(basic_json_content_handler& handler) override - { - std::error_code ec; - accept(handler, ec); - if (ec) - { - throw ser_error(ec,parser_.line(),parser_.column()); - } - } - - void accept(basic_json_content_handler& handler, - std::error_code& ec) override - { - switch (event_handler_.event().event_type()) - { - case staj_event_type::begin_array: - if (!handler.begin_array(semantic_tag::none, *this)) - { - return; - } - break; - case staj_event_type::end_array: - if (!handler.end_array(*this)) - { - return; - } - break; - case staj_event_type::begin_object: - if (!handler.begin_object(semantic_tag::none, *this)) - { - return; - } - break; - case staj_event_type::end_object: - if (!handler.end_object(*this)) - { - return; - } - break; - case staj_event_type::name: - if (!handler.name(event_handler_.event().template as>(), *this)) - { - return; - } - break; - case staj_event_type::string_value: - if (!handler.string_value(event_handler_.event().template as>(), semantic_tag::none, *this)) - { - return; - } - break; - case staj_event_type::null_value: - if (!handler.null_value(semantic_tag::none, *this)) - { - return; - } - break; - case staj_event_type::bool_value: - if (!handler.bool_value(event_handler_.event().template as(), semantic_tag::none, *this)) - { - return; - } - break; - case staj_event_type::int64_value: - if (!handler.int64_value(event_handler_.event().template as(), semantic_tag::none, *this)) - { - return; - } - break; - case staj_event_type::uint64_value: - if (!handler.uint64_value(event_handler_.event().template as(), semantic_tag::none, *this)) - { - return; - } - break; - case staj_event_type::double_value: - if (!handler.double_value(event_handler_.event().template as(), semantic_tag::none, *this)) - { - return; - } - break; - default: - break; - } - do - { - read_next(handler, ec); - } - while (!ec && !done() && !filter_.accept(event_handler_.event(), *this)); - } - - void next() override - { - std::error_code ec; - next(ec); - if (ec) - { - throw ser_error(ec,parser_.line(),parser_.column()); - } - } - - void next(std::error_code& ec) override - { - do - { - read_next(ec); - } - while (!ec && !done() && !filter_.accept(event_handler_.event(), *this)); - } - - void read_buffer(std::error_code& ec) - { - buffer_.clear(); - buffer_.resize(buffer_length_); - size_t count = source_.read(buffer_.data(), buffer_length_); - buffer_.resize(static_cast(count)); - if (buffer_.size() == 0) - { - eof_ = true; - } - else if (begin_) - { - auto result = unicons::skip_bom(buffer_.begin(), buffer_.end()); - if (result.ec != unicons::encoding_errc()) - { - ec = result.ec; - return; - } - size_t offset = result.it - buffer_.begin(); - parser_.update(buffer_.data()+offset,buffer_.size()-offset); - begin_ = false; - } - else - { - parser_.update(buffer_.data(),buffer_.size()); - } - } - - void read_next(std::error_code& ec) - { - read_next(event_handler_, ec); - } - - void read_next(basic_json_content_handler& handler, std::error_code& ec) - { - parser_.restart(); - while (!parser_.stopped()) - { - if (parser_.source_exhausted()) - { - if (!source_.eof()) - { - read_buffer(ec); - if (ec) return; - } - else - { - eof_ = true; - } - } - parser_.parse_some(handler, ec); - if (ec) return; - } - } - - void check_done() - { - std::error_code ec; - check_done(ec); - if (ec) - { - throw ser_error(ec,parser_.line(),parser_.column()); - } - } - - const ser_context& context() const override - { - return *this; - } - - void check_done(std::error_code& ec) - { - try - { - if (source_.is_error()) - { - ec = json_errc::source_error; - return; - } - if (eof_) - { - parser_.check_done(ec); - if (ec) return; - } - else - { - while (!eof_) - { - if (parser_.source_exhausted()) - { - if (!source_.eof()) - { - read_buffer(ec); - if (ec) return; - } - else - { - eof_ = true; - } - } - if (!eof_) - { - parser_.check_done(ec); - if (ec) return; - } - } - } - } - catch (const ser_error& e) - { - ec = e.code(); - } - } - - bool eof() const - { - return eof_; - } - - size_t line() const override - { - return parser_.line(); - } - - size_t column() const override - { - return parser_.column(); - } -private: -}; - -typedef basic_json_cursor json_cursor; -typedef basic_json_cursor wjson_cursor; - -#if !defined(JSONCONS_NO_DEPRECATED) -template> -using basic_json_pull_reader = basic_json_cursor; -typedef basic_json_cursor json_pull_reader; -typedef basic_json_cursor wjson_pull_reader; - -template> -using basic_json_stream_reader = basic_json_cursor; - -template> -using basic_json_staj_reader = basic_json_cursor; - -typedef basic_json_cursor json_stream_reader; -typedef basic_json_cursor wjson_stream_reader; - -typedef basic_json_cursor json_staj_reader; -typedef basic_json_cursor wjson_staj_reader; -#endif - -} - -#endif - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_decoder.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_decoder.hpp deleted file mode 100644 index 653d48addc..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_decoder.hpp +++ /dev/null @@ -1,357 +0,0 @@ -// Copyright 2013-2016 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSON_DECODER_HPP -#define JSONCONS_JSON_DECODER_HPP - -#include -#include -#include // std::true_type -#include // std::allocator -#include // std::make_move_iterator -#include // std::move -#include -#include - -namespace jsoncons { - -template > -class json_decoder final : public basic_json_content_handler -{ -public: - typedef typename Json::char_type char_type; - using typename basic_json_content_handler::string_view_type; - - typedef typename Json::key_value_type key_value_type; - typedef typename Json::string_type string_type; - typedef typename Json::array array; - typedef typename Json::object object; - typedef typename Json::allocator_type json_allocator_type; - typedef typename string_type::allocator_type json_string_allocator; - typedef typename array::allocator_type json_array_allocator; - typedef typename object::allocator_type json_object_allocator; - typedef typename std::allocator_traits:: template rebind_alloc json_byte_allocator_type; -private: - json_string_allocator string_allocator_; - json_object_allocator object_allocator_; - json_array_allocator array_allocator_; - json_byte_allocator_type byte_allocator_; - - Json result_; - - struct stack_item - { - template - stack_item(std::true_type, Args&& ... args) - : name_(std::forward(args)...) - { - } - template - stack_item(std::false_type, Args&& ... args) - : value_(std::forward(args)...) - { - } - - stack_item() = default; - stack_item(const stack_item&) = default; - stack_item(stack_item&&) = default; - stack_item& operator=(const stack_item&) = default; - stack_item& operator=(stack_item&&) = default; - - string_type name_; - Json value_; - }; - - enum class container_type {root_t, array_t, object_t}; - - struct structure_offset - { - size_t offset_; - container_type type_; - }; - - typedef Allocator allocator_type; - typedef typename std::allocator_traits:: template rebind_alloc stack_item_allocator_type; - typedef typename std::allocator_traits:: template rebind_alloc size_t_allocator_type; - - - std::vector stack_; - std::vector stack_offsets_; - bool is_valid_; - -public: - json_decoder(const json_allocator_type& jallocator = json_allocator_type()) - : string_allocator_(jallocator), - object_allocator_(jallocator), - array_allocator_(jallocator), - is_valid_(false) - - { - stack_.reserve(1000); - stack_offsets_.reserve(100); - stack_offsets_.push_back({0,container_type::root_t}); - } - - bool is_valid() const - { - return is_valid_; - } - - Json get_result() - { - is_valid_ = false; - return std::move(result_); - } - -#if !defined(JSONCONS_NO_DEPRECATED) - Json& root() - { - return result_; - } -#endif - -private: - - void do_flush() override - { - } - - bool do_begin_object(semantic_tag tag, const ser_context&) override - { - switch (stack_offsets_.back().type_) - { - case container_type::object_t: - stack_.back().value_ = Json(object(object_allocator_), tag); - break; - case container_type::array_t: - stack_.emplace_back(std::false_type(), object(object_allocator_), tag); - break; - case container_type::root_t: - stack_.clear(); - is_valid_ = false; - stack_.emplace_back(std::false_type(), object(object_allocator_), tag); - break; - } - stack_offsets_.push_back({stack_.size()-1,container_type::object_t}); - return true; - } - - bool do_end_object(const ser_context&) override - { - JSONCONS_ASSERT(stack_offsets_.size() > 0); - JSONCONS_ASSERT(stack_offsets_.back().type_ == container_type::object_t); - const size_t structure_index = stack_offsets_.back().offset_; - JSONCONS_ASSERT(stack_.size() > structure_index); - const size_t count = stack_.size() - (structure_index + 1); - auto first = stack_.begin() + (structure_index+1); - auto last = first + count; - stack_[structure_index].value_.object_value().insert( - std::make_move_iterator(first), - std::make_move_iterator(last), - [](stack_item&& val){return key_value_type(std::move(val.name_), std::move(val.value_));} - ); - stack_.erase(stack_.begin()+structure_index+1, stack_.end()); - stack_offsets_.pop_back(); - if (stack_offsets_.back().type_ == container_type::root_t) - { - result_.swap(stack_.front().value_); - stack_.pop_back(); - is_valid_ = true; - return false; - } - return true; - } - - bool do_begin_array(semantic_tag tag, const ser_context&) override - { - switch (stack_offsets_.back().type_) - { - case container_type::object_t: - stack_.back().value_ = Json(array(array_allocator_), tag); - break; - case container_type::array_t: - stack_.emplace_back(std::false_type(), array(array_allocator_), tag); - break; - case container_type::root_t: - stack_.clear(); - is_valid_ = false; - stack_.emplace_back(std::false_type(), array(array_allocator_), tag); - break; - } - stack_offsets_.push_back({stack_.size()-1,container_type::array_t}); - return true; - } - - bool do_end_array(const ser_context&) override - { - JSONCONS_ASSERT(stack_offsets_.size() > 0); - JSONCONS_ASSERT(stack_offsets_.back().type_ == container_type::array_t); - const size_t structure_index = stack_offsets_.back().offset_; - JSONCONS_ASSERT(stack_.size() > structure_index); - const size_t count = stack_.size() - (structure_index + 1); - auto first = stack_.begin() + (structure_index+1); - auto last = first + count; - auto& j = stack_[structure_index].value_; - j.reserve(count); - while (first != last) - { - j.push_back(std::move(first->value_)); - ++first; - } - stack_.erase(stack_.begin()+structure_index+1, stack_.end()); - stack_offsets_.pop_back(); - if (stack_offsets_.back().type_ == container_type::root_t) - { - result_.swap(stack_.front().value_); - stack_.pop_back(); - is_valid_ = true; - return false; - } - return true; - } - - bool do_name(const string_view_type& name, const ser_context&) override - { - stack_.emplace_back(std::true_type(), name.data(), name.length(), string_allocator_); - return true; - } - - bool do_string_value(const string_view_type& sv, semantic_tag tag, const ser_context&) override - { - switch (stack_offsets_.back().type_) - { - case container_type::object_t: - stack_.back().value_ = Json(sv, tag, string_allocator_); - break; - case container_type::array_t: - stack_.emplace_back(std::false_type(), sv, tag, string_allocator_); - break; - case container_type::root_t: - result_ = Json(sv, tag, string_allocator_); - is_valid_ = true; - return false; - } - return true; - } - - bool do_byte_string_value(const byte_string_view& b, semantic_tag tag, const ser_context&) override - { - switch (stack_offsets_.back().type_) - { - case container_type::object_t: - stack_.back().value_ = Json(b, tag, byte_allocator_); - break; - case container_type::array_t: - stack_.emplace_back(std::false_type(), b, tag, byte_allocator_); - break; - case container_type::root_t: - result_ = Json(b, tag, byte_allocator_); - is_valid_ = true; - return false; - } - return true; - } - - bool do_int64_value(int64_t value, - semantic_tag tag, - const ser_context&) override - { - switch (stack_offsets_.back().type_) - { - case container_type::object_t: - stack_.back().value_ = Json(value,tag); - break; - case container_type::array_t: - stack_.emplace_back(std::false_type(), value, tag); - break; - case container_type::root_t: - result_ = Json(value,tag); - is_valid_ = true; - return false; - } - return true; - } - - bool do_uint64_value(uint64_t value, - semantic_tag tag, - const ser_context&) override - { - switch (stack_offsets_.back().type_) - { - case container_type::object_t: - stack_.back().value_ = Json(value,tag); - break; - case container_type::array_t: - stack_.emplace_back(std::false_type(), value, tag); - break; - case container_type::root_t: - result_ = Json(value,tag); - is_valid_ = true; - return false; - } - return true; - } - - bool do_double_value(double value, - semantic_tag tag, - const ser_context&) override - { - switch (stack_offsets_.back().type_) - { - case container_type::object_t: - stack_.back().value_ = Json(value, tag); - break; - case container_type::array_t: - stack_.emplace_back(std::false_type(), value, tag); - break; - case container_type::root_t: - result_ = Json(value, tag); - is_valid_ = true; - return false; - } - return true; - } - - bool do_bool_value(bool value, semantic_tag tag, const ser_context&) override - { - switch (stack_offsets_.back().type_) - { - case container_type::object_t: - stack_.back().value_ = Json(value, tag); - break; - case container_type::array_t: - stack_.emplace_back(std::false_type(), value, tag); - break; - case container_type::root_t: - result_ = Json(value, tag); - is_valid_ = true; - return false; - } - return true; - } - - bool do_null_value(semantic_tag tag, const ser_context&) override - { - switch (stack_offsets_.back().type_) - { - case container_type::object_t: - stack_.back().value_ = Json(null_type(),tag); - break; - case container_type::array_t: - stack_.emplace_back(std::false_type(), null_type(), tag); - break; - case container_type::root_t: - result_ = Json(null_type(), tag); - is_valid_ = true; - return false; - } - return true; - } -}; - -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_encoder.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_encoder.hpp deleted file mode 100644 index 356348c15a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_encoder.hpp +++ /dev/null @@ -1,1470 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSON_ENCODER_HPP -#define JSONCONS_JSON_ENCODER_HPP - -#include // std::array -#include -#include -#include // std::isfinite, std::isnan -#include // std::numeric_limits -#include -#include // std::move -#include -#include -#include -#include -#include -#include -#include -#include - -namespace jsoncons { namespace detail { -template -size_t escape_string(const CharT* s, size_t length, - bool escape_all_non_ascii, bool escape_solidus, - Result& result) -{ - size_t count = 0; - const CharT* begin = s; - const CharT* end = s + length; - for (const CharT* it = begin; it != end; ++it) - { - CharT c = *it; - switch (c) - { - case '\\': - result.push_back('\\'); - result.push_back('\\'); - count += 2; - break; - case '"': - result.push_back('\\'); - result.push_back('\"'); - count += 2; - break; - case '\b': - result.push_back('\\'); - result.push_back('b'); - count += 2; - break; - case '\f': - result.push_back('\\'); - result.push_back('f'); - count += 2; - break; - case '\n': - result.push_back('\\'); - result.push_back('n'); - count += 2; - break; - case '\r': - result.push_back('\\'); - result.push_back('r'); - count += 2; - break; - case '\t': - result.push_back('\\'); - result.push_back('t'); - count += 2; - break; - default: - if (escape_solidus && c == '/') - { - result.push_back('\\'); - result.push_back('/'); - count += 2; - } - else if (is_control_character(c) || escape_all_non_ascii) - { - // convert utf8 to codepoint - unicons::sequence_generator g(it, end, unicons::conv_flags::strict); - if (g.done() || g.status() != unicons::conv_errc()) - { - throw ser_error(json_errc::illegal_codepoint); - } - uint32_t cp = g.get().codepoint(); - it += (g.get().length() - 1); - if (is_non_ascii_codepoint(cp) || is_control_character(c)) - { - if (cp > 0xFFFF) - { - cp -= 0x10000; - uint32_t first = (cp >> 10) + 0xD800; - uint32_t second = ((cp & 0x03FF) + 0xDC00); - - result.push_back('\\'); - result.push_back('u'); - result.push_back(to_hex_character(first >> 12 & 0x000F)); - result.push_back(to_hex_character(first >> 8 & 0x000F)); - result.push_back(to_hex_character(first >> 4 & 0x000F)); - result.push_back(to_hex_character(first & 0x000F)); - result.push_back('\\'); - result.push_back('u'); - result.push_back(to_hex_character(second >> 12 & 0x000F)); - result.push_back(to_hex_character(second >> 8 & 0x000F)); - result.push_back(to_hex_character(second >> 4 & 0x000F)); - result.push_back(to_hex_character(second & 0x000F)); - count += 12; - } - else - { - result.push_back('\\'); - result.push_back('u'); - result.push_back(to_hex_character(cp >> 12 & 0x000F)); - result.push_back(to_hex_character(cp >> 8 & 0x000F)); - result.push_back(to_hex_character(cp >> 4 & 0x000F)); - result.push_back(to_hex_character(cp & 0x000F)); - count += 6; - } - } - else - { - result.push_back(c); - ++count; - } - } - else - { - result.push_back(c); - ++count; - } - break; - } - } - return count; -} - -inline -byte_string_chars_format resolve_byte_string_chars_format(byte_string_chars_format format1, - byte_string_chars_format format2, - byte_string_chars_format default_format = byte_string_chars_format::base64url) -{ - byte_string_chars_format result; - switch (format1) - { - case byte_string_chars_format::base16: - case byte_string_chars_format::base64: - case byte_string_chars_format::base64url: - result = format1; - break; - default: - switch (format2) - { - case byte_string_chars_format::base64url: - case byte_string_chars_format::base64: - case byte_string_chars_format::base16: - result = format2; - break; - default: // base64url - { - result = default_format; - break; - } - } - break; - } - return result; -} - -}} - -namespace jsoncons { - -template> -class basic_json_encoder final : public basic_json_content_handler -{ - static const std::array& null_k() - { - static constexpr std::array k{'n','u','l','l'}; - return k; - } - static const std::array& true_k() - { - static constexpr std::array k{'t','r','u','e'}; - return k; - } - static const std::array& false_k() - { - static constexpr std::array k{'f','a','l','s','e'}; - return k; - } -public: - typedef CharT char_type; - using typename basic_json_content_handler::string_view_type; - typedef Result result_type; - typedef typename basic_json_options::string_type string_type; - -private: - enum class container_type {object, array}; - - class encoding_context - { - container_type type_; - size_t count_; - line_split_kind line_splits_; - bool indent_before_; - bool new_line_after_; - size_t begin_pos_; - size_t data_pos_; - public: - encoding_context(container_type type, line_split_kind split_lines, bool indent_once, - size_t begin_pos, size_t data_pos) - : type_(type), count_(0), line_splits_(split_lines), indent_before_(indent_once), new_line_after_(false), - begin_pos_(begin_pos), data_pos_(data_pos) - { - } - - void set_position(size_t pos) - { - data_pos_ = pos; - } - - size_t begin_pos() const - { - return begin_pos_; - } - - size_t data_pos() const - { - return data_pos_; - } - - size_t count() const - { - return count_; - } - - void increment_count() - { - ++count_; - } - - bool new_line_after() const - { - return new_line_after_; - } - - void new_line_after(bool value) - { - new_line_after_ = value; - } - - bool is_object() const - { - return type_ == container_type::object; - } - - bool is_array() const - { - return type_ == container_type::array; - } - - bool is_same_line() const - { - return line_splits_ == line_split_kind::same_line; - } - - bool is_new_line() const - { - return line_splits_ == line_split_kind::new_line; - } - - bool is_multi_line() const - { - return line_splits_ == line_split_kind::multi_line; - } - - bool is_indent_once() const - { - return count_ == 0 ? indent_before_ : false; - } - - }; - - size_t indent_size_; - - const basic_json_encode_options& options_; - - jsoncons::detail::print_double fp_; - - Result result_; - - std::vector stack_; - int indent_amount_; - size_t column_; - std::basic_string colon_str_; - std::basic_string comma_str_; - std::basic_string open_object_brace_str_; - std::basic_string close_object_brace_str_; - std::basic_string open_array_bracket_str_; - std::basic_string close_array_bracket_str_; - - // Noncopyable and nonmoveable - basic_json_encoder(const basic_json_encoder&) = delete; - basic_json_encoder& operator=(const basic_json_encoder&) = delete; -public: - basic_json_encoder(result_type result) - : basic_json_encoder(std::move(result), basic_json_options::default_options()) - { - } - - basic_json_encoder(result_type result, - const basic_json_encode_options& options) - : options_(options), - fp_(floating_point_options(options.floating_point_format(), - options.precision(), - 0)), - result_(std::move(result)), - indent_amount_(0), - column_(0) - { - switch (options.spaces_around_colon()) - { - case spaces_option::space_after: - colon_str_ = std::basic_string({':',' '}); - break; - case spaces_option::space_before: - colon_str_ = std::basic_string({' ',':'}); - break; - case spaces_option::space_before_and_after: - colon_str_ = std::basic_string({' ',':',' '}); - break; - default: - colon_str_.push_back(':'); - break; - } - switch (options.spaces_around_comma()) - { - case spaces_option::space_after: - comma_str_ = std::basic_string({',',' '}); - break; - case spaces_option::space_before: - comma_str_ = std::basic_string({' ',','}); - break; - case spaces_option::space_before_and_after: - comma_str_ = std::basic_string({' ',',',' '}); - break; - default: - comma_str_.push_back(','); - break; - } - if (options.pad_inside_object_braces()) - { - open_object_brace_str_ = std::basic_string({'{', ' '}); - close_object_brace_str_ = std::basic_string({' ', '}'}); - } - else - { - open_object_brace_str_.push_back('{'); - close_object_brace_str_.push_back('}'); - } - if (options.pad_inside_array_brackets()) - { - open_array_bracket_str_ = std::basic_string({'[', ' '}); - close_array_bracket_str_ = std::basic_string({' ', ']'}); - } - else - { - open_array_bracket_str_.push_back('['); - close_array_bracket_str_.push_back(']'); - } - } - - ~basic_json_encoder() - { - try - { - result_.flush(); - } - catch (...) - { - } - } - -private: - // Implementing methods - void do_flush() override - { - result_.flush(); - } - - bool do_begin_object(semantic_tag, const ser_context&) override - { - if (!stack_.empty() && stack_.back().is_array() && stack_.back().count() > 0) - { - result_.append(comma_str_.data(),comma_str_.length()); - column_ += comma_str_.length(); - } - - if (!stack_.empty()) // object or array - { - if (stack_.back().is_object()) - { - switch (options_.object_object_line_splits()) - { - case line_split_kind::same_line: - if (column_ >= options_.line_length_limit()) - { - break_line(); - } - break; - case line_split_kind::new_line: - if (column_ >= options_.line_length_limit()) - { - break_line(); - } - break; - default: // multi_line - break; - } - stack_.emplace_back(container_type::object,options_.object_object_line_splits(), false, - column_, column_+open_object_brace_str_.length()); - } - else // array - { - switch (options_.array_object_line_splits()) - { - case line_split_kind::same_line: - if (column_ >= options_.line_length_limit()) - { - //stack_.back().new_line_after(true); - new_line(); - } - break; - case line_split_kind::new_line: - stack_.back().new_line_after(true); - new_line(); - break; - default: // multi_line - stack_.back().new_line_after(true); - new_line(); - break; - } - stack_.emplace_back(container_type::object,options_.array_object_line_splits(), false, - column_, column_+open_object_brace_str_.length()); - } - } - else - { - stack_.emplace_back(container_type::object, line_split_kind::multi_line, false, - column_, column_+open_object_brace_str_.length()); - } - indent(); - - result_.append(open_object_brace_str_.data(), open_object_brace_str_.length()); - column_ += open_object_brace_str_.length(); - return true; - } - - bool do_end_object(const ser_context&) override - { - JSONCONS_ASSERT(!stack_.empty()); - unindent(); - if (stack_.back().new_line_after()) - { - new_line(); - } - stack_.pop_back(); - result_.append(close_object_brace_str_.data(), close_object_brace_str_.length()); - column_ += close_object_brace_str_.length(); - - end_value(); - return true; - } - - bool do_begin_array(semantic_tag, const ser_context&) override - { - if (!stack_.empty() && stack_.back().is_array() && stack_.back().count() > 0) - { - result_.append(comma_str_.data(),comma_str_.length()); - column_ += comma_str_.length(); - } - if (!stack_.empty()) - { - if (stack_.back().is_object()) - { - switch (options_.object_array_line_splits()) - { - case line_split_kind::same_line: - stack_.emplace_back(container_type::array,options_.object_array_line_splits(),false, - column_, column_ + open_array_bracket_str_.length()); - break; - case line_split_kind::new_line: - { - stack_.emplace_back(container_type::array,options_.object_array_line_splits(),true, - column_, column_+open_array_bracket_str_.length()); - break; - } - default: // multi_line - stack_.emplace_back(container_type::array,options_.object_array_line_splits(),true, - column_, column_+open_array_bracket_str_.length()); - break; - } - } - else // array - { - switch (options_.array_array_line_splits()) - { - case line_split_kind::same_line: - if (stack_.back().is_multi_line()) - { - stack_.back().new_line_after(true); - new_line(); - } - stack_.emplace_back(container_type::array,options_.array_array_line_splits(), false, - column_, column_+open_array_bracket_str_.length()); - break; - case line_split_kind::new_line: - stack_.back().new_line_after(true); - new_line(); - stack_.emplace_back(container_type::array,options_.array_array_line_splits(), false, - column_, column_+open_array_bracket_str_.length()); - break; - default: // multi_line - stack_.back().new_line_after(true); - new_line(); - stack_.emplace_back(container_type::array,options_.array_array_line_splits(), false, - column_, column_+open_array_bracket_str_.length()); - //new_line(); - break; - } - } - } - else - { - stack_.emplace_back(container_type::array, line_split_kind::multi_line, false, - column_, column_+open_array_bracket_str_.length()); - } - indent(); - result_.append(open_array_bracket_str_.data(), open_array_bracket_str_.length()); - column_ += open_array_bracket_str_.length(); - return true; - } - - bool do_end_array(const ser_context&) override - { - JSONCONS_ASSERT(!stack_.empty()); - unindent(); - if (stack_.back().new_line_after()) - { - new_line(); - } - stack_.pop_back(); - result_.append(close_array_bracket_str_.data(), close_array_bracket_str_.length()); - column_ += close_array_bracket_str_.length(); - end_value(); - return true; - } - - bool do_name(const string_view_type& name, const ser_context&) override - { - JSONCONS_ASSERT(!stack_.empty()); - if (stack_.back().count() > 0) - { - result_.append(comma_str_.data(),comma_str_.length()); - column_ += comma_str_.length(); - } - - if (stack_.back().is_multi_line()) - { - stack_.back().new_line_after(true); - new_line(); - } - else if (stack_.back().count() > 0 && column_ >= options_.line_length_limit()) - { - //stack_.back().new_line_after(true); - new_line(stack_.back().data_pos()); - } - - if (stack_.back().count() == 0) - { - stack_.back().set_position(column_); - } - result_.push_back('\"'); - size_t length = jsoncons::detail::escape_string(name.data(), name.length(),options_.escape_all_non_ascii(),options_.escape_solidus(),result_); - result_.push_back('\"'); - result_.append(colon_str_.data(),colon_str_.length()); - column_ += (length+2+colon_str_.length()); - return true; - } - - bool do_null_value(semantic_tag, const ser_context&) override - { - if (!stack_.empty()) - { - if (stack_.back().is_array()) - { - begin_scalar_value(); - } - if (!stack_.back().is_multi_line() && column_ >= options_.line_length_limit()) - { - break_line(); - } - } - - result_.append(null_k().data(), null_k().size()); - column_ += null_k().size(); - - end_value(); - return true; - } - - bool do_string_value(const string_view_type& sv, semantic_tag tag, const ser_context&) override - { - if (!stack_.empty()) - { - if (stack_.back().is_array()) - { - begin_scalar_value(); - } - if (!stack_.back().is_multi_line() && column_ >= options_.line_length_limit()) - { - break_line(); - } - } - - switch (tag) - { - case semantic_tag::bigint: - write_bigint_value(sv); - break; - default: - { - result_.push_back('\"'); - size_t length = jsoncons::detail::escape_string(sv.data(), sv.length(),options_.escape_all_non_ascii(),options_.escape_solidus(),result_); - result_.push_back('\"'); - column_ += (length+2); - break; - } - } - - end_value(); - return true; - } - - bool do_byte_string_value(const byte_string_view& b, - semantic_tag tag, - const ser_context&) override - { - if (!stack_.empty()) - { - if (stack_.back().is_array()) - { - begin_scalar_value(); - } - if (!stack_.back().is_multi_line() && column_ >= options_.line_length_limit()) - { - break_line(); - } - } - - byte_string_chars_format encoding_hint; - switch (tag) - { - case semantic_tag::base16: - encoding_hint = byte_string_chars_format::base16; - break; - case semantic_tag::base64: - encoding_hint = byte_string_chars_format::base64; - break; - case semantic_tag::base64url: - encoding_hint = byte_string_chars_format::base64url; - break; - default: - encoding_hint = byte_string_chars_format::none; - break; - } - - byte_string_chars_format format = jsoncons::detail::resolve_byte_string_chars_format(options_.byte_string_format(), - encoding_hint, - byte_string_chars_format::base64url); - switch (format) - { - case byte_string_chars_format::base16: - { - result_.push_back('\"'); - size_t length = encode_base16(b.begin(),b.end(),result_); - result_.push_back('\"'); - column_ += (length + 2); - break; - } - case byte_string_chars_format::base64: - { - result_.push_back('\"'); - size_t length = encode_base64(b.begin(), b.end(), result_); - result_.push_back('\"'); - column_ += (length + 2); - break; - } - case byte_string_chars_format::base64url: - { - result_.push_back('\"'); - size_t length = encode_base64url(b.begin(),b.end(),result_); - result_.push_back('\"'); - column_ += (length + 2); - break; - } - default: - { - JSONCONS_UNREACHABLE(); - } - } - - end_value(); - return true; - } - - bool do_double_value(double value, - semantic_tag, - const ser_context& context) override - { - if (!stack_.empty()) - { - if (stack_.back().is_array()) - { - begin_scalar_value(); - } - if (!stack_.back().is_multi_line() && column_ >= options_.line_length_limit()) - { - break_line(); - } - } - - if (!std::isfinite(value)) - { - if ((std::isnan)(value)) - { - if (options_.is_nan_to_num()) - { - result_.append(options_.nan_to_num().data(), options_.nan_to_num().length()); - column_ += options_.nan_to_num().length(); - } - else if (options_.is_nan_to_str()) - { - do_string_value(options_.nan_to_str(), semantic_tag::none, context); - } - else - { - result_.append(null_k().data(), null_k().size()); - column_ += null_k().size(); - } - } - else if (value == std::numeric_limits::infinity()) - { - if (options_.is_inf_to_num()) - { - result_.append(options_.inf_to_num().data(), options_.inf_to_num().length()); - column_ += options_.inf_to_num().length(); - } - else if (options_.is_inf_to_str()) - { - do_string_value(options_.inf_to_str(), semantic_tag::none, context); - } - else - { - result_.append(null_k().data(), null_k().size()); - column_ += null_k().size(); - } - } - else - { - if (options_.is_neginf_to_num()) - { - result_.append(options_.neginf_to_num().data(), options_.neginf_to_num().length()); - column_ += options_.neginf_to_num().length(); - } - else if (options_.is_neginf_to_str()) - { - do_string_value(options_.neginf_to_str(), semantic_tag::none, context); - } - else - { - result_.append(null_k().data(), null_k().size()); - column_ += null_k().size(); - } - } - } - else - { - size_t length = fp_(value, result_); - column_ += length; - } - - end_value(); - return true; - } - - bool do_int64_value(int64_t value, - semantic_tag, - const ser_context&) override - { - if (!stack_.empty()) - { - if (stack_.back().is_array()) - { - begin_scalar_value(); - } - if (!stack_.back().is_multi_line() && column_ >= options_.line_length_limit()) - { - break_line(); - } - } - size_t length = jsoncons::detail::print_integer(value, result_); - column_ += length; - end_value(); - return true; - } - - bool do_uint64_value(uint64_t value, - semantic_tag, - const ser_context&) override - { - if (!stack_.empty()) - { - if (stack_.back().is_array()) - { - begin_scalar_value(); - } - if (!stack_.back().is_multi_line() && column_ >= options_.line_length_limit()) - { - break_line(); - } - } - size_t length = jsoncons::detail::print_uinteger(value, result_); - column_ += length; - end_value(); - return true; - } - - bool do_bool_value(bool value, semantic_tag, const ser_context&) override - { - if (!stack_.empty()) - { - if (stack_.back().is_array()) - { - begin_scalar_value(); - } - if (!stack_.back().is_multi_line() && column_ >= options_.line_length_limit()) - { - break_line(); - } - } - - if (value) - { - result_.append(true_k().data(), true_k().size()); - column_ += true_k().size(); - } - else - { - result_.append(false_k().data(), false_k().size()); - column_ += false_k().size(); - } - - end_value(); - return true; - } - - void begin_scalar_value() - { - if (!stack_.empty()) - { - if (stack_.back().count() > 0) - { - result_.append(comma_str_.data(),comma_str_.length()); - column_ += comma_str_.length(); - } - if (stack_.back().is_multi_line() || stack_.back().is_indent_once()) - { - stack_.back().new_line_after(true); - new_line(); - } - } - } - - void write_bigint_value(const string_view_type& sv) - { - switch (options_.bigint_format()) - { - case bigint_chars_format::number: - { - result_.append(sv.data(),sv.size()); - column_ += sv.size(); - break; - } - case bigint_chars_format::base64: - { - bignum n(sv.data(), sv.length()); - int signum; - std::vector v; - n.dump(signum, v); - - result_.push_back('\"'); - if (signum == -1) - { - result_.push_back('~'); - ++column_; - } - size_t length = encode_base64(v.begin(), v.end(), result_); - result_.push_back('\"'); - column_ += (length+2); - break; - } - case bigint_chars_format::base64url: - { - bignum n(sv.data(), sv.length()); - int signum; - std::vector v; - n.dump(signum, v); - - result_.push_back('\"'); - if (signum == -1) - { - result_.push_back('~'); - ++column_; - } - size_t length = encode_base64url(v.begin(), v.end(), result_); - result_.push_back('\"'); - column_ += (length+2); - break; - } - default: - { - result_.push_back('\"'); - result_.append(sv.data(),sv.size()); - result_.push_back('\"'); - column_ += (sv.size() + 2); - break; - } - } - } - - void end_value() - { - if (!stack_.empty()) - { - stack_.back().increment_count(); - } - } - - void indent() - { - indent_amount_ += static_cast(options_.indent_size()); - } - - void unindent() - { - indent_amount_ -= static_cast(options_.indent_size()); - } - - void new_line() - { - result_.append(options_.new_line_chars().data(),options_.new_line_chars().length()); - for (int i = 0; i < indent_amount_; ++i) - { - result_.push_back(' '); - } - column_ = indent_amount_; - } - - void new_line(size_t len) - { - result_.append(options_.new_line_chars().data(),options_.new_line_chars().length()); - for (size_t i = 0; i < len; ++i) - { - result_.push_back(' '); - } - column_ = len; - } - - void break_line() - { - stack_.back().new_line_after(true); - new_line(); - } -}; - -template> -class basic_json_compressed_encoder final : public basic_json_content_handler -{ - static const std::array& null_k() - { - static constexpr std::array k{'n','u','l','l'}; - return k; - } - static const std::array& true_k() - { - static constexpr std::array k{'t','r','u','e'}; - return k; - } - static const std::array& false_k() - { - static constexpr std::array k{'f','a','l','s','e'}; - return k; - } -public: - typedef CharT char_type; - using typename basic_json_content_handler::string_view_type; - typedef Result result_type; - typedef typename basic_json_options::string_type string_type; - -private: - enum class container_type {object, array}; - - class encoding_context - { - container_type type_; - size_t count_; - public: - encoding_context(container_type type) - : type_(type), count_(0) - { - } - - size_t count() const - { - return count_; - } - - void increment_count() - { - ++count_; - } - - bool is_array() const - { - return type_ == container_type::array; - } - }; - - const basic_json_encode_options& options_; - - std::vector stack_; - jsoncons::detail::print_double fp_; - Result result_; - - // Noncopyable and nonmoveable - basic_json_compressed_encoder(const basic_json_compressed_encoder&) = delete; - basic_json_compressed_encoder& operator=(const basic_json_compressed_encoder&) = delete; -public: - basic_json_compressed_encoder(result_type result) - : basic_json_compressed_encoder(std::move(result), basic_json_options::default_options()) - { - } - - basic_json_compressed_encoder(result_type result, - const basic_json_encode_options& options) - : options_(options), - fp_(floating_point_options(options.floating_point_format(), - options.precision(), - 0)), - result_(std::move(result)) - { - } - - ~basic_json_compressed_encoder() - { - try - { - result_.flush(); - } - catch (...) - { - } - } - - -private: - // Implementing methods - void do_flush() override - { - result_.flush(); - } - - bool do_begin_object(semantic_tag, const ser_context&) override - { - if (!stack_.empty() && stack_.back().is_array() && stack_.back().count() > 0) - { - result_.push_back(','); - } - - stack_.emplace_back(container_type::object); - result_.push_back('{'); - return true; - } - - bool do_end_object(const ser_context&) override - { - JSONCONS_ASSERT(!stack_.empty()); - stack_.pop_back(); - result_.push_back('}'); - - if (!stack_.empty()) - { - stack_.back().increment_count(); - } - return true; - } - - - bool do_begin_array(semantic_tag, const ser_context&) override - { - if (!stack_.empty() && stack_.back().is_array() && stack_.back().count() > 0) - { - result_.push_back(','); - } - stack_.emplace_back(container_type::array); - result_.push_back('['); - return true; - } - - bool do_end_array(const ser_context&) override - { - JSONCONS_ASSERT(!stack_.empty()); - stack_.pop_back(); - result_.push_back(']'); - if (!stack_.empty()) - { - stack_.back().increment_count(); - } - return true; - } - - bool do_name(const string_view_type& name, const ser_context&) override - { - if (!stack_.empty() && stack_.back().count() > 0) - { - result_.push_back(','); - } - - result_.push_back('\"'); - jsoncons::detail::escape_string(name.data(), name.length(),options_.escape_all_non_ascii(),options_.escape_solidus(),result_); - result_.push_back('\"'); - result_.push_back(':'); - return true; - } - - bool do_null_value(semantic_tag, const ser_context&) override - { - if (!stack_.empty() && stack_.back().is_array() && stack_.back().count() > 0) - { - result_.push_back(','); - } - - result_.append(null_k().data(), null_k().size()); - - if (!stack_.empty()) - { - stack_.back().increment_count(); - } - return true; - } - - void write_bigint_value(const string_view_type& sv) - { - switch (options_.bigint_format()) - { - case bigint_chars_format::number: - { - result_.append(sv.data(),sv.size()); - break; - } - case bigint_chars_format::base64: - { - bignum n(sv.data(), sv.length()); - int signum; - std::vector v; - n.dump(signum, v); - - result_.push_back('\"'); - if (signum == -1) - { - result_.push_back('~'); - } - encode_base64(v.begin(), v.end(), result_); - result_.push_back('\"'); - break; - } - case bigint_chars_format::base64url: - { - bignum n(sv.data(), sv.length()); - int signum; - std::vector v; - n.dump(signum, v); - - result_.push_back('\"'); - if (signum == -1) - { - result_.push_back('~'); - } - encode_base64url(v.begin(), v.end(), result_); - result_.push_back('\"'); - break; - } - default: - { - result_.push_back('\"'); - result_.append(sv.data(),sv.size()); - result_.push_back('\"'); - break; - } - } - } - - bool do_string_value(const string_view_type& sv, semantic_tag tag, const ser_context&) override - { - if (!stack_.empty() && stack_.back().is_array() && stack_.back().count() > 0) - { - result_.push_back(','); - } - - switch (tag) - { - case semantic_tag::bigint: - write_bigint_value(sv); - break; - default: - { - result_.push_back('\"'); - jsoncons::detail::escape_string(sv.data(), sv.length(),options_.escape_all_non_ascii(),options_.escape_solidus(),result_); - result_.push_back('\"'); - break; - } - } - - if (!stack_.empty()) - { - stack_.back().increment_count(); - } - return true; - } - - bool do_byte_string_value(const byte_string_view& b, - semantic_tag tag, - const ser_context&) override - { - if (!stack_.empty() && stack_.back().is_array() && stack_.back().count() > 0) - { - result_.push_back(','); - } - - byte_string_chars_format encoding_hint; - switch (tag) - { - case semantic_tag::base16: - encoding_hint = byte_string_chars_format::base16; - break; - case semantic_tag::base64: - encoding_hint = byte_string_chars_format::base64; - break; - case semantic_tag::base64url: - encoding_hint = byte_string_chars_format::base64url; - break; - default: - encoding_hint = byte_string_chars_format::none; - break; - } - - byte_string_chars_format format = jsoncons::detail::resolve_byte_string_chars_format(options_.byte_string_format(), - encoding_hint, - byte_string_chars_format::base64url); - switch (format) - { - case byte_string_chars_format::base16: - { - result_.push_back('\"'); - encode_base16(b.begin(),b.end(),result_); - result_.push_back('\"'); - break; - } - case byte_string_chars_format::base64: - { - result_.push_back('\"'); - encode_base64(b.begin(), b.end(), result_); - result_.push_back('\"'); - break; - } - case byte_string_chars_format::base64url: - { - result_.push_back('\"'); - encode_base64url(b.begin(),b.end(),result_); - result_.push_back('\"'); - break; - } - default: - { - JSONCONS_UNREACHABLE(); - } - } - - if (!stack_.empty()) - { - stack_.back().increment_count(); - } - return true; - } - - bool do_double_value(double value, - semantic_tag, - const ser_context& context) override - { - if (!stack_.empty() && stack_.back().is_array() && stack_.back().count() > 0) - { - result_.push_back(','); - } - - if (JSONCONS_UNLIKELY(!std::isfinite(value))) - { - if ((std::isnan)(value)) - { - if (options_.is_nan_to_num()) - { - result_.append(options_.nan_to_num().data(), options_.nan_to_num().length()); - } - else if (options_.is_nan_to_str()) - { - do_string_value(options_.nan_to_str(), semantic_tag::none, context); - } - else - { - result_.append(null_k().data(), null_k().size()); - } - } - else if (value == std::numeric_limits::infinity()) - { - if (options_.is_inf_to_num()) - { - result_.append(options_.inf_to_num().data(), options_.inf_to_num().length()); - } - else if (options_.is_inf_to_str()) - { - do_string_value(options_.inf_to_str(), semantic_tag::none, context); - } - else - { - result_.append(null_k().data(), null_k().size()); - } - } - else - { - if (options_.is_neginf_to_num()) - { - result_.append(options_.neginf_to_num().data(), options_.neginf_to_num().length()); - } - else if (options_.is_neginf_to_str()) - { - do_string_value(options_.neginf_to_str(), semantic_tag::none, context); - } - else - { - result_.append(null_k().data(), null_k().size()); - } - } - } - else - { - fp_(value, result_); - } - - if (!stack_.empty()) - { - stack_.back().increment_count(); - } - return true; - } - - bool do_int64_value(int64_t value, - semantic_tag, - const ser_context&) override - { - if (!stack_.empty() && stack_.back().is_array() && stack_.back().count() > 0) - { - result_.push_back(','); - } - jsoncons::detail::print_integer(value, result_); - if (!stack_.empty()) - { - stack_.back().increment_count(); - } - return true; - } - - bool do_uint64_value(uint64_t value, - semantic_tag, - const ser_context&) override - { - if (!stack_.empty() && stack_.back().is_array() && stack_.back().count() > 0) - { - result_.push_back(','); - } - jsoncons::detail::print_uinteger(value, result_); - if (!stack_.empty()) - { - stack_.back().increment_count(); - } - return true; - } - - bool do_bool_value(bool value, semantic_tag, const ser_context&) override - { - if (!stack_.empty() && stack_.back().is_array() && stack_.back().count() > 0) - { - result_.push_back(','); - } - - if (value) - { - result_.append(true_k().data(), true_k().size()); - } - else - { - result_.append(false_k().data(), false_k().size()); - } - - if (!stack_.empty()) - { - stack_.back().increment_count(); - } - return true; - } -}; - -typedef basic_json_encoder> json_encoder; -typedef basic_json_encoder> wjson_encoder; - -typedef basic_json_compressed_encoder> json_compressed_encoder; -typedef basic_json_compressed_encoder> wjson_compressed_encoder; - -typedef basic_json_encoder> json_string_encoder; -typedef basic_json_encoder> wjson_string_encoder; - -typedef basic_json_compressed_encoder> json_compressed_string_encoder; -typedef basic_json_compressed_encoder> wjson_compressed_string_encoder; - -#if !defined(JSONCONS_NO_DEPRECATED) -template> -using basic_json_serializer = basic_json_encoder; - -template> -using basic_json_compressed_serializer = basic_json_compressed_encoder; - -typedef basic_json_serializer> json_serializer; -typedef basic_json_serializer> wjson_encoder; - -typedef basic_json_compressed_serializer> json_compressed_serializer; -typedef basic_json_compressed_serializer> wjson_compressed_serializer; - -typedef basic_json_serializer> json_string_serializer; -typedef basic_json_serializer> wjson_string_serializer; - -typedef basic_json_compressed_serializer> json_compressed_string_serializer; -typedef basic_json_compressed_serializer> wjson_compressed_string_serializer; -#endif - -} -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_error.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_error.hpp deleted file mode 100644 index 90410a41a1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_error.hpp +++ /dev/null @@ -1,154 +0,0 @@ -/// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSON_ERROR_HPP -#define JSONCONS_JSON_ERROR_HPP - -#include -#include - -namespace jsoncons { - - enum class json_errc - { - ok = 0, - unexpected_eof = 1, - source_error, - invalid_json_text, - extra_character, - max_depth_exceeded, - single_quote, - illegal_character_in_string, - extra_comma, - expected_name, - expected_value, - invalid_value, - expected_colon, - illegal_control_character, - illegal_escaped_character, - expected_codepoint_surrogate_pair, - invalid_hex_escape_sequence, - invalid_unicode_escape_sequence, - leading_zero, - invalid_number, - expected_comma_or_right_brace, - expected_comma_or_right_bracket, - unexpected_right_bracket, - unexpected_right_brace, - illegal_comment, - expected_continuation_byte, - over_long_utf8_sequence, - illegal_codepoint, - illegal_surrogate_value, - unpaired_high_surrogate - }; - -class json_error_category_impl - : public std::error_category -{ -public: - const char* name() const noexcept override - { - return "jsoncons/json"; - } - std::string message(int ev) const override - { - switch (static_cast(ev)) - { - case json_errc::unexpected_eof: - return "Unexpected end of file"; - case json_errc::source_error: - return "Source error"; - case json_errc::invalid_json_text: - return "Invalid JSON text"; - case json_errc::extra_character: - return "Unexpected non-whitespace character after JSON text"; - case json_errc::max_depth_exceeded: - return "Maximum JSON depth exceeded"; - case json_errc::single_quote: - return "JSON strings cannot be quoted with single quotes"; - case json_errc::illegal_character_in_string: - return "Illegal character in string"; - case json_errc::extra_comma: - return "Extra comma"; - case json_errc::expected_name: - return "Expected object member name"; - case json_errc::expected_value: - return "Expected value"; - case json_errc::invalid_value: - return "Invalid value"; - case json_errc::expected_colon: - return "Expected name separator ':'"; - case json_errc::illegal_control_character: - return "Illegal control character in string"; - case json_errc::illegal_escaped_character: - return "Illegal escaped character in string"; - case json_errc::expected_codepoint_surrogate_pair: - return "Invalid codepoint, expected another \\u token to begin the second half of a codepoint surrogate pair."; - case json_errc::invalid_hex_escape_sequence: - return "Invalid codepoint, expected hexadecimal digit."; - case json_errc::invalid_unicode_escape_sequence: - return "Invalid codepoint, expected four hexadecimal digits."; - case json_errc::leading_zero: - return "A number cannot have a leading zero"; - case json_errc::invalid_number: - return "Invalid number"; - case json_errc::expected_comma_or_right_brace: - return "Expected comma or right brace '}'"; - case json_errc::expected_comma_or_right_bracket: - return "Expected comma or right bracket ']'"; - case json_errc::unexpected_right_brace: - return "Unexpected right brace '}'"; - case json_errc::unexpected_right_bracket: - return "Unexpected right bracket ']'"; - case json_errc::illegal_comment: - return "Illegal comment"; - case json_errc::expected_continuation_byte: - return "Expected continuation byte"; - case json_errc::over_long_utf8_sequence: - return "Over long UTF-8 sequence"; - case json_errc::illegal_codepoint: - return "Illegal codepoint (>= 0xd800 && <= 0xdfff)"; - case json_errc::illegal_surrogate_value: - return "UTF-16 surrogate values are illegal in UTF-32"; - case json_errc::unpaired_high_surrogate: - return "Expected low surrogate following the high surrogate"; - default: - return "Unknown JSON parser error"; - } - } -}; - -inline -const std::error_category& json_error_category() -{ - static json_error_category_impl instance; - return instance; -} - -inline -std::error_code make_error_code(json_errc result) -{ - return std::error_code(static_cast(result),json_error_category()); -} - -#if !defined(JSONCONS_NO_DEPRECATED) -typedef json_errc json_parser_errc; - -typedef json_errc json_parse_errc; -#endif - - -} - -namespace std { - template<> - struct is_error_code_enum : public true_type - { - }; -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_exception.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_exception.hpp deleted file mode 100644 index 87269c7d5e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_exception.hpp +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSON_EXCEPTION_HPP -#define JSON_EXCEPTION_HPP - -#include // std::string -#include // std::ostringstream -#include // std::error_code -#include // unicons::convert -#include - -namespace jsoncons { - -// json_exception - -class json_exception -{ -public: - virtual const char* what() const noexcept = 0; -}; - -template -class json_runtime_error : public Base, public virtual json_exception -{ -public: - json_runtime_error(const std::string& s) noexcept - : Base(""), message_(s) - { - } - ~json_runtime_error() noexcept - { - } - const char* what() const noexcept override - { - return message_.c_str(); - } -private: - std::string message_; -}; - -class key_not_found : public std::out_of_range, public virtual json_exception -{ -public: - template - explicit key_not_found(const CharT* key, size_t length) noexcept - : std::out_of_range("") - { - buffer_.append("Key '"); - unicons::convert(key, key+length, std::back_inserter(buffer_), - unicons::conv_flags::strict); - buffer_.append("' not found"); - } - ~key_not_found() noexcept - { - } - const char* what() const noexcept override - { - return buffer_.c_str(); - } -private: - std::string buffer_; -}; - -class not_an_object : public std::runtime_error, public virtual json_exception -{ -public: - template - explicit not_an_object(const CharT* key, size_t length) noexcept - : std::runtime_error("") - { - buffer_.append("Attempting to access or modify '"); - unicons::convert(key, key+length, std::back_inserter(buffer_), - unicons::conv_flags::strict); - buffer_.append("' on a value that is not an object"); - } - ~not_an_object() noexcept - { - } - const char* what() const noexcept override - { - return buffer_.c_str(); - } -private: - std::string buffer_; -}; - -class ser_error : public std::system_error, public virtual json_exception -{ -public: - ser_error(std::error_code ec) - : std::system_error(ec), line_number_(0), column_number_(0) - { - } - ser_error(std::error_code ec, size_t position) - : std::system_error(ec), line_number_(0), column_number_(position) - { - } - ser_error(std::error_code ec, size_t line, size_t column) - : std::system_error(ec), line_number_(line), column_number_(column) - { - } - ser_error(const ser_error& other) = default; - - ser_error(ser_error&& other) = default; - - const char* what() const noexcept override - { - try - { - std::ostringstream os; - os << this->code().message(); - if (line_number_ != 0 && column_number_ != 0) - { - os << " at line " << line_number_ << " and column " << column_number_; - } - else if (column_number_ != 0) - { - os << " at position " << column_number_; - } - const_cast(buffer_) = os.str(); - return buffer_.c_str(); - } - catch (...) - { - return std::system_error::what(); - } - } - - size_t line() const noexcept - { - return line_number_; - } - - size_t column() const noexcept - { - return column_number_; - } - -#if !defined(JSONCONS_NO_DEPRECATED) - size_t line_number() const noexcept - { - return line(); - } - - size_t column_number() const noexcept - { - return column(); - } -#endif -private: - std::string buffer_; - size_t line_number_; - size_t column_number_; -}; - -#if !defined(JSONCONS_NO_DEPRECATED) -typedef ser_error serialization_error; -typedef ser_error json_parse_exception; -typedef ser_error parse_exception; -typedef ser_error parse_error; -#endif - -#define JSONCONS_STR2(x) #x -#define JSONCONS_STR(x) JSONCONS_STR2(x) - -#ifdef _DEBUG -#define JSONCONS_ASSERT(x) if (!(x)) { \ - throw jsoncons::json_runtime_error("assertion '" #x "' failed at " __FILE__ ":" \ - JSONCONS_STR(__LINE__)); } -#else -#define JSONCONS_ASSERT(x) if (!(x)) { \ - throw jsoncons::json_runtime_error("assertion '" #x "' failed at <> :" \ - JSONCONS_STR( 0 )); } -#endif // _DEBUG - -#define JSONCONS_THROW(x) throw (x) - -} -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_filter.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_filter.hpp deleted file mode 100644 index 116e6b822e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_filter.hpp +++ /dev/null @@ -1,382 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSON_FILTER_HPP -#define JSONCONS_JSON_FILTER_HPP - -#include - -#include -#include - -namespace jsoncons { - -template -class basic_json_filter : public basic_json_content_handler -{ -public: - using typename basic_json_content_handler::string_view_type; -private: - basic_json_content_handler& to_handler_; - - // noncopyable and nonmoveable - basic_json_filter(const basic_json_filter&) = delete; - basic_json_filter& operator=(const basic_json_filter&) = delete; -public: - basic_json_filter(basic_json_content_handler& handler) - : to_handler_(handler) - { - } - -#if !defined(JSONCONS_NO_DEPRECATED) - basic_json_content_handler& input_handler() - { - return to_handler_; - } - - basic_json_content_handler& downstream_handler() - { - return to_handler_; - } - - basic_json_content_handler& destination_handler() - { - return to_handler_; - } -#endif - - basic_json_content_handler& to_handler() - { - return to_handler_; - } - -private: - void do_flush() override - { - to_handler_.flush(); - } - - bool do_begin_object(semantic_tag tag, const ser_context& context) override - { - return to_handler_.begin_object(tag, context); - } - - bool do_begin_object(size_t length, semantic_tag tag, const ser_context& context) override - { - return to_handler_.begin_object(length, tag, context); - } - - bool do_end_object(const ser_context& context) override - { - return to_handler_.end_object(context); - } - - bool do_begin_array(semantic_tag tag, const ser_context& context) override - { - return to_handler_.begin_array(tag, context); - } - - bool do_begin_array(size_t length, semantic_tag tag, const ser_context& context) override - { - return to_handler_.begin_array(length, tag, context); - } - - bool do_end_array(const ser_context& context) override - { - return to_handler_.end_array(context); - } - - bool do_name(const string_view_type& name, - const ser_context& context) override - { - return to_handler_.name(name, context); - } - - bool do_string_value(const string_view_type& value, - semantic_tag tag, - const ser_context& context) override - { - return to_handler_.string_value(value, tag, context); - } - - bool do_byte_string_value(const byte_string_view& b, - semantic_tag tag, - const ser_context& context) override - { - return to_handler_.byte_string_value(b, tag, context); - } - - bool do_double_value(double value, - semantic_tag tag, - const ser_context& context) override - { - return to_handler_.double_value(value, tag, context); - } - - bool do_int64_value(int64_t value, - semantic_tag tag, - const ser_context& context) override - { - return to_handler_.int64_value(value, tag, context); - } - - bool do_uint64_value(uint64_t value, - semantic_tag tag, - const ser_context& context) override - { - return to_handler_.uint64_value(value, tag, context); - } - - bool do_bool_value(bool value, semantic_tag tag, const ser_context& context) override - { - return to_handler_.bool_value(value, tag, context); - } - - bool do_null_value(semantic_tag tag, const ser_context& context) override - { - return to_handler_.null_value(tag, context); - } - -}; - -// Filters out begin_document and end_document events -template -class basic_json_fragment_filter : public basic_json_filter -{ -public: - using typename basic_json_filter::string_view_type; - - basic_json_fragment_filter(basic_json_content_handler& handler) - : basic_json_filter(handler) - { - } -private: - void do_flush() override - { - } -}; - -template -class basic_rename_object_member_filter : public basic_json_filter -{ -public: - using typename basic_json_filter::string_view_type; - -private: - std::basic_string name_; - std::basic_string new_name_; -public: - basic_rename_object_member_filter(const std::basic_string& name, - const std::basic_string& new_name, - basic_json_content_handler& handler) - : basic_json_filter(handler), - name_(name), new_name_(new_name) - { - } - -private: - bool do_name(const string_view_type& name, - const ser_context& context) override - { - if (name == name_) - { - return this->to_handler().name(new_name_,context); - } - else - { - return this->to_handler().name(name,context); - } - } -}; - -template -class json_content_handler_adaptor : public From -{ -public: - using typename From::string_view_type; -private: - To* to_handler_; - - // noncopyable - json_content_handler_adaptor(const json_content_handler_adaptor&) = delete; - json_content_handler_adaptor& operator=(const json_content_handler_adaptor&) = delete; -public: - - json_content_handler_adaptor() - : to_handler_(nullptr) - { - } - json_content_handler_adaptor(To& handler) - : to_handler_(std::addressof(handler)) - { - } - - // moveable - json_content_handler_adaptor(json_content_handler_adaptor&&) = default; - json_content_handler_adaptor& operator=(json_content_handler_adaptor&&) = default; - - To& to_handler() - { - return *to_handler_; - } - -private: - void do_flush() override - { - to_handler_->flush(); - } - - bool do_begin_object(semantic_tag tag, - const ser_context& context) override - { - return to_handler_->begin_object(tag, context); - } - - bool do_begin_object(size_t length, - semantic_tag tag, - const ser_context& context) override - { - return to_handler_->begin_object(length, tag, context); - } - - bool do_end_object(const ser_context& context) override - { - return to_handler_->end_object(context); - } - - bool do_begin_array(semantic_tag tag, - const ser_context& context) override - { - return to_handler_->begin_array(tag, context); - } - - bool do_begin_array(size_t length, - semantic_tag tag, - const ser_context& context) override - { - return to_handler_->begin_array(length, tag, context); - } - - bool do_end_array(const ser_context& context) override - { - return to_handler_->end_array(context); - } - - bool do_name(const string_view_type& name, - const ser_context& context) override - { - std::basic_string target; - auto result = unicons::convert(name.begin(),name.end(),std::back_inserter(target),unicons::conv_flags::strict); - if (result.ec != unicons::conv_errc()) - { - throw ser_error(result.ec); - } - return to_handler().name(target, context); - } - - bool do_string_value(const string_view_type& value, - semantic_tag tag, - const ser_context& context) override - { - std::basic_string target; - auto result = unicons::convert(value.begin(),value.end(),std::back_inserter(target),unicons::conv_flags::strict); - if (result.ec != unicons::conv_errc()) - { - throw ser_error(result.ec); - } - return to_handler().string_value(target, tag, context); - } - - bool do_byte_string_value(const byte_string_view& b, - semantic_tag tag, - const ser_context& context) override - { - return to_handler_->byte_string_value(b, tag, context); - } - - bool do_double_value(double value, - semantic_tag tag, - const ser_context& context) override - { - return to_handler_->double_value(value, tag, context); - } - - bool do_int64_value(int64_t value, - semantic_tag tag, - const ser_context& context) override - { - return to_handler_->int64_value(value, tag, context); - } - - bool do_uint64_value(uint64_t value, - semantic_tag tag, - const ser_context& context) override - { - return to_handler_->uint64_value(value, tag, context); - } - - bool do_bool_value(bool value, semantic_tag tag, const ser_context& context) override - { - return to_handler_->bool_value(value, tag, context); - } - - bool do_null_value(semantic_tag tag, const ser_context& context) override - { - return to_handler_->null_value(tag, context); - } - -}; - -template -class json_content_handler_adaptor::value>::type> -{ -public: - typedef typename From::char_type char_type; - typedef typename From::char_traits_type char_traits_type; - typedef typename From::string_view_type string_view_type; -private: - To* to_handler_; -public: - json_content_handler_adaptor() - : to_handler_(nullptr) - { - } - json_content_handler_adaptor(To& handler) - : to_handler_(std::addressof(handler)) - { - } - - operator From&() { return *to_handler_; } - - // moveable - json_content_handler_adaptor(json_content_handler_adaptor&&) = default; - json_content_handler_adaptor& operator=(json_content_handler_adaptor&&) = default; - - To& to_handler() - { - return *to_handler_; - } -}; - -template -json_content_handler_adaptor make_json_content_handler_adaptor(To& to) -{ - return json_content_handler_adaptor(to); -} - -typedef basic_json_filter json_filter; -typedef basic_json_filter wjson_filter; -typedef basic_rename_object_member_filter rename_object_member_filter; -typedef basic_rename_object_member_filter wrename_object_member_filter; - -#if !defined(JSONCONS_NO_DEPRECATED) -typedef basic_rename_object_member_filter rename_name_filter; -typedef basic_rename_object_member_filter wrename_name_filter; -#endif - -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_fwd.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_fwd.hpp deleted file mode 100644 index d9b44cf8b1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_fwd.hpp +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSON_FWD_HPP -#define JSONCONS_JSON_FWD_HPP - -#include // std::allocator - -namespace jsoncons { - -struct sorted_policy; - -template > -class basic_json; - -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_options.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_options.hpp deleted file mode 100644 index fa434ee934..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_options.hpp +++ /dev/null @@ -1,837 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSON_OPTIONS_HPP -#define JSONCONS_JSON_OPTIONS_HPP - -#include -#include // std::numeric_limits -#include -#include -#include - -namespace jsoncons { - -#if !defined(JSONCONS_NO_TO_CHARS) -using chars_format = std::chars_format; -#else -enum class chars_format : uint8_t {fixed=1,scientific=2,hex=4,general=fixed|scientific}; -#endif - -// floating_point_options - -class floating_point_options -{ - chars_format format_; - int precision_; - uint8_t decimal_places_; -public: - floating_point_options() - : format_(chars_format::general), precision_(0), decimal_places_(0) - { - } - - floating_point_options(chars_format format, int precision, uint8_t decimal_places = 0) - : format_(format), precision_(precision), decimal_places_(decimal_places) - { - } - - explicit floating_point_options(chars_format format) - : format_(format), precision_(0), decimal_places_(0) - { - } - - floating_point_options(const floating_point_options&) = default; - floating_point_options(floating_point_options&&) = default; - floating_point_options& operator=(const floating_point_options& e) = default; - floating_point_options& operator=(floating_point_options&& e) = default; - - int precision() const - { - return precision_; - } - - uint8_t decimal_places() const - { - return decimal_places_; - } - - chars_format format() const - { - return format_; - } -}; - -enum class indenting : uint8_t {no_indent = 0, indent = 1}; - -enum class line_split_kind : uint8_t {same_line,new_line,multi_line}; - -enum class bigint_chars_format : uint8_t {number, base10, base64, base64url -#if !defined(JSONCONS_NO_DEPRECATED) -,integer = number -#endif -}; - -#if !defined(JSONCONS_NO_DEPRECATED) -typedef bigint_chars_format bignum_chars_format; -typedef bigint_chars_format big_integer_chars_format; -#endif - -enum class byte_string_chars_format : uint8_t {none=0,base16,base64,base64url}; - -enum class spaces_option{no_spaces=0,space_after,space_before,space_before_and_after}; - -template -class basic_json_decode_options -{ -public: - typedef std::basic_string string_type; - - virtual ~basic_json_decode_options() = default; - - virtual size_t max_nesting_depth() const = 0; - - virtual bool is_str_to_nan() const = 0; - - virtual std::basic_string nan_to_str() const = 0; - - virtual bool is_str_to_inf() const = 0; - - virtual std::basic_string inf_to_str() const = 0; - - virtual bool is_str_to_neginf() const = 0; - - virtual std::basic_string neginf_to_str() const = 0; - - virtual bool lossless_number() const = 0; -}; - -template -class basic_json_encode_options -{ -public: - typedef std::basic_string string_type; - - virtual ~basic_json_encode_options() = default; - - virtual size_t max_nesting_depth() const = 0; - - virtual byte_string_chars_format byte_string_format() const = 0; - - virtual bigint_chars_format bigint_format() const = 0; - - virtual line_split_kind object_object_line_splits() const = 0; - - virtual line_split_kind array_object_line_splits() const = 0; - - virtual line_split_kind object_array_line_splits() const = 0; - - virtual line_split_kind array_array_line_splits() const = 0; - - virtual size_t indent_size() const = 0; - - virtual size_t line_length_limit() const = 0; - - virtual chars_format floating_point_format() const = 0; - - virtual int precision() const = 0; - - virtual bool escape_all_non_ascii() const = 0; - - virtual bool escape_solidus() const = 0; - - virtual spaces_option spaces_around_colon() const = 0; - - virtual spaces_option spaces_around_comma() const = 0; - - virtual bool pad_inside_object_braces() const = 0; - - virtual bool pad_inside_array_brackets() const = 0; - - virtual std::basic_string new_line_chars() const = 0; - - virtual bool is_nan_to_num() const = 0; - - virtual std::basic_string nan_to_num() const = 0; - - virtual bool is_inf_to_num() const = 0; - - virtual std::basic_string inf_to_num() const = 0; - - virtual bool is_neginf_to_num() const = 0; - - virtual std::basic_string neginf_to_num() const = 0; - - virtual bool is_nan_to_str() const = 0; - - virtual std::basic_string nan_to_str() const = 0; - - virtual bool is_inf_to_str() const = 0; - - virtual std::basic_string inf_to_str() const = 0; - - virtual bool is_neginf_to_str() const = 0; - - virtual std::basic_string neginf_to_str() const = 0; -}; - -template -class basic_json_options : public virtual basic_json_decode_options, - public virtual basic_json_encode_options -{ -public: - typedef CharT char_type; - typedef std::basic_string string_type; -private: - size_t indent_size_; - chars_format floating_point_format_; - int precision_; -#if !defined(JSONCONS_NO_DEPRECATED) - bool can_read_nan_replacement_; - bool can_read_pos_inf_replacement_; - bool can_read_neg_inf_replacement_; - string_type nan_replacement_; - string_type pos_inf_replacement_; - string_type neg_inf_replacement_; -#endif - bool escape_all_non_ascii_; - bool escape_solidus_; - byte_string_chars_format byte_string_format_; - bigint_chars_format bigint_format_; - line_split_kind object_object_line_splits_; - line_split_kind object_array_line_splits_; - line_split_kind array_array_line_splits_; - line_split_kind array_object_line_splits_; - size_t line_length_limit_; - - size_t max_nesting_depth_; - spaces_option spaces_around_colon_; - spaces_option spaces_around_comma_; - bool pad_inside_object_braces_; - bool pad_inside_array_brackets_; - std::basic_string new_line_chars_; - - bool is_nan_to_num_; - bool is_inf_to_num_; - bool is_neginf_to_num_; - bool is_nan_to_str_; - bool is_inf_to_str_; - bool is_neginf_to_str_; - bool is_str_to_nan_; - bool is_str_to_inf_; - bool is_str_to_neginf_; - - std::basic_string nan_to_num_; - std::basic_string inf_to_num_; - std::basic_string neginf_to_num_; - std::basic_string nan_to_str_; - std::basic_string inf_to_str_; - std::basic_string neginf_to_str_; - - bool lossless_number_; -public: - static const size_t indent_size_default = 4; - static const size_t line_length_limit_default = 120; - - static const basic_json_options& default_options() - { - static basic_json_options options{}; - return options; - } - -// Constructors - - basic_json_options() - : indent_size_(indent_size_default), - floating_point_format_(chars_format()), - precision_(0), -#if !defined(JSONCONS_NO_DEPRECATED) - can_read_nan_replacement_(false), - can_read_pos_inf_replacement_(false), - can_read_neg_inf_replacement_(false), -#endif - escape_all_non_ascii_(false), - escape_solidus_(false), - byte_string_format_(byte_string_chars_format::none), - bigint_format_(bigint_chars_format::base10), - object_object_line_splits_(line_split_kind::multi_line), - object_array_line_splits_(line_split_kind::same_line), - array_array_line_splits_(line_split_kind::new_line), - array_object_line_splits_(line_split_kind::multi_line), - line_length_limit_(line_length_limit_default), - max_nesting_depth_((std::numeric_limits::max)()), - spaces_around_colon_(spaces_option::space_after), - spaces_around_comma_(spaces_option::space_after), - pad_inside_object_braces_(false), - pad_inside_array_brackets_(false), - is_nan_to_num_(false), - is_inf_to_num_(false), - is_neginf_to_num_(false), - is_nan_to_str_(false), - is_inf_to_str_(false), - is_neginf_to_str_(false), - is_str_to_nan_(false), - is_str_to_inf_(false), - is_str_to_neginf_(false), - lossless_number_(false) - { - new_line_chars_.push_back('\n'); - } - -// Properties - byte_string_chars_format byte_string_format() const override {return byte_string_format_;} - basic_json_options& byte_string_format(byte_string_chars_format value) {byte_string_format_ = value; return *this;} - - bigint_chars_format bigint_format() const override {return bigint_format_;} - basic_json_options& bigint_format(bigint_chars_format value) {bigint_format_ = value; return *this;} - -#if !defined(JSONCONS_NO_DEPRECATED) - basic_json_options& big_integer_format(bigint_chars_format value) {bigint_format_ = value; return *this;} - bignum_chars_format bignum_format() const {return bigint_format_;} - basic_json_options& bignum_format(bignum_chars_format value) {bigint_format_ = value; return *this;} -#endif - line_split_kind object_object_line_splits() const override {return object_object_line_splits_;} - basic_json_options& object_object_line_splits(line_split_kind value) {object_object_line_splits_ = value; return *this;} - - line_split_kind array_object_line_splits() const override {return array_object_line_splits_;} - basic_json_options& array_object_line_splits(line_split_kind value) {array_object_line_splits_ = value; return *this;} - - line_split_kind object_array_line_splits() const override {return object_array_line_splits_;} - basic_json_options& object_array_line_splits(line_split_kind value) {object_array_line_splits_ = value; return *this;} - - line_split_kind array_array_line_splits() const override {return array_array_line_splits_;} - basic_json_options& array_array_line_splits(line_split_kind value) {array_array_line_splits_ = value; return *this;} - - size_t indent_size() const override - { - return indent_size_; - } - - basic_json_options& indent_size(size_t value) - { - indent_size_ = value; - return *this; - } - - spaces_option spaces_around_colon() const override - { - return spaces_around_colon_; - } - - basic_json_options& spaces_around_colon(spaces_option value) - { - spaces_around_colon_ = value; - return *this; - } - - spaces_option spaces_around_comma() const override - { - return spaces_around_comma_; - } - - basic_json_options& spaces_around_comma(spaces_option value) - { - spaces_around_comma_ = value; - return *this; - } - - bool pad_inside_object_braces() const override - { - return pad_inside_object_braces_; - } - - basic_json_options& pad_inside_object_braces(bool value) - { - pad_inside_object_braces_ = value; - return *this; - } - - bool pad_inside_array_brackets() const override - { - return pad_inside_array_brackets_; - } - - basic_json_options& pad_inside_array_brackets(bool value) - { - pad_inside_array_brackets_ = value; - return *this; - } - - std::basic_string new_line_chars() const override - { - return new_line_chars_; - } - - basic_json_options& new_line_chars(const std::basic_string& value) - { - new_line_chars_ = value; - return *this; - } - - bool is_nan_to_num() const override - { - return is_nan_to_num_; - } - - bool is_inf_to_num() const override - { - return is_inf_to_num_; - } - - bool is_neginf_to_num() const override - { - return is_neginf_to_num_ || is_inf_to_num_; - } - - bool is_nan_to_str() const override - { - return is_nan_to_str_; - } - - bool is_str_to_nan() const override - { - return is_str_to_nan_; - } - - bool is_inf_to_str() const override - { - return is_inf_to_str_; - } - - bool is_str_to_inf() const override - { - return is_str_to_inf_; - } - - bool is_neginf_to_str() const override - { - return is_neginf_to_str_ || is_inf_to_str_; - } - - bool is_str_to_neginf() const override - { - return is_str_to_neginf_ || is_str_to_inf_; - } - - std::basic_string nan_to_num() const override - { - if (is_nan_to_num_) - { - return nan_to_num_; - } -#if !defined(JSONCONS_NO_DEPRECATED) - else if (!can_read_nan_replacement_) // not string - { - return nan_replacement_; - } -#endif - else - { - return nan_to_num_; // empty string - } - } - - basic_json_options& nan_to_num(const std::basic_string& value) - { - is_nan_to_num_ = true; - nan_to_str_.clear(); - nan_to_num_ = value; - return *this; - } - - std::basic_string inf_to_num() const override - { - if (is_inf_to_num_) - { - return inf_to_num_; - } -#if !defined(JSONCONS_NO_DEPRECATED) - else if (!can_read_pos_inf_replacement_) // not string - { - return pos_inf_replacement_; - } -#endif - else - { - return inf_to_num_; // empty string - } - } - - basic_json_options& inf_to_num(const std::basic_string& value) - { - is_inf_to_num_ = true; - inf_to_str_.clear(); - inf_to_num_ = value; - return *this; - } - - std::basic_string neginf_to_num() const override - { - if (is_neginf_to_num_) - { - return neginf_to_num_; - } - else if (is_inf_to_num_) - { - std::basic_string s; - s.push_back('-'); - s.append(inf_to_num_); - return s; - } -#if !defined(JSONCONS_NO_DEPRECATED) - else if (!can_read_neg_inf_replacement_) // not string - { - return neg_inf_replacement_; - } -#endif - else - { - return neginf_to_num_; // empty string - } - } - - basic_json_options& neginf_to_num(const std::basic_string& value) - { - is_neginf_to_num_ = true; - neginf_to_str_.clear(); - neginf_to_num_ = value; - return *this; - } - - std::basic_string nan_to_str() const override - { - if (is_nan_to_str_) - { - return nan_to_str_; - } -#if !defined(JSONCONS_NO_DEPRECATED) - else if (can_read_nan_replacement_ && nan_replacement_.size() >= 2) // string - { - return nan_replacement_.substr(1,nan_replacement_.size()-2); // Remove quotes - } -#endif - else - { - return nan_to_str_; // empty string - } - } - - basic_json_options& nan_to_str(const std::basic_string& value, bool is_str_to_nan = true) - { - is_nan_to_str_ = true; - is_str_to_nan_ = is_str_to_nan; - nan_to_num_.clear(); - nan_to_str_ = value; - return *this; - } - - std::basic_string inf_to_str() const override - { - if (is_inf_to_str_) - { - return inf_to_str_; - } -#if !defined(JSONCONS_NO_DEPRECATED) - else if (can_read_pos_inf_replacement_ && pos_inf_replacement_.size() >= 2) // string - { - return pos_inf_replacement_.substr(1,pos_inf_replacement_.size()-2); // Strip quotes - } -#endif - else - { - return inf_to_str_; // empty string - } - } - - basic_json_options& inf_to_str(const std::basic_string& value, bool is_inf_to_str = true) - { - is_inf_to_str_ = true; - is_inf_to_str_ = is_inf_to_str; - inf_to_num_.clear(); - inf_to_str_ = value; - return *this; - } - - std::basic_string neginf_to_str() const override - { - if (is_neginf_to_str_) - { - return neginf_to_str_; - } - else if (is_inf_to_str_) - { - std::basic_string s; - s.push_back('-'); - s.append(inf_to_str_); - return s; - } -#if !defined(JSONCONS_NO_DEPRECATED) - else if (can_read_neg_inf_replacement_ && neg_inf_replacement_.size() >= 2) // string - { - return neg_inf_replacement_.substr(1,neg_inf_replacement_.size()-2); // Strip quotes - } -#endif - else - { - return neginf_to_str_; // empty string - } - } - - basic_json_options& neginf_to_str(const std::basic_string& value, bool is_neginf_to_str = true) - { - is_neginf_to_str_ = true; - is_neginf_to_str_ = is_neginf_to_str; - neginf_to_num_.clear(); - neginf_to_str_ = value; - return *this; - } - - bool lossless_number() const override - { - return lossless_number_; - } - - basic_json_options& lossless_number(bool value) - { - lossless_number_ = value; - return *this; - } - - size_t line_length_limit() const override - { - return line_length_limit_; - } - - basic_json_options& line_length_limit(size_t value) - { - line_length_limit_ = value; - return *this; - } - - chars_format floating_point_format() const override - { - return floating_point_format_; - } - - basic_json_options& floating_point_format(chars_format value) - { - floating_point_format_ = value; - return *this; - } - - int precision() const override - { - return precision_; - } - - basic_json_options& precision(int value) - { - precision_ = value; - return *this; - } - - bool escape_all_non_ascii() const override - { - return escape_all_non_ascii_; - } - - basic_json_options& escape_all_non_ascii(bool value) - { - escape_all_non_ascii_ = value; - return *this; - } - - bool escape_solidus() const override - { - return escape_solidus_; - } - - basic_json_options& escape_solidus(bool value) - { - escape_solidus_ = value; - return *this; - } - - size_t max_nesting_depth() const override - { - return max_nesting_depth_; - } - - void max_nesting_depth(size_t value) - { - max_nesting_depth_ = value; - } - -#if !defined(JSONCONS_NO_DEPRECATED) - - bool dec_to_str() const - { - return lossless_number_; - } - - basic_json_options& dec_to_str(bool value) - { - lossless_number_ = value; - return *this; - } - - size_t indent() const - { - return indent_size(); - } - - basic_json_options& indent(size_t value) - { - return indent_size(value); - } - - bool can_read_nan_replacement() const {return can_read_nan_replacement_;} - - bool can_read_pos_inf_replacement() const {return can_read_pos_inf_replacement_;} - - bool can_read_neg_inf_replacement() const {return can_read_neg_inf_replacement_;} - - bool can_write_nan_replacement() const {return !nan_replacement_.empty();} - - bool can_write_pos_inf_replacement() const {return !pos_inf_replacement_.empty();} - - bool can_write_neg_inf_replacement() const {return !neg_inf_replacement_.empty();} - - basic_json_options& replace_inf(bool replace) - { - can_read_pos_inf_replacement_ = replace; - can_read_neg_inf_replacement_ = replace; - return *this; - } - - basic_json_options& replace_pos_inf(bool replace) - { - can_read_pos_inf_replacement_ = replace; - return *this; - } - - basic_json_options& replace_neg_inf(bool replace) - { - can_read_neg_inf_replacement_ = replace; - return *this; - } - - const string_type& nan_replacement() const - { - return nan_replacement_; - } - - basic_json_options& nan_replacement(const string_type& value) - { - nan_replacement_ = value; - - can_read_nan_replacement_ = is_string(value); - - return *this; - } - - const string_type& pos_inf_replacement() const - { - return pos_inf_replacement_; - } - - basic_json_options& pos_inf_replacement(const string_type& value) - { - pos_inf_replacement_ = value; - can_read_pos_inf_replacement_ = is_string(value); - return *this; - } - - const string_type& neg_inf_replacement() const - { - return neg_inf_replacement_; - } - - basic_json_options& neg_inf_replacement(const string_type& value) - { - neg_inf_replacement_ = value; - can_read_neg_inf_replacement_ = is_string(value); - return *this; - } - - line_split_kind object_object_split_lines() const {return object_object_line_splits_;} - basic_json_options& object_object_split_lines(line_split_kind value) {object_object_line_splits_ = value; return *this;} - - line_split_kind array_object_split_lines() const {return array_object_line_splits_;} - basic_json_options& array_object_split_lines(line_split_kind value) {array_object_line_splits_ = value; return *this;} - - line_split_kind object_array_split_lines() const {return object_array_line_splits_;} - basic_json_options& object_array_split_lines(line_split_kind value) {object_array_line_splits_ = value; return *this;} - - line_split_kind array_array_split_lines() const {return array_array_line_splits_;} - basic_json_options& array_array_split_lines(line_split_kind value) {array_array_line_splits_ = value; return *this;} -#endif -private: - enum class input_state {initial,begin_quote,character,end_quote,escape,error}; - bool is_string(const string_type& s) const - { - input_state state = input_state::initial; - for (CharT c : s) - { - switch (c) - { - case '\t': case ' ': case '\n': case'\r': - break; - case '\\': - state = input_state::escape; - break; - case '\"': - switch (state) - { - case input_state::initial: - state = input_state::begin_quote; - break; - case input_state::begin_quote: - state = input_state::end_quote; - break; - case input_state::character: - state = input_state::end_quote; - break; - case input_state::end_quote: - state = input_state::error; - break; - case input_state::escape: - state = input_state::character; - break; - default: - state = input_state::character; - break; - } - break; - default: - break; - } - - } - return state == input_state::end_quote; - } -}; - -typedef basic_json_options json_options; -typedef basic_json_options wjson_options; - -typedef basic_json_decode_options json_decode_options; -typedef basic_json_decode_options wjson_decode_options; - -typedef basic_json_encode_options json_encode_options; -typedef basic_json_encode_options wjson_encode_options; - -#if !defined(JSONCONS_NO_DEPRECATED) -typedef basic_json_options output_format; -typedef basic_json_options woutput_format; -typedef basic_json_options serialization_options; -typedef basic_json_options wserialization_options; -typedef basic_json_options json_serializing_options; -typedef basic_json_options wjson_serializing_options; -#endif - -} -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_parser.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_parser.hpp deleted file mode 100644 index 7f6747a746..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_parser.hpp +++ /dev/null @@ -1,2841 +0,0 @@ -// Copyright 2015 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSON_PARSER_HPP -#define JSONCONS_JSON_PARSER_HPP - -#include // std::allocator -#include -#include -#include -#include -#include // std::numeric_limits -#include -#include -#include -#include -#include -#include -#include - -#define JSONCONS_ILLEGAL_CONTROL_CHARACTER \ - case 0x00:case 0x01:case 0x02:case 0x03:case 0x04:case 0x05:case 0x06:case 0x07:case 0x08:case 0x0b: \ - case 0x0c:case 0x0e:case 0x0f:case 0x10:case 0x11:case 0x12:case 0x13:case 0x14:case 0x15:case 0x16: \ - case 0x17:case 0x18:case 0x19:case 0x1a:case 0x1b:case 0x1c:case 0x1d:case 0x1e:case 0x1f - -namespace jsoncons { - -namespace detail { - -template -class replacement_filter : public basic_json_filter -{ - typedef typename basic_json_content_handler::string_view_type string_view_type; - typedef typename basic_json_options::string_type string_type; - - bool is_str_to_nan_; - bool is_str_to_inf_; - bool is_str_to_neginf_; - string_type nan_to_str_; - string_type inf_to_str_; - string_type neginf_to_str_; - -public: - replacement_filter() = delete; - - replacement_filter(basic_json_content_handler& handler, - bool is_str_to_nan, - bool is_str_to_inf, - bool is_str_to_neginf, - const string_type& nan_to_str, - const string_type& inf_to_str, - const string_type& neginf_to_str) - : basic_json_filter(handler), - is_str_to_nan_(is_str_to_nan), - is_str_to_inf_(is_str_to_inf), - is_str_to_neginf_(is_str_to_neginf), - nan_to_str_(nan_to_str), - inf_to_str_(inf_to_str), - neginf_to_str_(neginf_to_str) - { - } - - bool do_string_value(const string_view_type& s, - semantic_tag tag, - const ser_context& context) override - { - if (tag == semantic_tag::none) - { - if (is_str_to_nan_ && s == nan_to_str_) - { - return this->to_handler().double_value(std::nan(""), tag, context); - } - else if (is_str_to_inf_ && s == inf_to_str_) - { - return this->to_handler().double_value(std::numeric_limits::infinity(), tag, context); - } - else if (is_str_to_neginf_ && s == neginf_to_str_) - { - return this->to_handler().double_value(-std::numeric_limits::infinity(), tag, context); - } - else - { - return this->to_handler().string_value(s, tag, context); - } - } - else - { - return this->to_handler().string_value(s, tag, context); - } - } -}; - -} - -enum class json_parse_state : uint8_t -{ - root, - start, - before_done, - slash, - slash_slash, - slash_star, - slash_star_star, - expect_comma_or_end, - object, - expect_member_name_or_end, - expect_member_name, - expect_colon, - expect_value_or_end, - expect_value, - array, - string, - member_name, - escape, - escape_u1, - escape_u2, - escape_u3, - escape_u4, - escape_expect_surrogate_pair1, - escape_expect_surrogate_pair2, - escape_u6, - escape_u7, - escape_u8, - escape_u9, - minus, - zero, - integer, - fraction1, - fraction2, - exp1, - exp2, - exp3, - n, - nu, - nul, - t, - tr, - tru, - f, - fa, - fal, - fals, - cr, - done -}; - -template > -class basic_json_parser : public ser_context -{ - typedef std::basic_string string_type; - typedef typename basic_json_content_handler::string_view_type string_view_type; - typedef Allocator allocator_type; - typedef typename std::allocator_traits:: template rebind_alloc char_allocator_type; - typedef typename std::allocator_traits:: template rebind_alloc numeral_allocator_type; - typedef typename std::allocator_traits:: template rebind_alloc byte_allocator_type; - - static const size_t initial_string_buffer_capacity_ = 1024; - static const int default_initial_stack_capacity_ = 100; - - default_parse_error_handler default_err_handler_; - - const basic_json_decode_options& options_; - - parse_error_handler& err_handler_; - int initial_stack_capacity_; - size_t nesting_depth_; - uint32_t cp_; - uint32_t cp2_; - size_t line_; - size_t column_; - const CharT* begin_input_; - const CharT* input_end_; - const CharT* input_ptr_; - json_parse_state state_; - bool continue_; - bool done_; - - std::basic_string,char_allocator_type> string_buffer_; - jsoncons::detail::string_to_double to_double_; - - typedef typename std::allocator_traits:: template rebind_alloc parse_state_allocator_type; - std::vector state_stack_; - - // Noncopyable and nonmoveable - basic_json_parser(const basic_json_parser&) = delete; - basic_json_parser& operator=(const basic_json_parser&) = delete; - -public: - basic_json_parser() - : basic_json_parser(basic_json_options::default_options(), default_err_handler_) - { - } - - basic_json_parser(parse_error_handler& err_handler) - : basic_json_parser(basic_json_options::default_options(), err_handler) - { - } - - basic_json_parser(const basic_json_decode_options& options) - : basic_json_parser(options, default_err_handler_) - { - } - - basic_json_parser(const basic_json_decode_options& options, - parse_error_handler& err_handler) - : options_(options), - err_handler_(err_handler), - initial_stack_capacity_(default_initial_stack_capacity_), - nesting_depth_(0), - cp_(0), - cp2_(0), - line_(1), - column_(1), - begin_input_(nullptr), - input_end_(nullptr), - input_ptr_(nullptr), - state_(json_parse_state::start), - continue_(true), - done_(false) - { - string_buffer_.reserve(initial_string_buffer_capacity_); - - state_stack_.reserve(initial_stack_capacity_); - push_state(json_parse_state::root); - } - - void set_column_number(size_t column) - { - column_ = column; - } - - bool source_exhausted() const - { - return input_ptr_ == input_end_; - } - - ~basic_json_parser() - { - } - -#if !defined(JSONCONS_NO_DEPRECATED) - size_t max_nesting_depth() const - { - return options_.max_nesting_depth(); - } - - void max_nesting_depth(size_t value) - { - options_.max_nesting_depth() = value; - } -#endif - json_parse_state parent() const - { - JSONCONS_ASSERT(state_stack_.size() >= 1); - return state_stack_.back(); - } - - bool done() const - { - return done_; - } - - bool stopped() const - { - return !continue_; - } - - bool finished() const - { - return !continue_ && state_ != json_parse_state::before_done; - } - - void skip_space() - { - const CharT* local_input_end = input_end_; - while (input_ptr_ != local_input_end) - { - switch (*input_ptr_) - { - case ' ': - case '\t': - ++input_ptr_; - ++column_; - break; - case '\r': - push_state(state_); - ++input_ptr_; - ++column_; - state_ = json_parse_state::cr; - return; - case '\n': - ++input_ptr_; - ++line_; - column_ = 1; - return; - default: - return; - } - } - } - - void skip_whitespace() - { - const CharT* local_input_end = input_end_; - - while (input_ptr_ != local_input_end) - { - switch (state_) - { - case json_parse_state::cr: - ++line_; - column_ = 1; - switch (*input_ptr_) - { - case '\n': - ++input_ptr_; - state_ = pop_state(); - break; - default: - state_ = pop_state(); - break; - } - break; - - default: - switch (*input_ptr_) - { - case ' ': - case '\t': - case '\n': - case '\r': - skip_space(); - break; - default: - return; - } - break; - } - } - } - - void begin_object(basic_json_content_handler& handler, std::error_code& ec) - { - if (++nesting_depth_ > options_.max_nesting_depth()) - { - continue_ = err_handler_.error(json_errc::max_depth_exceeded, *this); - if (!continue_) - { - ec = json_errc::max_depth_exceeded; - return; - } - } - push_state(json_parse_state::object); - state_ = json_parse_state::expect_member_name_or_end; - continue_ = handler.begin_object(semantic_tag::none, *this); - } - - void end_object(basic_json_content_handler& handler, std::error_code& ec) - { - if (nesting_depth_ < 1) - { - err_handler_.fatal_error(json_errc::unexpected_right_brace, *this); - ec = json_errc::unexpected_right_brace; - continue_ = false; - return; - } - --nesting_depth_; - state_ = pop_state(); - if (state_ == json_parse_state::object) - { - continue_ = handler.end_object(*this); - } - else if (state_ == json_parse_state::array) - { - err_handler_.fatal_error(json_errc::expected_comma_or_right_bracket, *this); - ec = json_errc::expected_comma_or_right_bracket; - continue_ = false; - return; - } - else - { - err_handler_.fatal_error(json_errc::unexpected_right_brace, *this); - ec = json_errc::unexpected_right_brace; - continue_ = false; - return; - } - - if (parent() == json_parse_state::root) - { - state_ = json_parse_state::before_done; - } - else - { - state_ = json_parse_state::expect_comma_or_end; - } - } - - void begin_array(basic_json_content_handler& handler, std::error_code& ec) - { - if (++nesting_depth_ > options_.max_nesting_depth()) - { - continue_ = err_handler_.error(json_errc::max_depth_exceeded, *this); - if (!continue_) - { - ec = json_errc::max_depth_exceeded; - return; - } - } - push_state(json_parse_state::array); - state_ = json_parse_state::expect_value_or_end; - continue_ = handler.begin_array(semantic_tag::none, *this); - } - - void end_array(basic_json_content_handler& handler, std::error_code& ec) - { - if (nesting_depth_ < 1) - { - err_handler_.fatal_error(json_errc::unexpected_right_bracket, *this); - ec = json_errc::unexpected_right_bracket; - continue_ = false; - return; - } - --nesting_depth_; - state_ = pop_state(); - if (state_ == json_parse_state::array) - { - continue_ = handler.end_array(*this); - } - else if (state_ == json_parse_state::object) - { - err_handler_.fatal_error(json_errc::expected_comma_or_right_brace, *this); - ec = json_errc::expected_comma_or_right_brace; - continue_ = false; - return; - } - else - { - err_handler_.fatal_error(json_errc::unexpected_right_bracket, *this); - ec = json_errc::unexpected_right_bracket; - continue_ = false; - return; - } - if (parent() == json_parse_state::root) - { - state_ = json_parse_state::before_done; - } - else - { - state_ = json_parse_state::expect_comma_or_end; - } - } - - void reset() - { - state_stack_.clear(); - state_stack_.reserve(initial_stack_capacity_); - push_state(json_parse_state::root); - state_ = json_parse_state::start; - continue_ = true; - done_ = false; - line_ = 1; - column_ = 1; - nesting_depth_ = 0; - } - - void restart() - { - continue_ = true; - } - - void check_done() - { - std::error_code ec; - check_done(ec); - if (ec) - { - throw ser_error(ec,line_,column_); - } - } - - void check_done(std::error_code& ec) - { - for (; input_ptr_ != input_end_; ++input_ptr_) - { - CharT curr_char_ = *input_ptr_; - switch (curr_char_) - { - case '\n': - case '\r': - case '\t': - case ' ': - break; - default: - continue_ = err_handler_.error(json_errc::extra_character, *this); - if (!continue_) - { - ec = json_errc::extra_character; - return; - } - break; - } - } - } - - json_parse_state state() const - { - return state_; - } - - void update(const string_view_type sv) - { - update(sv.data(),sv.length()); - } - - void update(const CharT* data, size_t length) - { - begin_input_ = data; - input_end_ = data + length; - input_ptr_ = begin_input_; - } - - void parse_some(basic_json_content_handler& handler) - { - std::error_code ec; - parse_some(handler, ec); - if (ec) - { - throw ser_error(ec,line_,column_); - } - } - - void parse_some(basic_json_content_handler& handler, std::error_code& ec) - { - if (options_.is_str_to_nan() || options_.is_str_to_inf() || options_.is_str_to_neginf()) - { - jsoncons::detail::replacement_filter h(handler, - options_.is_str_to_nan(), - options_.is_str_to_inf(), - options_.is_str_to_neginf(), - options_.nan_to_str(), - options_.inf_to_str(), - options_.neginf_to_str()); - parse_some_(h, ec); - } - else - { - parse_some_(handler, ec); - } - } - - void finish_parse(basic_json_content_handler& handler) - { - std::error_code ec; - finish_parse(handler, ec); - if (ec) - { - throw ser_error(ec,line_,column_); - } - } - - void finish_parse(basic_json_content_handler& handler, std::error_code& ec) - { - while (!finished()) - { - parse_some(handler, ec); - } - } - - void parse_some_(basic_json_content_handler& handler, std::error_code& ec) - { - if (state_ == json_parse_state::before_done) - { - handler.flush(); - done_ = true; - state_ = json_parse_state::done; - continue_ = false; - return; - } - const CharT* local_input_end = input_end_; - - if (input_ptr_ == local_input_end && continue_) - { - switch (state_) - { - case json_parse_state::zero: - case json_parse_state::integer: - end_integer_value(handler, ec); - if (ec) return; - break; - case json_parse_state::fraction2: - end_fraction_value(handler, ec); - if (ec) return; - break; - case json_parse_state::exp3: - end_fraction_value(handler, ec); - if (ec) return; - break; - case json_parse_state::before_done: - handler.flush(); - done_ = true; - state_ = json_parse_state::done; - continue_ = false; - break; - case json_parse_state::done: - continue_ = false; - break; - case json_parse_state::cr: - state_ = pop_state(); - break; - default: - err_handler_.fatal_error(json_errc::unexpected_eof, *this); - ec = json_errc::unexpected_eof; - continue_ = false; - return; - } - } - - while ((input_ptr_ < local_input_end) && continue_) - { - switch (state_) - { - case json_parse_state::before_done: - handler.flush(); - done_ = true; - state_ = json_parse_state::done; - continue_ = false; - break; - case json_parse_state::cr: - ++line_; - column_ = 1; - switch (*input_ptr_) - { - case '\n': - ++input_ptr_; - state_ = pop_state(); - break; - default: - state_ = pop_state(); - break; - } - break; - case json_parse_state::start: - { - switch (*input_ptr_) - { - JSONCONS_ILLEGAL_CONTROL_CHARACTER: - continue_ = err_handler_.error(json_errc::illegal_control_character, *this); - if (!continue_) - { - ec = json_errc::illegal_control_character; - return; - } - break; - case '\r': - push_state(state_); - ++input_ptr_; - ++column_; - state_ = json_parse_state::cr; - break; - case '\n': - ++input_ptr_; - ++line_; - column_ = 1; - break; - case ' ':case '\t': - skip_space(); - break; - case '/': - ++input_ptr_; - ++column_; - push_state(state_); - state_ = json_parse_state::slash; - break; - case '{': - begin_object(handler, ec); - if (ec) return; - ++input_ptr_; - ++column_; - break; - case '[': - begin_array(handler, ec); - if (ec) return; - ++input_ptr_; - ++column_; - break; - case '\"': - state_ = json_parse_state::string; - ++input_ptr_; - ++column_; - string_buffer_.clear(); - parse_string(handler, ec); - if (ec) return; - break; - case '-': - string_buffer_.clear(); - string_buffer_.push_back('-'); - ++input_ptr_; - ++column_; - state_ = json_parse_state::minus; - parse_number(handler, ec); - if (ec) {return;} - break; - case '0': - string_buffer_.clear(); - string_buffer_.push_back(static_cast(*input_ptr_)); - state_ = json_parse_state::zero; - ++input_ptr_; - ++column_; - parse_number(handler, ec); - if (ec) {return;} - break; - case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - string_buffer_.clear(); - string_buffer_.push_back(static_cast(*input_ptr_)); - ++input_ptr_; - ++column_; - state_ = json_parse_state::integer; - parse_number(handler, ec); - if (ec) {return;} - break; - case 'n': - parse_null(handler, ec); - if (ec) {return;} - break; - case 't': - parse_true(handler, ec); - if (ec) {return;} - break; - case 'f': - parse_false(handler, ec); - if (ec) {return;} - break; - case '}': - err_handler_.fatal_error(json_errc::unexpected_right_brace, *this); - ec = json_errc::unexpected_right_brace; - continue_ = false; - return; - case ']': - err_handler_.fatal_error(json_errc::unexpected_right_bracket, *this); - ec = json_errc::unexpected_right_bracket; - continue_ = false; - return; - default: - err_handler_.fatal_error(json_errc::invalid_json_text, *this); - ec = json_errc::invalid_json_text; - continue_ = false; - return; - } - } - break; - - case json_parse_state::expect_comma_or_end: - { - switch (*input_ptr_) - { - JSONCONS_ILLEGAL_CONTROL_CHARACTER: - continue_ = err_handler_.error(json_errc::illegal_control_character, *this); - if (!continue_) - { - ec = json_errc::illegal_control_character; - return; - } - ++input_ptr_; - ++column_; - break; - case '\r': - ++input_ptr_; - ++column_; - push_state(state_); - state_ = json_parse_state::cr; - break; - case '\n': - ++input_ptr_; - ++line_; - column_ = 1; - break; - case ' ':case '\t': - skip_space(); - break; - case '/': - ++input_ptr_; - ++column_; - push_state(state_); - state_ = json_parse_state::slash; - break; - case '}': - end_object(handler, ec); - if (ec) return; - ++input_ptr_; - ++column_; - break; - case ']': - end_array(handler, ec); - if (ec) return; - ++input_ptr_; - ++column_; - break; - case ',': - begin_member_or_element(ec); - if (ec) return; - ++input_ptr_; - ++column_; - break; - default: - if (parent() == json_parse_state::array) - { - continue_ = err_handler_.error(json_errc::expected_comma_or_right_bracket, *this); - if (!continue_) - { - ec = json_errc::expected_comma_or_right_bracket; - return; - } - } - else if (parent() == json_parse_state::object) - { - continue_ = err_handler_.error(json_errc::expected_comma_or_right_brace, *this); - if (!continue_) - { - ec = json_errc::expected_comma_or_right_brace; - return; - } - } - ++input_ptr_; - ++column_; - break; - } - } - break; - case json_parse_state::expect_member_name_or_end: - { - switch (*input_ptr_) - { - JSONCONS_ILLEGAL_CONTROL_CHARACTER: - continue_ = err_handler_.error(json_errc::illegal_control_character, *this); - if (!continue_) - { - ec = json_errc::illegal_control_character; - return; - } - ++input_ptr_; - ++column_; - break; - case '\r': - ++input_ptr_; - ++column_; - push_state(state_); - state_ = json_parse_state::cr; - break; - case '\n': - ++input_ptr_; - ++line_; - column_ = 1; - break; - case ' ':case '\t': - skip_space(); - break; - case '/': - ++input_ptr_; - ++column_; - push_state(state_); - state_ = json_parse_state::slash; - break; - case '}': - end_object(handler, ec); - if (ec) return; - ++input_ptr_; - ++column_; - break; - case '\"': - ++input_ptr_; - ++column_; - push_state(json_parse_state::member_name); - state_ = json_parse_state::string; - string_buffer_.clear(); - parse_string(handler, ec); - if (ec) return; - break; - case '\'': - continue_ = err_handler_.error(json_errc::single_quote, *this); - if (!continue_) - { - ec = json_errc::single_quote; - return; - } - ++input_ptr_; - ++column_; - break; - default: - continue_ = err_handler_.error(json_errc::expected_name, *this); - if (!continue_) - { - ec = json_errc::expected_name; - return; - } - ++input_ptr_; - ++column_; - break; - } - } - break; - case json_parse_state::expect_member_name: - { - switch (*input_ptr_) - { - JSONCONS_ILLEGAL_CONTROL_CHARACTER: - continue_ = err_handler_.error(json_errc::illegal_control_character, *this); - if (!continue_) - { - ec = json_errc::illegal_control_character; - return; - } - ++input_ptr_; - ++column_; - break; - case '\r': - ++input_ptr_; - ++column_; - push_state(state_); - state_ = json_parse_state::cr; - break; - case '\n': - ++input_ptr_; - ++line_; - column_ = 1; - break; - case ' ':case '\t': - skip_space(); - break; - case '/': - ++input_ptr_; - ++column_; - push_state(state_); - state_ = json_parse_state::slash; - break; - case '\"': - ++input_ptr_; - ++column_; - push_state(json_parse_state::member_name); - state_ = json_parse_state::string; - string_buffer_.clear(); - parse_string(handler, ec); - if (ec) return; - break; - case '}': - continue_ = err_handler_.error(json_errc::extra_comma, *this); - if (!continue_) - { - ec = json_errc::extra_comma; - return; - } - end_object(handler, ec); // Recover - if (ec) return; - ++input_ptr_; - ++column_; - break; - case '\'': - continue_ = err_handler_.error(json_errc::single_quote, *this); - if (!continue_) - { - ec = json_errc::single_quote; - return; - } - ++input_ptr_; - ++column_; - break; - default: - continue_ = err_handler_.error(json_errc::expected_name, *this); - if (!continue_) - { - ec = json_errc::expected_name; - return; - } - ++input_ptr_; - ++column_; - break; - } - } - break; - case json_parse_state::expect_colon: - { - switch (*input_ptr_) - { - JSONCONS_ILLEGAL_CONTROL_CHARACTER: - continue_ = err_handler_.error(json_errc::illegal_control_character, *this); - if (!continue_) - { - ec = json_errc::illegal_control_character; - return; - } - ++input_ptr_; - ++column_; - break; - case '\r': - push_state(state_); - state_ = json_parse_state::cr; - ++input_ptr_; - ++column_; - break; - case '\n': - ++input_ptr_; - ++line_; - column_ = 1; - break; - case ' ':case '\t': - skip_space(); - break; - case '/': - push_state(state_); - state_ = json_parse_state::slash; - ++input_ptr_; - ++column_; - break; - case ':': - state_ = json_parse_state::expect_value; - ++input_ptr_; - ++column_; - break; - default: - continue_ = err_handler_.error(json_errc::expected_colon, *this); - if (!continue_) - { - ec = json_errc::expected_colon; - return; - } - ++input_ptr_; - ++column_; - break; - } - } - break; - - case json_parse_state::expect_value: - { - switch (*input_ptr_) - { - JSONCONS_ILLEGAL_CONTROL_CHARACTER: - continue_ = err_handler_.error(json_errc::illegal_control_character, *this); - if (!continue_) - { - ec = json_errc::illegal_control_character; - return; - } - ++input_ptr_; - ++column_; - break; - case '\r': - push_state(state_); - ++input_ptr_; - ++column_; - state_ = json_parse_state::cr; - break; - case '\n': - ++input_ptr_; - ++line_; - column_ = 1; - break; - case ' ':case '\t': - skip_space(); - break; - case '/': - push_state(state_); - ++input_ptr_; - ++column_; - state_ = json_parse_state::slash; - break; - case '{': - begin_object(handler, ec); - if (ec) return; - ++input_ptr_; - ++column_; - break; - case '[': - begin_array(handler, ec); - if (ec) return; - ++input_ptr_; - ++column_; - break; - case '\"': - ++input_ptr_; - ++column_; - state_ = json_parse_state::string; - string_buffer_.clear(); - parse_string(handler, ec); - if (ec) return; - break; - case '-': - string_buffer_.clear(); - string_buffer_.push_back('-'); - ++input_ptr_; - ++column_; - state_ = json_parse_state::minus; - parse_number(handler, ec); - if (ec) {return;} - break; - case '0': - string_buffer_.clear(); - string_buffer_.push_back(static_cast(*input_ptr_)); - ++input_ptr_; - ++column_; - state_ = json_parse_state::zero; - parse_number(handler, ec); - if (ec) {return;} - break; - case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - string_buffer_.clear(); - string_buffer_.push_back(static_cast(*input_ptr_)); - ++input_ptr_; - ++column_; - state_ = json_parse_state::integer; - parse_number(handler, ec); - if (ec) {return;} - break; - case 'n': - parse_null(handler, ec); - if (ec) {return;} - break; - case 't': - parse_true(handler, ec); - if (ec) {return;} - break; - case 'f': - parse_false(handler, ec); - if (ec) {return;} - break; - case ']': - if (parent() == json_parse_state::array) - { - continue_ = err_handler_.error(json_errc::extra_comma, *this); - if (!continue_) - { - ec = json_errc::extra_comma; - return; - } - end_array(handler, ec); // Recover - if (ec) return; - } - else - { - continue_ = err_handler_.error(json_errc::expected_value, *this); - if (!continue_) - { - ec = json_errc::expected_value; - return; - } - } - ++input_ptr_; - ++column_; - break; - case '\'': - continue_ = err_handler_.error(json_errc::single_quote, *this); - if (!continue_) - { - ec = json_errc::single_quote; - return; - } - ++input_ptr_; - ++column_; - break; - default: - continue_ = err_handler_.error(json_errc::expected_value, *this); - if (!continue_) - { - ec = json_errc::expected_value; - return; - } - ++input_ptr_; - ++column_; - break; - } - } - break; - case json_parse_state::expect_value_or_end: - { - switch (*input_ptr_) - { - JSONCONS_ILLEGAL_CONTROL_CHARACTER: - continue_ = err_handler_.error(json_errc::illegal_control_character, *this); - if (!continue_) - { - ec = json_errc::illegal_control_character; - return; - } - ++input_ptr_; - ++column_; - break; - case '\r': - ++input_ptr_; - ++column_; - push_state(state_); - state_ = json_parse_state::cr; - break; - case '\n': - ++input_ptr_; - ++line_; - column_ = 1; - break; - case ' ':case '\t': - skip_space(); - break; - case '/': - ++input_ptr_; - ++column_; - push_state(state_); - state_ = json_parse_state::slash; - break; - case '{': - begin_object(handler, ec); - if (ec) return; - ++input_ptr_; - ++column_; - break; - case '[': - begin_array(handler, ec); - if (ec) return; - ++input_ptr_; - ++column_; - break; - case ']': - end_array(handler, ec); - if (ec) return; - ++input_ptr_; - ++column_; - break; - case '\"': - ++input_ptr_; - ++column_; - state_ = json_parse_state::string; - string_buffer_.clear(); - parse_string(handler, ec); - if (ec) return; - break; - case '-': - string_buffer_.clear(); - string_buffer_.push_back('-'); - ++input_ptr_; - ++column_; - state_ = json_parse_state::minus; - parse_number(handler, ec); - if (ec) {return;} - break; - case '0': - string_buffer_.clear(); - string_buffer_.push_back(static_cast(*input_ptr_)); - ++input_ptr_; - ++column_; - state_ = json_parse_state::zero; - parse_number(handler, ec); - if (ec) {return;} - break; - case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - string_buffer_.clear(); - string_buffer_.push_back(static_cast(*input_ptr_)); - ++input_ptr_; - ++column_; - state_ = json_parse_state::integer; - parse_number(handler, ec); - if (ec) {return;} - break; - case 'n': - parse_null(handler, ec); - if (ec) {return;} - break; - case 't': - parse_true(handler, ec); - if (ec) {return;} - break; - case 'f': - parse_false(handler, ec); - if (ec) {return;} - break; - case '\'': - continue_ = err_handler_.error(json_errc::single_quote, *this); - if (!continue_) - { - ec = json_errc::single_quote; - return; - } - ++input_ptr_; - ++column_; - break; - default: - continue_ = err_handler_.error(json_errc::expected_value, *this); - if (!continue_) - { - ec = json_errc::expected_value; - return; - } - ++input_ptr_; - ++column_; - break; - } - } - break; - case json_parse_state::string: - case json_parse_state::escape: - case json_parse_state::escape_u1: - case json_parse_state::escape_u2: - case json_parse_state::escape_u3: - case json_parse_state::escape_u4: - case json_parse_state::escape_expect_surrogate_pair1: - case json_parse_state::escape_expect_surrogate_pair2: - case json_parse_state::escape_u6: - case json_parse_state::escape_u7: - case json_parse_state::escape_u8: - case json_parse_state::escape_u9: - parse_string(handler, ec); - if (ec) return; - break; - case json_parse_state::minus: - case json_parse_state::zero: - case json_parse_state::integer: - case json_parse_state::fraction1: - case json_parse_state::fraction2: - case json_parse_state::exp1: - case json_parse_state::exp2: - case json_parse_state::exp3: - parse_number(handler, ec); - if (ec) return; - break; - case json_parse_state::t: - switch (*input_ptr_) - { - case 'r': - ++input_ptr_; - ++column_; - state_ = json_parse_state::tr; - break; - default: - err_handler_.fatal_error(json_errc::invalid_value, *this); - ec = json_errc::invalid_value; - continue_ = false; - return; - } - break; - case json_parse_state::tr: - switch (*input_ptr_) - { - case 'u': - state_ = json_parse_state::tru; - break; - default: - err_handler_.fatal_error(json_errc::invalid_value, *this); - ec = json_errc::invalid_value; - continue_ = false; - return; - } - ++input_ptr_; - ++column_; - break; - case json_parse_state::tru: - switch (*input_ptr_) - { - case 'e': - continue_ = handler.bool_value(true, semantic_tag::none, *this); - if (parent() == json_parse_state::root) - { - state_ = json_parse_state::before_done; - } - else - { - state_ = json_parse_state::expect_comma_or_end; - } - break; - default: - err_handler_.fatal_error(json_errc::invalid_value, *this); - ec = json_errc::invalid_value; - continue_ = false; - return; - } - ++input_ptr_; - ++column_; - break; - case json_parse_state::f: - switch (*input_ptr_) - { - case 'a': - ++input_ptr_; - ++column_; - state_ = json_parse_state::fa; - break; - default: - err_handler_.fatal_error(json_errc::invalid_value, *this); - ec = json_errc::invalid_value; - continue_ = false; - return; - } - break; - case json_parse_state::fa: - switch (*input_ptr_) - { - case 'l': - state_ = json_parse_state::fal; - break; - default: - err_handler_.fatal_error(json_errc::invalid_value, *this); - ec = json_errc::invalid_value; - continue_ = false; - return; - } - ++input_ptr_; - ++column_; - break; - case json_parse_state::fal: - switch (*input_ptr_) - { - case 's': - state_ = json_parse_state::fals; - break; - default: - err_handler_.fatal_error(json_errc::invalid_value, *this); - ec = json_errc::invalid_value; - continue_ = false; - return; - } - ++input_ptr_; - ++column_; - break; - case json_parse_state::fals: - switch (*input_ptr_) - { - case 'e': - continue_ = handler.bool_value(false, semantic_tag::none, *this); - if (parent() == json_parse_state::root) - { - state_ = json_parse_state::before_done; - } - else - { - state_ = json_parse_state::expect_comma_or_end; - } - break; - default: - err_handler_.fatal_error(json_errc::invalid_value, *this); - ec = json_errc::invalid_value; - continue_ = false; - return; - } - ++input_ptr_; - ++column_; - break; - case json_parse_state::n: - switch (*input_ptr_) - { - case 'u': - ++input_ptr_; - ++column_; - state_ = json_parse_state::nu; - break; - default: - err_handler_.fatal_error(json_errc::invalid_value, *this); - ec = json_errc::invalid_value; - continue_ = false; - return; - } - break; - case json_parse_state::nu: - switch (*input_ptr_) - { - case 'l': - state_ = json_parse_state::nul; - break; - default: - err_handler_.fatal_error(json_errc::invalid_value, *this); - ec = json_errc::invalid_value; - continue_ = false; - return; - } - ++input_ptr_; - ++column_; - break; - case json_parse_state::nul: - switch (*input_ptr_) - { - case 'l': - continue_ = handler.null_value(semantic_tag::none, *this); - if (parent() == json_parse_state::root) - { - state_ = json_parse_state::before_done; - } - else - { - state_ = json_parse_state::expect_comma_or_end; - } - break; - default: - err_handler_.fatal_error(json_errc::invalid_value, *this); - ec = json_errc::invalid_value; - continue_ = false; - return; - } - ++input_ptr_; - ++column_; - break; - case json_parse_state::slash: - { - switch (*input_ptr_) - { - case '*': - state_ = json_parse_state::slash_star; - continue_ = err_handler_.error(json_errc::illegal_comment, *this); - if (!continue_) - { - ec = json_errc::illegal_comment; - return; - } - break; - case '/': - state_ = json_parse_state::slash_slash; - continue_ = err_handler_.error(json_errc::illegal_comment, *this); - if (!continue_) - { - ec = json_errc::illegal_comment; - return; - } - break; - default: - continue_ = err_handler_.error(json_errc::invalid_json_text, *this); - if (!continue_) - { - ec = json_errc::invalid_json_text; - return; - } - break; - } - ++input_ptr_; - ++column_; - break; - } - case json_parse_state::slash_star: - { - switch (*input_ptr_) - { - case '\r': - push_state(state_); - state_ = json_parse_state::cr; - break; - case '\n': - ++line_; - column_ = 1; - break; - case '*': - state_ = json_parse_state::slash_star_star; - break; - } - ++input_ptr_; - ++column_; - break; - } - case json_parse_state::slash_slash: - { - switch (*input_ptr_) - { - case '\r': - state_ = pop_state(); - break; - case '\n': - state_ = pop_state(); - break; - default: - ++input_ptr_; - ++column_; - } - break; - } - case json_parse_state::slash_star_star: - { - switch (*input_ptr_) - { - case '/': - state_ = pop_state(); - break; - default: - state_ = json_parse_state::slash_star; - break; - } - ++input_ptr_; - ++column_; - break; - } - default: - JSONCONS_ASSERT(false); - break; - } - } - } - - void parse_true(basic_json_content_handler& handler, std::error_code& ec) - { - if (JSONCONS_LIKELY(input_end_ - input_ptr_ >= 4)) - { - if (*(input_ptr_+1) == 'r' && *(input_ptr_+2) == 'u' && *(input_ptr_+3) == 'e') - { - continue_ = handler.bool_value(true, semantic_tag::none, *this); - input_ptr_ += 4; - column_ += 4; - if (parent() == json_parse_state::root) - { - state_ = json_parse_state::before_done; - } - else - { - state_ = json_parse_state::expect_comma_or_end; - } - } - else - { - err_handler_.fatal_error(json_errc::invalid_value, *this); - ec = json_errc::invalid_value; - continue_ = false; - return; - } - } - else - { - ++input_ptr_; - ++column_; - state_ = json_parse_state::t; - } - } - - void parse_null(basic_json_content_handler& handler, std::error_code& ec) - { - if (JSONCONS_LIKELY(input_end_ - input_ptr_ >= 4)) - { - if (*(input_ptr_+1) == 'u' && *(input_ptr_+2) == 'l' && *(input_ptr_+3) == 'l') - { - continue_ = handler.null_value(semantic_tag::none, *this); - input_ptr_ += 4; - column_ += 4; - if (parent() == json_parse_state::root) - { - state_ = json_parse_state::before_done; - } - else - { - state_ = json_parse_state::expect_comma_or_end; - } - } - else - { - err_handler_.fatal_error(json_errc::invalid_value, *this); - ec = json_errc::invalid_value; - continue_ = false; - return; - } - } - else - { - ++input_ptr_; - ++column_; - state_ = json_parse_state::n; - } - } - - void parse_false(basic_json_content_handler& handler, std::error_code& ec) - { - if (JSONCONS_LIKELY(input_end_ - input_ptr_ >= 5)) - { - if (*(input_ptr_+1) == 'a' && *(input_ptr_+2) == 'l' && *(input_ptr_+3) == 's' && *(input_ptr_+4) == 'e') - { - continue_ = handler.bool_value(false, semantic_tag::none, *this); - input_ptr_ += 5; - column_ += 5; - if (parent() == json_parse_state::root) - { - state_ = json_parse_state::before_done; - } - else - { - state_ = json_parse_state::expect_comma_or_end; - } - } - else - { - err_handler_.fatal_error(json_errc::invalid_value, *this); - ec = json_errc::invalid_value; - continue_ = false; - return; - } - } - else - { - ++input_ptr_; - ++column_; - state_ = json_parse_state::f; - } - } - - void parse_number(basic_json_content_handler& handler, std::error_code& ec) - { - const CharT* local_input_end = input_end_; - - switch (state_) - { - case json_parse_state::minus: - goto minus_sign; - case json_parse_state::zero: - goto zero; - case json_parse_state::integer: - goto integer; - case json_parse_state::fraction1: - goto fraction1; - case json_parse_state::fraction2: - goto fraction2; - case json_parse_state::exp1: - goto exp1; - case json_parse_state::exp2: - goto exp2; - case json_parse_state::exp3: - goto exp3; - default: - JSONCONS_UNREACHABLE(); - } -minus_sign: - if (JSONCONS_UNLIKELY(input_ptr_ >= local_input_end)) // Buffer exhausted - { - state_ = json_parse_state::minus; - return; - } - switch (*input_ptr_) - { - case '0': - string_buffer_.push_back(static_cast(*input_ptr_)); - ++input_ptr_; - ++column_; - goto zero; - case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - string_buffer_.push_back(static_cast(*input_ptr_)); - ++input_ptr_; - ++column_; - goto integer; - default: - err_handler_.fatal_error(json_errc::expected_value, *this); - ec = json_errc::expected_value; - continue_ = false; - return; - } -zero: - if (JSONCONS_UNLIKELY(input_ptr_ >= local_input_end)) // Buffer exhausted - { - state_ = json_parse_state::zero; - return; - } - switch (*input_ptr_) - { - case '\r': - end_integer_value(handler, ec); - if (ec) return; - ++input_ptr_; - ++column_; - push_state(state_); - state_ = json_parse_state::cr; - return; - case '\n': - end_integer_value(handler, ec); - if (ec) return; - ++input_ptr_; - ++line_; - column_ = 1; - return; - case ' ':case '\t': - end_integer_value(handler, ec); - if (ec) return; - skip_space(); - return; - case '/': - end_integer_value(handler, ec); - if (ec) return; - ++input_ptr_; - ++column_; - push_state(state_); - state_ = json_parse_state::slash; - return; - case '}': - end_integer_value(handler, ec); - if (ec) return; - state_ = json_parse_state::expect_comma_or_end; - return; - case ']': - end_integer_value(handler, ec); - if (ec) return; - state_ = json_parse_state::expect_comma_or_end; - return; - case '.': - string_buffer_.push_back(to_double_.get_decimal_point()); - ++input_ptr_; - ++column_; - goto fraction1; - case 'e':case 'E': - string_buffer_.push_back(static_cast(*input_ptr_)); - ++input_ptr_; - ++column_; - goto exp1; - case ',': - end_integer_value(handler, ec); - if (ec) return; - begin_member_or_element(ec); - if (ec) return; - ++input_ptr_; - ++column_; - return; - case '0': case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - err_handler_.fatal_error(json_errc::leading_zero, *this); - ec = json_errc::leading_zero; - continue_ = false; - state_ = json_parse_state::zero; - return; - default: - err_handler_.fatal_error(json_errc::invalid_number, *this); - ec = json_errc::invalid_number; - continue_ = false; - state_ = json_parse_state::zero; - return; - } -integer: - if (JSONCONS_UNLIKELY(input_ptr_ >= local_input_end)) // Buffer exhausted - { - state_ = json_parse_state::integer; - return; - } - switch (*input_ptr_) - { - case '\r': - end_integer_value(handler, ec); - if (ec) return; - push_state(state_); - ++input_ptr_; - ++column_; - state_ = json_parse_state::cr; - return; - case '\n': - end_integer_value(handler, ec); - if (ec) return; - ++input_ptr_; - ++line_; - column_ = 1; - return; - case ' ':case '\t': - end_integer_value(handler, ec); - if (ec) return; - skip_space(); - return; - case '/': - end_integer_value(handler, ec); - if (ec) return; - push_state(state_); - ++input_ptr_; - ++column_; - state_ = json_parse_state::slash; - return; - case '}': - end_integer_value(handler, ec); - if (ec) return; - state_ = json_parse_state::expect_comma_or_end; - return; - case ']': - end_integer_value(handler, ec); - if (ec) return; - state_ = json_parse_state::expect_comma_or_end; - return; - case '0': case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - string_buffer_.push_back(static_cast(*input_ptr_)); - ++input_ptr_; - ++column_; - goto integer; - case '.': - string_buffer_.push_back(to_double_.get_decimal_point()); - ++input_ptr_; - ++column_; - goto fraction1; - case 'e':case 'E': - string_buffer_.push_back(static_cast(*input_ptr_)); - ++input_ptr_; - ++column_; - goto exp1; - case ',': - end_integer_value(handler, ec); - if (ec) return; - begin_member_or_element(ec); - if (ec) return; - ++input_ptr_; - ++column_; - return; - default: - err_handler_.fatal_error(json_errc::invalid_number, *this); - ec = json_errc::invalid_number; - continue_ = false; - state_ = json_parse_state::integer; - return; - } -fraction1: - if (JSONCONS_UNLIKELY(input_ptr_ >= local_input_end)) // Buffer exhausted - { - state_ = json_parse_state::fraction1; - return; - } - switch (*input_ptr_) - { - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - string_buffer_.push_back(static_cast(*input_ptr_)); - ++input_ptr_; - ++column_; - goto fraction2; - default: - err_handler_.fatal_error(json_errc::invalid_number, *this); - ec = json_errc::invalid_number; - continue_ = false; - state_ = json_parse_state::fraction1; - return; - } -fraction2: - if (JSONCONS_UNLIKELY(input_ptr_ >= local_input_end)) // Buffer exhausted - { - state_ = json_parse_state::fraction2; - return; - } - switch (*input_ptr_) - { - case '\r': - end_fraction_value(handler, ec); - if (ec) return; - push_state(state_); - ++input_ptr_; - ++column_; - state_ = json_parse_state::cr; - return; - case '\n': - end_fraction_value(handler, ec); - if (ec) return; - ++input_ptr_; - ++line_; - column_ = 1; - return; - case ' ':case '\t': - end_fraction_value(handler, ec); - if (ec) return; - skip_space(); - return; - case '/': - end_fraction_value(handler, ec); - if (ec) return; - push_state(state_); - ++input_ptr_; - ++column_; - state_ = json_parse_state::slash; - return; - case '}': - end_fraction_value(handler, ec); - if (ec) return; - state_ = json_parse_state::expect_comma_or_end; - return; - case ']': - end_fraction_value(handler, ec); - if (ec) return; - state_ = json_parse_state::expect_comma_or_end; - return; - case ',': - end_fraction_value(handler, ec); - if (ec) return; - begin_member_or_element(ec); - if (ec) return; - ++input_ptr_; - ++column_; - return; - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - string_buffer_.push_back(static_cast(*input_ptr_)); - ++input_ptr_; - ++column_; - goto fraction2; - case 'e':case 'E': - string_buffer_.push_back(static_cast(*input_ptr_)); - ++input_ptr_; - ++column_; - goto exp1; - default: - err_handler_.fatal_error(json_errc::invalid_number, *this); - ec = json_errc::invalid_number; - continue_ = false; - state_ = json_parse_state::fraction2; - return; - } -exp1: - if (JSONCONS_UNLIKELY(input_ptr_ >= local_input_end)) // Buffer exhausted - { - state_ = json_parse_state::exp1; - return; - } - switch (*input_ptr_) - { - case '+': - ++input_ptr_; - ++column_; - goto exp2; - case '-': - string_buffer_.push_back(static_cast(*input_ptr_)); - ++input_ptr_; - ++column_; - goto exp2; - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - string_buffer_.push_back(static_cast(*input_ptr_)); - ++input_ptr_; - ++column_; - goto exp3; - default: - err_handler_.fatal_error(json_errc::expected_value, *this); - ec = json_errc::expected_value; - continue_ = false; - state_ = json_parse_state::exp1; - return; - } -exp2: - if (JSONCONS_UNLIKELY(input_ptr_ >= local_input_end)) // Buffer exhausted - { - state_ = json_parse_state::exp2; - return; - } - switch (*input_ptr_) - { - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - string_buffer_.push_back(static_cast(*input_ptr_)); - ++input_ptr_; - ++column_; - goto exp3; - default: - err_handler_.fatal_error(json_errc::expected_value, *this); - ec = json_errc::expected_value; - continue_ = false; - state_ = json_parse_state::exp2; - return; - } - -exp3: - if (JSONCONS_UNLIKELY(input_ptr_ >= local_input_end)) // Buffer exhausted - { - state_ = json_parse_state::exp3; - return; - } - switch (*input_ptr_) - { - case '\r': - end_fraction_value(handler, ec); - if (ec) return; - ++input_ptr_; - ++column_; - push_state(state_); - state_ = json_parse_state::cr; - return; - case '\n': - end_fraction_value(handler, ec); - if (ec) return; - ++input_ptr_; - ++line_; - column_ = 1; - return; - case ' ':case '\t': - end_fraction_value(handler, ec); - if (ec) return; - skip_space(); - return; - case '/': - end_fraction_value(handler, ec); - if (ec) return; - push_state(state_); - ++input_ptr_; - ++column_; - state_ = json_parse_state::slash; - return; - case '}': - end_fraction_value(handler, ec); - if (ec) return; - state_ = json_parse_state::expect_comma_or_end; - return; - case ']': - end_fraction_value(handler, ec); - if (ec) return; - state_ = json_parse_state::expect_comma_or_end; - return; - case ',': - end_fraction_value(handler, ec); - if (ec) return; - begin_member_or_element(ec); - if (ec) return; - ++input_ptr_; - ++column_; - return; - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - string_buffer_.push_back(static_cast(*input_ptr_)); - ++input_ptr_; - ++column_; - goto exp3; - default: - err_handler_.fatal_error(json_errc::invalid_number, *this); - ec = json_errc::invalid_number; - continue_ = false; - state_ = json_parse_state::exp3; - return; - } - - JSONCONS_UNREACHABLE(); - } - - void parse_string(basic_json_content_handler& handler, std::error_code& ec) - { - const CharT* local_input_end = input_end_; - const CharT* sb = input_ptr_; - - switch (state_) - { - case json_parse_state::string: - goto string_u1; - case json_parse_state::escape: - goto escape; - case json_parse_state::escape_u1: - goto escape_u1; - case json_parse_state::escape_u2: - goto escape_u2; - case json_parse_state::escape_u3: - goto escape_u3; - case json_parse_state::escape_u4: - goto escape_u4; - case json_parse_state::escape_expect_surrogate_pair1: - goto escape_expect_surrogate_pair1; - case json_parse_state::escape_expect_surrogate_pair2: - goto escape_expect_surrogate_pair2; - case json_parse_state::escape_u6: - goto escape_u6; - case json_parse_state::escape_u7: - goto escape_u7; - case json_parse_state::escape_u8: - goto escape_u8; - case json_parse_state::escape_u9: - goto escape_u9; - default: - JSONCONS_UNREACHABLE(); - } - -string_u1: - while (input_ptr_ < local_input_end) - { - switch (*input_ptr_) - { - JSONCONS_ILLEGAL_CONTROL_CHARACTER: - { - column_ += (input_ptr_ - sb + 1); - continue_ = err_handler_.error(json_errc::illegal_control_character, *this); - if (!continue_) - { - ec = json_errc::illegal_control_character; - state_ = json_parse_state::string; - return; - } - // recovery - skip - string_buffer_.append(sb,input_ptr_-sb); - ++input_ptr_; - state_ = json_parse_state::string; - return; - } - case '\r': - { - column_ += (input_ptr_ - sb + 1); - continue_ = err_handler_.error(json_errc::illegal_character_in_string, *this); - if (!continue_) - { - ec = json_errc::illegal_character_in_string; - state_ = json_parse_state::string; - return; - } - // recovery - keep - string_buffer_.append(sb, input_ptr_ - sb + 1); - ++input_ptr_; - push_state(state_); - state_ = json_parse_state::cr; - return; - } - case '\n': - { - ++line_; - column_ = 1; - continue_ = err_handler_.error(json_errc::illegal_character_in_string, *this); - if (!continue_) - { - ec = json_errc::illegal_character_in_string; - state_ = json_parse_state::string; - return; - } - // recovery - keep - string_buffer_.append(sb, input_ptr_ - sb + 1); - ++input_ptr_; - return; - } - case '\t': - { - column_ += (input_ptr_ - sb + 1); - continue_ = err_handler_.error(json_errc::illegal_character_in_string, *this); - if (!continue_) - { - ec = json_errc::illegal_character_in_string; - state_ = json_parse_state::string; - return; - } - // recovery - keep - string_buffer_.append(sb, input_ptr_ - sb + 1); - ++input_ptr_; - state_ = json_parse_state::string; - return; - } - case '\\': - { - string_buffer_.append(sb,input_ptr_-sb); - column_ += (input_ptr_ - sb + 1); - ++input_ptr_; - goto escape; - } - case '\"': - { - if (string_buffer_.length() == 0) - { - end_string_value(sb,input_ptr_-sb, handler, ec); - if (ec) {return;} - } - else - { - string_buffer_.append(sb,input_ptr_-sb); - end_string_value(string_buffer_.data(),string_buffer_.length(), handler, ec); - if (ec) {return;} - } - column_ += (input_ptr_ - sb + 1); - ++input_ptr_; - return; - } - default: - break; - } - ++input_ptr_; - } - - // Buffer exhausted - { - string_buffer_.append(sb,input_ptr_-sb); - column_ += (input_ptr_ - sb + 1); - state_ = json_parse_state::string; - return; - } - -escape: - if (JSONCONS_UNLIKELY(input_ptr_ >= local_input_end)) // Buffer exhausted - { - state_ = json_parse_state::escape; - return; - } - switch (*input_ptr_) - { - case '\"': - string_buffer_.push_back('\"'); - sb = ++input_ptr_; - ++column_; - goto string_u1; - case '\\': - string_buffer_.push_back('\\'); - sb = ++input_ptr_; - ++column_; - goto string_u1; - case '/': - string_buffer_.push_back('/'); - sb = ++input_ptr_; - ++column_; - goto string_u1; - case 'b': - string_buffer_.push_back('\b'); - sb = ++input_ptr_; - ++column_; - goto string_u1; - case 'f': - string_buffer_.push_back('\f'); - sb = ++input_ptr_; - ++column_; - goto string_u1; - case 'n': - string_buffer_.push_back('\n'); - sb = ++input_ptr_; - ++column_; - goto string_u1; - case 'r': - string_buffer_.push_back('\r'); - sb = ++input_ptr_; - ++column_; - goto string_u1; - case 't': - string_buffer_.push_back('\t'); - sb = ++input_ptr_; - ++column_; - goto string_u1; - case 'u': - cp_ = 0; - ++input_ptr_; - ++column_; - goto escape_u1; - default: - err_handler_.fatal_error(json_errc::illegal_escaped_character, *this); - ec = json_errc::illegal_escaped_character; - continue_ = false; - state_ = json_parse_state::escape; - return; - } - -escape_u1: - if (JSONCONS_UNLIKELY(input_ptr_ >= local_input_end)) // Buffer exhausted - { - state_ = json_parse_state::escape_u1; - return; - } - { - append_codepoint(*input_ptr_,ec); - if (ec) - { - state_ = json_parse_state::escape_u1; - return; - } - ++input_ptr_; - ++column_; - goto escape_u2; - } - -escape_u2: - if (JSONCONS_UNLIKELY(input_ptr_ >= local_input_end)) // Buffer exhausted - { - state_ = json_parse_state::escape_u2; - return; - } - { - append_codepoint(*input_ptr_, ec); - if (ec) - { - state_ = json_parse_state::escape_u2; - return; - } - ++input_ptr_; - ++column_; - goto escape_u3; - } - -escape_u3: - if (JSONCONS_UNLIKELY(input_ptr_ >= local_input_end)) // Buffer exhausted - { - state_ = json_parse_state::escape_u3; - return; - } - { - append_codepoint(*input_ptr_, ec); - if (ec) - { - state_ = json_parse_state::escape_u3; - return; - } - ++input_ptr_; - ++column_; - goto escape_u4; - } - -escape_u4: - if (JSONCONS_UNLIKELY(input_ptr_ >= local_input_end)) // Buffer exhausted - { - state_ = json_parse_state::escape_u4; - return; - } - { - append_codepoint(*input_ptr_, ec); - if (ec) - { - state_ = json_parse_state::escape_u4; - return; - } - if (unicons::is_high_surrogate(cp_)) - { - ++input_ptr_; - ++column_; - goto escape_expect_surrogate_pair1; - } - else - { - unicons::convert(&cp_, &cp_ + 1, std::back_inserter(string_buffer_)); - sb = ++input_ptr_; - ++column_; - state_ = json_parse_state::string; - return; - } - } - -escape_expect_surrogate_pair1: - if (JSONCONS_UNLIKELY(input_ptr_ >= local_input_end)) // Buffer exhausted - { - state_ = json_parse_state::escape_expect_surrogate_pair1; - return; - } - { - switch (*input_ptr_) - { - case '\\': - cp2_ = 0; - ++input_ptr_; - ++column_; - goto escape_expect_surrogate_pair2; - default: - err_handler_.fatal_error(json_errc::expected_codepoint_surrogate_pair, *this); - ec = json_errc::expected_codepoint_surrogate_pair; - continue_ = false; - state_ = json_parse_state::escape_expect_surrogate_pair1; - return; - } - } - -escape_expect_surrogate_pair2: - if (JSONCONS_UNLIKELY(input_ptr_ >= local_input_end)) // Buffer exhausted - { - state_ = json_parse_state::escape_expect_surrogate_pair2; - return; - } - { - switch (*input_ptr_) - { - case 'u': - ++input_ptr_; - ++column_; - goto escape_u6; - default: - err_handler_.fatal_error(json_errc::expected_codepoint_surrogate_pair, *this); - ec = json_errc::expected_codepoint_surrogate_pair; - continue_ = false; - state_ = json_parse_state::escape_expect_surrogate_pair2; - return; - } - } - -escape_u6: - if (JSONCONS_UNLIKELY(input_ptr_ >= local_input_end)) // Buffer exhausted - { - state_ = json_parse_state::escape_u6; - return; - } - { - append_second_codepoint(*input_ptr_, ec); - if (ec) - { - state_ = json_parse_state::escape_u6; - return; - } - } - ++input_ptr_; - ++column_; - goto escape_u7; - -escape_u7: - if (JSONCONS_UNLIKELY(input_ptr_ >= local_input_end)) // Buffer exhausted - { - state_ = json_parse_state::escape_u7; - return; - } - { - append_second_codepoint(*input_ptr_, ec); - if (ec) - { - state_ = json_parse_state::escape_u7; - return; - } - ++input_ptr_; - ++column_; - goto escape_u8; - } - -escape_u8: - if (JSONCONS_UNLIKELY(input_ptr_ >= local_input_end)) // Buffer exhausted - { - state_ = json_parse_state::escape_u8; - return; - } - { - append_second_codepoint(*input_ptr_, ec); - if (ec) - { - state_ = json_parse_state::escape_u8; - return; - } - ++input_ptr_; - ++column_; - goto escape_u9; - } - -escape_u9: - if (JSONCONS_UNLIKELY(input_ptr_ >= local_input_end)) // Buffer exhausted - { - state_ = json_parse_state::escape_u9; - return; - } - { - append_second_codepoint(*input_ptr_, ec); - if (ec) - { - state_ = json_parse_state::escape_u9; - return; - } - uint32_t cp = 0x10000 + ((cp_ & 0x3FF) << 10) + (cp2_ & 0x3FF); - unicons::convert(&cp, &cp + 1, std::back_inserter(string_buffer_)); - sb = ++input_ptr_; - ++column_; - goto string_u1; - } - - JSONCONS_UNREACHABLE(); - } - - void translate_conv_errc(unicons::conv_errc result, std::error_code& ec) - { - switch (result) - { - case unicons::conv_errc(): - break; - case unicons::conv_errc::over_long_utf8_sequence: - continue_ = err_handler_.error(json_errc::over_long_utf8_sequence, *this); - if (!continue_) - { - ec = json_errc::over_long_utf8_sequence; - return; - } - break; - case unicons::conv_errc::unpaired_high_surrogate: - continue_ = err_handler_.error(json_errc::unpaired_high_surrogate, *this); - if (!continue_) - { - ec = json_errc::unpaired_high_surrogate; - return; - } - break; - case unicons::conv_errc::expected_continuation_byte: - continue_ = err_handler_.error(json_errc::expected_continuation_byte, *this); - if (!continue_) - { - ec = json_errc::expected_continuation_byte; - return; - } - break; - case unicons::conv_errc::illegal_surrogate_value: - continue_ = err_handler_.error(json_errc::illegal_surrogate_value, *this); - if (!continue_) - { - ec = json_errc::illegal_surrogate_value; - return; - } - break; - default: - continue_ = err_handler_.error(json_errc::illegal_codepoint, *this); - if (!continue_) - { - ec = json_errc::illegal_codepoint; - return; - } - break; - } - } - -#if !defined(JSONCONS_NO_DEPRECATED) - - void end_parse(basic_json_content_handler& handler) - { - std::error_code ec; - finish_parse(handler, ec); - if (ec) - { - throw ser_error(ec,line_,column_); - } - } - - void end_parse(basic_json_content_handler& handler, std::error_code& ec) - { - while (!finished()) - { - parse_some(handler, ec); - } - } - - void set_source(const CharT* data, size_t length) - { - begin_input_ = data; - input_end_ = data + length; - input_ptr_ = begin_input_; - } -#endif - - size_t line() const override - { - return line_; - } - - size_t column() const override - { - return column_; - } -private: - - void end_integer_value(basic_json_content_handler& handler, std::error_code& ec) - { - if (string_buffer_[0] == '-') - { - end_negative_value(handler, ec); - } - else - { - end_positive_value(handler, ec); - } - } - - void end_negative_value(basic_json_content_handler& handler, std::error_code& ec) - { - auto result = jsoncons::detail::to_integer(string_buffer_.data(), string_buffer_.length()); - if (result.ec == jsoncons::detail::to_integer_errc()) - { - continue_ = handler.int64_value(result.value, semantic_tag::none, *this); - } - else // Must be overflow - { - continue_ = handler.string_value(string_buffer_, semantic_tag::bigint, *this); - } - after_value(ec); - } - - void end_positive_value(basic_json_content_handler& handler, std::error_code& ec) - { - auto result = jsoncons::detail::to_integer(string_buffer_.data(), string_buffer_.length()); - if (result.ec == jsoncons::detail::to_integer_errc()) - { - continue_ = handler.uint64_value(result.value, semantic_tag::none, *this); - } - else // Must be overflow - { - continue_ = handler.string_value(string_buffer_, semantic_tag::bigint, *this); - } - after_value(ec); - } - - void end_fraction_value(basic_json_content_handler& handler, std::error_code& ec) - { - try - { - if (options_.lossless_number()) - { - continue_ = handler.string_value(string_buffer_, semantic_tag::bigdec, *this); - } - else - { - double d = to_double_(string_buffer_.c_str(), string_buffer_.length()); - continue_ = handler.double_value(d, semantic_tag::none, *this); - } - } - catch (...) - { - continue_ = err_handler_.error(json_errc::invalid_number, *this); - if (!continue_) - { - ec = json_errc::invalid_number; - return; - } - continue_ = handler.null_value(semantic_tag::none, *this); // recovery - } - - after_value(ec); - } - - void append_codepoint(int c, std::error_code& ec) - { - switch (c) - { - case '0': case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - case 'a':case 'b':case 'c':case 'd':case 'e':case 'f': - case 'A':case 'B':case 'C':case 'D':case 'E':case 'F': - cp_ = append_to_codepoint(cp_, c, ec); - if (ec) return; - break; - default: - continue_ = err_handler_.error(json_errc::expected_value, *this); - if (!continue_) - { - ec = json_errc::expected_value; - return; - } - break; - } - } - - void append_second_codepoint(int c, std::error_code& ec) - { - switch (c) - { - case '0': - case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - case 'a':case 'b':case 'c':case 'd':case 'e':case 'f': - case 'A':case 'B':case 'C':case 'D':case 'E':case 'F': - cp2_ = append_to_codepoint(cp2_, c, ec); - if (ec) return; - break; - default: - continue_ = err_handler_.error(json_errc::expected_value, *this); - if (!continue_) - { - ec = json_errc::expected_value; - return; - } - break; - } - } - - void end_string_value(const CharT* s, size_t length, basic_json_content_handler& handler, std::error_code& ec) - { - auto result = unicons::validate(s,s+length); - if (result.ec != unicons::conv_errc()) - { - translate_conv_errc(result.ec,ec); - column_ += (result.it - s); - return; - } - switch (parent()) - { - case json_parse_state::member_name: - continue_ = handler.name(string_view_type(s, length), *this); - state_ = pop_state(); - state_ = json_parse_state::expect_colon; - break; - case json_parse_state::object: - case json_parse_state::array: - continue_ = handler.string_value(string_view_type(s, length), semantic_tag::none, *this); - state_ = json_parse_state::expect_comma_or_end; - break; - case json_parse_state::root: - continue_ = handler.string_value(string_view_type(s, length), semantic_tag::none, *this); - state_ = json_parse_state::before_done; - break; - default: - continue_ = err_handler_.error(json_errc::invalid_json_text, *this); - if (!continue_) - { - ec = json_errc::invalid_json_text; - return; - } - break; - } - } - - void begin_member_or_element(std::error_code& ec) - { - switch (parent()) - { - case json_parse_state::object: - state_ = json_parse_state::expect_member_name; - break; - case json_parse_state::array: - state_ = json_parse_state::expect_value; - break; - case json_parse_state::root: - break; - default: - continue_ = err_handler_.error(json_errc::invalid_json_text, *this); - if (!continue_) - { - ec = json_errc::invalid_json_text; - return; - } - break; - } - } - - void after_value(std::error_code& ec) - { - switch (parent()) - { - case json_parse_state::array: - case json_parse_state::object: - state_ = json_parse_state::expect_comma_or_end; - break; - case json_parse_state::root: - state_ = json_parse_state::before_done; - break; - default: - continue_ = err_handler_.error(json_errc::invalid_json_text, *this); - if (!continue_) - { - ec = json_errc::invalid_json_text; - return; - } - break; - } - } - - void push_state(json_parse_state state) - { - state_stack_.push_back(state); - } - - json_parse_state pop_state() - { - JSONCONS_ASSERT(!state_stack_.empty()) - json_parse_state state = state_stack_.back(); - state_stack_.pop_back(); - return state; - } - - uint32_t append_to_codepoint(uint32_t cp, int c, std::error_code& ec) - { - cp *= 16; - if (c >= '0' && c <= '9') - { - cp += c - '0'; - } - else if (c >= 'a' && c <= 'f') - { - cp += c - 'a' + 10; - } - else if (c >= 'A' && c <= 'F') - { - cp += c - 'A' + 10; - } - else - { - continue_ = err_handler_.error(json_errc::invalid_hex_escape_sequence, *this); - if (!continue_) - { - ec = json_errc::invalid_hex_escape_sequence; - return cp; - } - } - return cp; - } -}; - -typedef basic_json_parser json_parser; -typedef basic_json_parser wjson_parser; - -} - -#endif - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_pull_reader.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_pull_reader.hpp deleted file mode 100644 index 4a9b58a685..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_pull_reader.hpp +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSON_PULL_READER_HPP -#define JSONCONS_JSON_PULL_READER_HPP - -#include - -#endif - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_reader.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_reader.hpp deleted file mode 100644 index d67d1ec12f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_reader.hpp +++ /dev/null @@ -1,514 +0,0 @@ -// Copyright 2015 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSON_READER_HPP -#define JSONCONS_JSON_READER_HPP - -#include // std::allocator -#include -#include -#include -#include -#include -#include // std::move -#include -#include -#include -#include -#include - -namespace jsoncons { - -// utf8_other_json_input_adapter - -template -class json_utf8_other_content_handler_adapter : public json_content_handler -{ -public: - using json_content_handler::string_view_type; -private: - basic_null_json_content_handler default_content_handler_; - basic_json_content_handler& other_handler_; - //parse_error_handler& err_handler_; - - // noncopyable and nonmoveable - json_utf8_other_content_handler_adapter(const json_utf8_other_content_handler_adapter&) = delete; - json_utf8_other_content_handler_adapter& operator=(const json_utf8_other_content_handler_adapter&) = delete; - -public: - json_utf8_other_content_handler_adapter() - : other_handler_(default_content_handler_) - { - } - - json_utf8_other_content_handler_adapter(basic_json_content_handler& other_handler/*, - parse_error_handler& err_handler*/) - : other_handler_(other_handler)/*, - err_handler_(err_handler)*/ - { - } - -private: - - void do_flush() override - { - other_handler_.flush(); - } - - bool do_begin_object(semantic_tag tag, const ser_context& context) override - { - return other_handler_.begin_object(tag, context); - } - - bool do_end_object(const ser_context& context) override - { - return other_handler_.end_object(context); - } - - bool do_begin_array(semantic_tag tag, const ser_context& context) override - { - return other_handler_.begin_array(tag, context); - } - - bool do_end_array(const ser_context& context) override - { - return other_handler_.end_array(context); - } - - bool do_name(const string_view_type& name, const ser_context& context) override - { - std::basic_string target; - auto result = unicons::convert( - name.begin(), name.end(), std::back_inserter(target), - unicons::conv_flags::strict); - if (result.ec != unicons::conv_errc()) - { - throw ser_error(result.ec,context.line(),context.column()); - } - return other_handler_.name(target, context); - } - - bool do_string_value(const string_view_type& value, semantic_tag tag, const ser_context& context) override - { - std::basic_string target; - auto result = unicons::convert( - value.begin(), value.end(), std::back_inserter(target), - unicons::conv_flags::strict); - if (result.ec != unicons::conv_errc()) - { - throw ser_error(result.ec,context.line(),context.column()); - } - return other_handler_.string_value(target, tag, context); - } - - bool do_int64_value(int64_t value, - semantic_tag tag, - const ser_context& context) override - { - return other_handler_.int64_value(value, tag, context); - } - - bool do_uint64_value(uint64_t value, - semantic_tag tag, - const ser_context& context) override - { - return other_handler_.uint64_value(value, tag, context); - } - - bool do_double_value(double value, - semantic_tag tag, - const ser_context& context) override - { - return other_handler_.double_value(value, tag, context); - } - - bool do_bool_value(bool value, semantic_tag tag, const ser_context& context) override - { - return other_handler_.bool_value(value, tag, context); - } - - bool do_null_value(semantic_tag tag, const ser_context& context) override - { - return other_handler_.null_value(tag, context); - } -}; - -template,class Allocator=std::allocator> -class basic_json_reader -{ -public: - typedef CharT char_type; - typedef Src source_type; - typedef basic_string_view string_view_type; - typedef Allocator allocator_type; -private: - typedef typename std::allocator_traits:: template rebind_alloc char_allocator_type; - - static const size_t default_max_buffer_length = 16384; - - basic_null_json_content_handler default_content_handler_; - default_parse_error_handler default_err_handler_; - - basic_json_content_handler& handler_; - - basic_json_parser parser_; - - source_type source_; - std::vector buffer_; - size_t buffer_length_; - bool eof_; - bool begin_; - - // Noncopyable and nonmoveable - basic_json_reader(const basic_json_reader&) = delete; - basic_json_reader& operator=(const basic_json_reader&) = delete; - -public: - template - explicit basic_json_reader(Source&& source) - : basic_json_reader(std::forward(source), - default_content_handler_, - basic_json_options::default_options(), - default_err_handler_) - { - } - - template - basic_json_reader(Source&& source, - const basic_json_decode_options& options) - : basic_json_reader(std::forward(source), - default_content_handler_, - options, - default_err_handler_) - { - } - - template - basic_json_reader(Source&& source, - parse_error_handler& err_handler) - : basic_json_reader(std::forward(source), - default_content_handler_, - basic_json_options::default_options(), - err_handler) - { - } - - template - basic_json_reader(Source&& source, - const basic_json_decode_options& options, - parse_error_handler& err_handler) - : basic_json_reader(std::forward(source), - default_content_handler_, - options, - err_handler) - { - } - - template - basic_json_reader(Source&& source, - basic_json_content_handler& handler) - : basic_json_reader(std::forward(source), - handler, - basic_json_options::default_options(), - default_err_handler_) - { - } - - template - basic_json_reader(Source&& source, - basic_json_content_handler& handler, - const basic_json_decode_options& options) - : basic_json_reader(std::forward(source), - handler, - options, - default_err_handler_) - { - } - - template - basic_json_reader(Source&& source, - basic_json_content_handler& handler, - parse_error_handler& err_handler) - : basic_json_reader(std::forward(source), - handler, - basic_json_options::default_options(), - err_handler) - { - } - - template - basic_json_reader(Source&& source, - basic_json_content_handler& handler, - const basic_json_decode_options& options, - parse_error_handler& err_handler, - typename std::enable_if,Source>::value>::type* = 0) - : handler_(handler), - parser_(options,err_handler), - source_(std::forward(source)), - buffer_length_(default_max_buffer_length), - eof_(false), - begin_(true) - { - buffer_.reserve(buffer_length_); - } - - template - basic_json_reader(Source&& source, - basic_json_content_handler& handler, - const basic_json_decode_options& options, - parse_error_handler& err_handler, - typename std::enable_if,Source>::value>::type* = 0) - : handler_(handler), - parser_(options,err_handler), - buffer_length_(0), - eof_(false), - begin_(false) - { - basic_string_view sv(std::forward(source)); - auto result = unicons::skip_bom(sv.begin(), sv.end()); - if (result.ec != unicons::encoding_errc()) - { - throw ser_error(result.ec,parser_.line(),parser_.column()); - } - size_t offset = result.it - sv.begin(); - parser_.update(sv.data()+offset,sv.size()-offset); - } - - size_t buffer_length() const - { - return buffer_length_; - } - - void buffer_length(size_t length) - { - buffer_length_ = length; - buffer_.reserve(buffer_length_); - } -#if !defined(JSONCONS_NO_DEPRECATED) - size_t max_nesting_depth() const - { - return parser_.max_nesting_depth(); - } - - void max_nesting_depth(size_t depth) - { - parser_.max_nesting_depth(depth); - } -#endif - void read_next() - { - std::error_code ec; - read_next(ec); - if (ec) - { - throw ser_error(ec,parser_.line(),parser_.column()); - } - } - - void read_next(std::error_code& ec) - { - try - { - if (source_.is_error()) - { - ec = json_errc::source_error; - return; - } - parser_.reset(); - while (!parser_.finished()) - { - if (parser_.source_exhausted()) - { - if (!source_.eof()) - { - read_buffer(ec); - if (ec) return; - } - else - { - eof_ = true; - } - } - parser_.parse_some(handler_, ec); - if (ec) return; - } - - while (!eof_) - { - parser_.skip_whitespace(); - if (parser_.source_exhausted()) - { - if (!source_.eof()) - { - read_buffer(ec); - if (ec) return; - } - else - { - eof_ = true; - } - } - else - { - break; - } - } - } - catch (const ser_error& e) - { - ec = e.code(); - } - } - - void check_done() - { - std::error_code ec; - check_done(ec); - if (ec) - { - throw ser_error(ec,parser_.line(),parser_.column()); - } - } - - size_t line() const - { - return parser_.line(); - } - - size_t column() const - { - return parser_.column(); - } - - void check_done(std::error_code& ec) - { - try - { - if (source_.is_error()) - { - ec = json_errc::source_error; - return; - } - if (eof_) - { - parser_.check_done(ec); - if (ec) return; - } - else - { - while (!eof_) - { - if (parser_.source_exhausted()) - { - if (!source_.eof()) - { - read_buffer(ec); - if (ec) return; - } - else - { - eof_ = true; - } - } - if (!eof_) - { - parser_.check_done(ec); - if (ec) return; - } - } - } - } - catch (const ser_error& e) - { - ec = e.code(); - } - } - - bool eof() const - { - return eof_; - } - - void read() - { - read_next(); - check_done(); - } - - void read(std::error_code& ec) - { - read_next(ec); - if (!ec) - { - check_done(ec); - } - } - -#if !defined(JSONCONS_NO_DEPRECATED) - - size_t buffer_capacity() const - { - return buffer_length_; - } - - void buffer_capacity(size_t length) - { - buffer_length_ = length; - buffer_.reserve(buffer_length_); - } - size_t max_depth() const - { - return parser_.max_nesting_depth(); - } - - void max_depth(size_t depth) - { - parser_.max_nesting_depth(depth); - } -#endif - -private: - - void read_buffer(std::error_code& ec) - { - buffer_.clear(); - buffer_.resize(buffer_length_); - size_t count = source_.read(buffer_.data(), buffer_length_); - buffer_.resize(static_cast(count)); - if (buffer_.size() == 0) - { - eof_ = true; - } - else if (begin_) - { - auto result = unicons::skip_bom(buffer_.begin(), buffer_.end()); - if (result.ec != unicons::encoding_errc()) - { - ec = result.ec; - return; - } - size_t offset = result.it - buffer_.begin(); - parser_.update(buffer_.data()+offset,buffer_.size()-offset); - begin_ = false; - } - else - { - parser_.update(buffer_.data(),buffer_.size()); - } - } -}; - -typedef basic_json_reader json_reader; -typedef basic_json_reader wjson_reader; -#if !defined(JSONCONS_NO_DEPRECATED) -typedef basic_json_reader> json_string_reader; -typedef basic_json_reader> wjson_string_reader; -#endif - -} - -#endif - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_serializer.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_serializer.hpp deleted file mode 100644 index 2db2635cda..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_serializer.hpp +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSON_SERIALIZER_HPP -#define JSONCONS_JSON_SERIALIZER_HPP - -#include - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_type_traits.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_type_traits.hpp deleted file mode 100644 index d21995eca0..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_type_traits.hpp +++ /dev/null @@ -1,1047 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSON_TYPE_TRAITS_HPP -#define JSONCONS_JSON_TYPE_TRAITS_HPP - -#include -#include -#include -#include -#include -#include -#include -#include // std::swap -#include // std::numeric_limits -#include // std::enable_if -#include // std::iterator_traits, std::input_iterator_tag -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace jsoncons { - - -template -struct is_basic_json_class : std::false_type -{}; - -#if defined(_MSC_VER) && _MSC_VER < 1916 -template -struct is_basic_json_class::value && - !std::is_void::value && - !std::is_void::value>::type> : std::true_type -{}; -#else -template -struct is_basic_json_class>(),void())> : std::true_type -{}; -#endif - -template -struct is_json_type_traits_declared : public std::false_type -{}; - -#if !defined(JSONCONS_NO_DEPRECATED) -template -using is_json_type_traits_impl = is_json_type_traits_declared; -#endif - -// json_type_traits - -template -struct unimplemented : std::false_type -{}; - -template -struct json_type_traits -{ - typedef typename Json::allocator_type allocator_type; - - static constexpr bool is_compatible = false; - - static constexpr bool is(const Json&) - { - return false; - } - - static T as(const Json&) - { - static_assert(unimplemented::value, "as not implemented"); - } - - static Json to_json(const T&, allocator_type = allocator_type()) - { - static_assert(unimplemented::value, "to_json not implemented"); - } -}; - -namespace detail { - -// is_incompatible -template -struct is_incompatible : std::false_type {}; - - -// is_incompatible -template -struct is_incompatible::is_compatible>::value>::type -> : std::true_type {}; - -// is_compatible_string_type -template -struct is_compatible_string_type : std::false_type {}; - -template -struct is_compatible_string_type::value && - jsoncons::detail::is_string_like::value && - !is_incompatible::value_type>::value ->::type> : std::true_type {}; - -// is_compatible_string_view_type -template -struct is_compatible_string_view_type : std::false_type {}; - -template -struct is_compatible_string_view_type::value && - jsoncons::detail::is_string_view_like::value && - !is_incompatible::value_type>::value ->::type> : std::true_type {}; - -// is_compatible_array_type -template -struct is_compatible_array_type : std::false_type {}; - -template -struct is_compatible_array_type::value && - jsoncons::detail::is_vector_like::value && - !is_incompatible::value_type>::value ->::type> : std::true_type {}; - -// is_compatible_object_type -template -struct is_compatible_object_type : std::false_type {}; - -template -struct is_compatible_object_type::value ->::type> : std::true_type {}; - -template -class json_array_input_iterator -{ -public: - typedef typename Json::const_array_iterator iterator_base; - typedef typename std::iterator_traits::value_type value_type; - typedef typename std::iterator_traits::difference_type difference_type; - typedef typename std::iterator_traits::pointer pointer; - typedef T reference; - typedef std::input_iterator_tag iterator_category; - - json_array_input_iterator() - { - } - - json_array_input_iterator(iterator_base it) - : it_(it) - { - } - - json_array_input_iterator& operator=(json_array_input_iterator rhs) - { - swap(*this,rhs); - return *this; - } - - json_array_input_iterator& operator++() - { - ++it_; - return *this; - } - - json_array_input_iterator operator++(int) // postfix increment - { - json_array_input_iterator temp(*this); - ++it_; - return temp; - } - - json_array_input_iterator& operator--() - { - --it_; - return *this; - } - - json_array_input_iterator operator--(int) - { - json_array_input_iterator temp(*this); - --it_; - return temp; - } - - reference operator*() const - { - return json_type_traits::as(*it_); - } - - friend bool operator==(const json_array_input_iterator& it1, const json_array_input_iterator& it2) - { - return it1.it_ == it2.it_; - } - friend bool operator!=(const json_array_input_iterator& it1, const json_array_input_iterator& it2) - { - return !(it1.it_ == it2.it_); - } - friend void swap(json_array_input_iterator& lhs, json_array_input_iterator& rhs) - { - using std::swap; - swap(lhs.it_,rhs.it_); - swap(lhs.empty_,rhs.empty_); - } - -private: - iterator_base it_; -}; - -template -class json_object_input_iterator -{ -public: - typedef typename Json::const_object_iterator iterator_base; - typedef typename std::iterator_traits::value_type value_type; - typedef typename std::iterator_traits::difference_type difference_type; - typedef typename std::iterator_traits::pointer pointer; - typedef T reference; - typedef std::input_iterator_tag iterator_category; - typedef typename T::first_type key_type; - typedef typename T::second_type mapped_type; - - json_object_input_iterator() - { - } - - json_object_input_iterator(iterator_base it) - : it_(it) - { - } - - json_object_input_iterator& operator=(json_object_input_iterator rhs) - { - swap(*this,rhs); - return *this; - } - - json_object_input_iterator& operator++() - { - ++it_; - return *this; - } - - json_object_input_iterator operator++(int) // postfix increment - { - json_object_input_iterator temp(*this); - ++it_; - return temp; - } - - json_object_input_iterator& operator--() - { - --it_; - return *this; - } - - json_object_input_iterator operator--(int) - { - json_object_input_iterator temp(*this); - --it_; - return temp; - } - - reference operator*() const - { - return T(key_type(it_->key()),json_type_traits::as(it_->value())); - } - - friend bool operator==(const json_object_input_iterator& it1, const json_object_input_iterator& it2) - { - return it1.it_ == it2.it_; - } - friend bool operator!=(const json_object_input_iterator& it1, const json_object_input_iterator& it2) - { - return !(it1.it_ == it2.it_); - } - friend void swap(json_object_input_iterator& lhs, json_object_input_iterator& rhs) - { - using std::swap; - swap(lhs.it_,rhs.it_); - swap(lhs.empty_,rhs.empty_); - } - -private: - iterator_base it_; -}; - -} - -template -struct json_type_traits::const_pointer_type> -{ - typedef typename Json::char_type char_type; - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - return j.is_string(); - } - static const char_type* as(const Json& j) - { - return j.as_cstring(); - } - template - static Json to_json(const char_type* s, Args&&... args) - { - return Json(s, semantic_tag::none, std::forward(args)...); - } -}; - -template -struct json_type_traits::pointer_type> -{ - typedef typename Json::char_type char_type; - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - return j.is_string(); - } - template - static Json to_json(const char_type* s, Args&&... args) - { - return Json(s, semantic_tag::none, std::forward(args)...); - } -}; - -// integral - -template -struct json_type_traits::value ->::type> -{ - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - if (j.is_int64()) - { - return (j.template as_integer() >= (std::numeric_limits::lowest)()) && (j.template as_integer() <= (std::numeric_limits::max)()); - } - else if (j.is_uint64()) - { - return j.template as_integer() <= static_cast((std::numeric_limits::max)()); - } - else - { - return false; - } - } - static T as(const Json& j) - { - return j.template as_integer(); - } - static Json to_json(T val, allocator_type = allocator_type()) - { - return Json(val, semantic_tag::none); - } -}; - -template -struct json_type_traits::value>::type> -{ - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - if (j.is_int64()) - { - return j.template as_integer() >= 0 && static_cast(j.template as_integer()) <= (std::numeric_limits::max)(); - } - else if (j.is_uint64()) - { - return j.template as_integer() <= (std::numeric_limits::max)(); - } - else - { - return false; - } - } - - static T as(const Json& j) - { - return j.template as_integer(); - } - - static Json to_json(T val, allocator_type = allocator_type()) - { - return Json(val, semantic_tag::none); - } -}; - -template -struct json_type_traits::value ->::type> -{ - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - return j.is_double(); - } - static T as(const Json& j) - { - return static_cast(j.as_double()); - } - static Json to_json(T val, allocator_type = allocator_type()) - { - return Json(val, semantic_tag::none); - } -}; - -template -struct json_type_traits -{ - typedef typename Json::object json_object; - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - return j.is_object(); - } - static Json to_json(const json_object& o) - { - return Json(o,semantic_tag::none); - } -}; - -template -struct json_type_traits -{ - typedef typename Json::array json_array; - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - return j.is_array(); - } - static Json to_json(const json_array& a) - { - return Json(a, semantic_tag::none); - } -}; - -template -struct json_type_traits -{ - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json&) noexcept - { - return true; - } - static Json as(Json j) - { - return j; - } - static Json to_json(const Json& val, allocator_type = allocator_type()) - { - return val; - } -}; - -template -struct json_type_traits -{ - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - return j.is_null(); - } - static typename jsoncons::null_type as(const Json& j) - { - JSONCONS_ASSERT(j.is_null()); - return jsoncons::null_type(); - } - static Json to_json(jsoncons::null_type, allocator_type = allocator_type()) - { - return Json::null(); - } -}; - -template -struct json_type_traits -{ - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - return j.is_bool(); - } - static bool as(const Json& j) - { - return j.as_bool(); - } - static Json to_json(bool val, allocator_type = allocator_type()) - { - return Json(val, semantic_tag::none); - } -}; - -template -struct json_type_traits::const_reference>::value, - std::vector::const_reference, - void>::type>::value>::type> -{ - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - return j.is_bool(); - } - static bool as(const Json& j) - { - return j.as_bool(); - } - static Json to_json(bool val, allocator_type = allocator_type()) - { - return Json(val, semantic_tag::none); - } -}; - -template -struct json_type_traits::reference> -{ - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - return j.is_bool(); - } - static bool as(const Json& j) - { - return j.as_bool(); - } - static Json to_json(bool val, allocator_type = allocator_type()) - { - return Json(val, semantic_tag::none); - } -}; - -template -struct json_type_traits::value && jsoncons::detail::is_compatible_array_type::value>::type> -{ - typedef typename std::iterator_traits::value_type element_type; - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - bool result = j.is_array(); - if (result) - { - for (auto e : j.array_range()) - { - if (!e.template is()) - { - result = false; - break; - } - } - } - return result; - } - - template - static typename std::enable_if::value && !std::is_same::value),T>::type - as(const Json& j) - { - if (j.is_array()) - { - T v(jsoncons::detail::json_array_input_iterator(j.array_range().begin()), - jsoncons::detail::json_array_input_iterator(j.array_range().end())); - return v; - } - else - { - JSONCONS_THROW(json_runtime_error("Attempt to cast json non-array to array")); - } - } - - template - static typename std::enable_if::value && !std::is_same::value,T>::type - as(const Json& j) - { - if (j.is_array()) - { - T v(jsoncons::detail::json_array_input_iterator(j.array_range().begin()), - jsoncons::detail::json_array_input_iterator(j.array_range().end())); - return v; - } - else if (j.is_byte_string_view()) - { - T v(j.as_byte_string_view().begin(),j.as_byte_string_view().end()); - return v; - } - else if (j.is_byte_string()) - { - auto s = j.as_byte_string(); - T v(s.begin(),s.end()); - return v; - } - else - { - JSONCONS_THROW(json_runtime_error("Attempt to cast json non-array to array")); - } - } - - static Json to_json(const T& val) - { - Json j = typename Json::array(); - auto first = std::begin(val); - auto last = std::end(val); - size_t size = std::distance(first,last); - j.reserve(size); - for (auto it = first; it != last; ++it) - { - j.push_back(*it); - } - return j; - } - - static Json to_json(const T& val, const allocator_type& allocator) - { - Json j = typename Json::array(allocator); - auto first = std::begin(val); - auto last = std::end(val); - size_t size = std::distance(first, last); - j.reserve(size); - for (auto it = first; it != last; ++it) - { - j.push_back(*it); - } - return j; - } -}; - -template -struct json_type_traits::value && jsoncons::detail::is_compatible_string_type::value>::type> -{ - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - return j.is_string(); - } - - static T as(const Json& j) - { - return T(j.as_string()); - } - - static Json to_json(const T& val) - { - return Json(val, semantic_tag::none); - } - - static Json to_json(const T& val, const allocator_type& allocator) - { - return Json(val, semantic_tag::none, allocator); - } -}; - -template -struct json_type_traits::value && jsoncons::detail::is_compatible_string_view_type::value>::type> -{ - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - return j.is_string_view(); - } - - static T as(const Json& j) - { - return T(j.as_string_view().data(),j.as_string_view().size()); - } - - static Json to_json(const T& val) - { - return Json(val, semantic_tag::none); - } - - static Json to_json(const T& val, const allocator_type& allocator) - { - return Json(val, semantic_tag::none, allocator); - } -}; - -template -struct json_type_traits::value && jsoncons::detail::is_compatible_object_type::value>::type -> -{ - typedef typename T::mapped_type mapped_type; - typedef typename T::value_type value_type; - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - bool result = j.is_object(); - for (auto member : j.object_range()) - { - if (!member.value().template is()) - { - result = false; - } - } - return result; - } - - static T as(const Json& j) - { - T v(jsoncons::detail::json_object_input_iterator(j.object_range().begin()), - jsoncons::detail::json_object_input_iterator(j.object_range().end())); - return v; - } - - static Json to_json(const T& val) - { - Json j = typename Json::object(val.begin(), val.end()); - return j; - } - - static Json to_json(const T& val, const allocator_type& allocator) - { - Json j = typename Json::object(val.begin(), val.end(), allocator); - return j; - } -}; - -template -struct json_type_traits> -{ - typedef typename Json::allocator_type allocator_type; - - typedef E element_type; - - static bool is(const Json& j) noexcept - { - bool result = j.is_array() && j.size() == N; - if (result) - { - for (auto e : j.array_range()) - { - if (!e.template is()) - { - result = false; - break; - } - } - } - return result; - } - - static std::array as(const Json& j) - { - std::array buff; - JSONCONS_ASSERT(j.size() == N); - for (size_t i = 0; i < N; i++) - { - buff[i] = j[i].template as(); - } - return buff; - } - - static Json to_json(const std::array& val) - { - Json j = typename Json::array(); - j.reserve(N); - for (auto it = val.begin(); it != val.end(); ++it) - { - j.push_back(*it); - } - return j; - } - - static Json to_json(const std::array& val, - const allocator_type& allocator) - { - Json j = typename Json::array(allocator); - j.reserve(N); - for (auto it = val.begin(); it != val.end(); ++it) - { - j.push_back(*it); - } - return j; - } -}; - -namespace detail -{ - -template -struct json_tuple_helper -{ - using element_type = typename std::tuple_element::type; - using next = json_tuple_helper; - - static bool is(const Json& j) noexcept - { - if(j[Pos - 1].template is()) - { - return next::is(j); - } - else - { - return false; - } - } - - static void as(Tuple& tuple, const Json& j) - { - std::get(tuple) = j[Pos - 1].template as(); - next::as(tuple, j); - } - - static void to_json(const Tuple& tuple, std::array::value>& jsons) - { - jsons[Pos - 1] = json_type_traits::to_json(std::get(tuple)); - next::to_json(tuple, jsons); - } -}; - -template -struct json_tuple_helper<0, Json, Tuple> -{ - static bool is(const Json&) noexcept - { - return true; - } - - static void as(Tuple&, const Json&) - { - } - - static void to_json(const Tuple&, std::array::value>&) - { - } -}; - -} - -template -struct json_type_traits> -{ -private: - using helper = jsoncons::detail::json_tuple_helper>; - -public: - static bool is(const Json& j) noexcept - { - return helper::is(j); - } - - static std::tuple as(const Json& j) - { - std::tuple buff; - helper::as(buff, j); - return buff; - } - - static Json to_json(const std::tuple& val) - { - std::array buf; - helper::to_json(val, buf); - return Json(typename Json::array(buf.begin(), buf.end())); - } -}; - -template -struct json_type_traits> -{ -public: - static bool is(const Json& j) noexcept - { - return j.is_array() && j.size() == 2; - } - - static std::pair as(const Json& j) - { - return std::make_pair(j[0].template as(),j[1].template as()); - } - - static Json to_json(const std::pair& val) - { - return typename Json::array{val.first,val.second}; - } -}; - -template -struct json_type_traits> -{ -public: - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - return j.is_byte_string(); - } - - static basic_byte_string as(const Json& j) - { - return j.template as_byte_string(); - } - - static Json to_json(const basic_byte_string& val, - const allocator_type& allocator = allocator_type()) - { - return Json(val, semantic_tag::none, allocator); - } -}; - -template -struct json_type_traits -{ -public: - static bool is(const Json& j) noexcept - { - return j.is_byte_string_view(); - } - - static byte_string_view as(const Json& j) - { - return j.as_byte_string_view(); - } - - static Json to_json(const byte_string_view& val) - { - return Json(val); - } -}; - -// basic_bignum - -template -struct json_type_traits> -{ -public: - static bool is(const Json& j) noexcept - { - return j.is_bignum(); - } - - static basic_bignum as(const Json& j) - { - return j.as_bignum(); - } - - static Json to_json(const basic_bignum& val) - { - std::basic_string s; - val.dump(s); - return Json(s,semantic_tag::bigint); - } -}; - -// std::valarray - -template -struct json_type_traits> -{ - typedef typename Json::allocator_type allocator_type; - - static bool is(const Json& j) noexcept - { - bool result = j.is_array(); - if (result) - { - for (auto e : j.array_range()) - { - if (!e.template is()) - { - result = false; - break; - } - } - } - return result; - } - - static std::valarray as(const Json& j) - { - if (j.is_array()) - { - std::valarray v(j.size()); - for (size_t i = 0; i < j.size(); ++i) - { - v[i] = j[i].template as(); - } - return v; - } - else - { - JSONCONS_THROW(json_runtime_error("Attempt to cast json non-array to array")); - } - } - - static Json to_json(const std::valarray& val) - { - Json j = typename Json::array(); - auto first = std::begin(val); - auto last = std::end(val); - size_t size = std::distance(first,last); - j.reserve(size); - for (auto it = first; it != last; ++it) - { - j.push_back(*it); - } - return j; - } - - static Json to_json(const std::valarray& val, const allocator_type& allocator) - { - Json j = typename Json::array(allocator); - auto first = std::begin(val); - auto last = std::end(val); - size_t size = std::distance(first,last); - j.reserve(size); - for (auto it = first; it != last; ++it) - { - j.push_back(*it); - } - return j; - } -}; - -} // jsoncons - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_type_traits_macros.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_type_traits_macros.hpp deleted file mode 100644 index 8d29fed0c3..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/json_type_traits_macros.hpp +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright 2019 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSON_TYPE_TRAITS_MACROS_HPP -#define JSONCONS_JSON_TYPE_TRAITS_MACROS_HPP - -#include -#include -#include -#include -#include -#include -#include -#include // std::swap -#include // std::numeric_limits -#include // std::enable_if -#include // std::iterator_traits, std::input_iterator_tag -#include -#include -#include -#include -#include -#include -#include -#include - -// This follows https://github.com/Loki-Astari/ThorsSerializer/blob/master/src/Serialize/Traits.h - -#define JSONCONS_EXPAND(X) X - -#define JSONCONS_NARGS(...) JSONCONS_NARG_(__VA_ARGS__, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) -#define JSONCONS_NARG_(...) JSONCONS_EXPAND( JSONCONS_ARG_N(__VA_ARGS__) ) -#define JSONCONS_ARG_N(e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13, e14, e15, e16, e17, e18, e19, e20, e21, e22, e23, e24, e25, e26, e27, e28, e29, e30, e31, e32, e33, e34, e35, e36, e37, e38, e39, e40, e41, e42, e43, e44, e45, e46, e47, e48, e49, e50, N, ...) N - -#define JSONCONS_QUOTE(A) JSONCONS_EXPAND(#A) - -#define JSONCONS_GENERATE(Call, TC, JVal, TVal, P2) Call(TC, JVal, TVal, P2) - -#define JSONCONS_REP_N(Call, TC, JVal, TVal, ...) JSONCONS_REP_OF_N(Call, TC, JVal, TVal, JSONCONS_NARGS(__VA_ARGS__), __VA_ARGS__) -#define JSONCONS_REP_OF_N(Call, TC, JVal, TVal, Count, ...) JSONCONS_REP_OF_N_(Call, TC, JVal, TVal, Count, __VA_ARGS__) -#define JSONCONS_REP_OF_N_(Call, TC, JVal, TVal, Count, ...) JSONCONS_EXPAND(JSONCONS_REP_OF_ ## Count(Call, TC, JVal, TVal, __VA_ARGS__)) - -#define JSONCONS_REP_OF_50(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_49(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_49(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_48(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_48(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_47(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_47(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_46(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_46(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_45(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_45(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_44(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_44(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_43(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_43(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_42(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_42(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_41(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_41(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_40(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_40(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_39(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_39(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_38(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_38(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_37(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_37(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_36(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_36(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_35(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_35(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_34(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_34(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_33(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_33(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_32(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_32(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_31(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_31(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_30(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_30(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_29(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_29(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_28(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_28(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_27(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_27(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_26(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_26(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_25(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_25(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_24(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_24(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_23(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_23(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_22(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_22(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_21(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_21(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_20(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_20(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_19(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_19(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_18(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_18(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_17(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_17(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_16(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_16(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_15(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_15(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_14(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_14(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_13(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_13(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_12(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_12(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_11(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_11(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_10(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_10(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_9(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_9(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_8(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_8(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_7(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_7(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_6(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_6(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_5(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_5(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_4(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_4(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_3(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_3(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal ,P2) JSONCONS_EXPAND(JSONCONS_REP_OF_2(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_2(Call, TC, JVal, TVal, P2, ...) JSONCONS_GENERATE(Call, TC, JVal, TVal, P2) JSONCONS_EXPAND(JSONCONS_REP_OF_1(Call, TC, JVal, TVal, __VA_ARGS__)) -#define JSONCONS_REP_OF_1(Call, TC, JVal, TVal, P2) JSONCONS_EXPAND(Call ## _LAST(TC, JVal, TVal, P2)) - -#define JSONCONS_IS(TC, JVal, TVal, Member) if (!(JVal).contains(JSONCONS_QUOTE(Member))) return false; -#define JSONCONS_IS_LAST(TC, JVal, TVal, Member) if (!(JVal).contains(JSONCONS_QUOTE(Member))) return false; - -#define JSONCONS_TO_JSON(TC, JVal, TVal, Member) (JVal).try_emplace(JSONCONS_QUOTE(Member), TVal.Member); -#define JSONCONS_TO_JSON_LAST(TC, JVal, TVal, Member) (JVal).try_emplace(JSONCONS_QUOTE(Member), TVal.Member); - -#define JSONCONS_AS(TC, JVal, TVal, Member) if ((JVal).contains(JSONCONS_QUOTE(Member))) {val.Member = (JVal).at(JSONCONS_QUOTE(Member)).template as();} -#define JSONCONS_AS_LAST(TC, JVal, TVal, Member) if ((JVal).contains(JSONCONS_QUOTE(Member))) {val.Member = (JVal).at(JSONCONS_QUOTE(Member)).template as();} - -#define JSONCONS_MEMBER_TRAITS_DECL(ValueType, ...) \ -namespace jsoncons \ -{ \ - template \ - struct json_type_traits \ - { \ - typedef ValueType value_type; \ - typedef typename Json::allocator_type allocator_type; \ - static bool is(const Json& j) noexcept \ - { \ - if (!j.is_object()) return false; \ - JSONCONS_REP_N(JSONCONS_IS, 0, j, void(), __VA_ARGS__)\ - return true; \ - } \ - static ValueType as(const Json& j) \ - { \ - ValueType val{}; \ - JSONCONS_REP_N(JSONCONS_AS, 0, j, val, __VA_ARGS__) \ - return val; \ - } \ - static Json to_json(const ValueType& val, allocator_type allocator=allocator_type()) \ - { \ - Json j(allocator); \ - JSONCONS_REP_N(JSONCONS_TO_JSON, 0, j, val, __VA_ARGS__) \ - return j; \ - } \ - }; \ -} \ - /**/ - -#define JSONCONS_IS2(TC, JVal, TVal, Member) if (!(JVal).contains(JSONCONS_QUOTE(Member))) return false; -#define JSONCONS_IS2_LAST(TC, JVal, TVal, Member) if (!(JVal).contains(JSONCONS_QUOTE(Member))) return false; - -#define JSONCONS_TO_JSON2(TC, JVal, TVal, Member) (JVal).try_emplace(JSONCONS_QUOTE(Member), TVal.Member() ); -#define JSONCONS_TO_JSON2_LAST(TC, JVal, TVal, Member) (JVal).try_emplace(JSONCONS_QUOTE(Member), TVal.Member() ); - -#define JSONCONS_AS2(TC, JVal, TVal, Member) ((JVal).at(JSONCONS_QUOTE(Member))).template asMember())>::type>(), -#define JSONCONS_AS2_LAST(TC, JVal, TVal, Member) ((JVal).at(JSONCONS_QUOTE(Member))).template asMember())>::type>() - -#define JSONCONS_GETTER_CTOR_TRAITS_DECL(ValueType, ...) \ -namespace jsoncons \ -{ \ - template \ - struct json_type_traits \ - { \ - typedef ValueType value_type; \ - typedef typename Json::allocator_type allocator_type; \ - static bool is(const Json& j) noexcept \ - { \ - if (!j.is_object()) return false; \ - JSONCONS_REP_N(JSONCONS_IS2, 0, j, void(), __VA_ARGS__)\ - return true; \ - } \ - static value_type as(const Json& j) \ - { \ - return value_type ( JSONCONS_REP_N(JSONCONS_AS2, 0, j, void(), __VA_ARGS__) ); \ - } \ - static Json to_json(const value_type& val, allocator_type allocator=allocator_type()) \ - { \ - Json j(allocator); \ - JSONCONS_REP_N(JSONCONS_TO_JSON2, 0, j, val, __VA_ARGS__) \ - return j; \ - } \ - }; \ -} \ - /**/ - -#define JSONCONS_TYPE_TRAITS_FRIEND \ - template \ - friend struct jsoncons::json_type_traits - -#if !defined(JSONCONS_NO_DEPRECATED) -#define JSONCONS_TYPE_TRAITS_DECL JSONCONS_MEMBER_TRAITS_DECL -#endif - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/parse_error_handler.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/parse_error_handler.hpp deleted file mode 100644 index 0f8b86e948..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/parse_error_handler.hpp +++ /dev/null @@ -1,74 +0,0 @@ -/// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_PARSE_ERROR_HANDLER_HPP -#define JSONCONS_PARSE_ERROR_HANDLER_HPP - -#include -#include -#include - -namespace jsoncons { - -class parse_error_handler -{ -public: - virtual ~parse_error_handler() - { - } - - bool error(std::error_code ec, - const ser_context& context) noexcept - { - return do_error(ec,context); - } - - void fatal_error(std::error_code ec, - const ser_context& context) noexcept - { - do_fatal_error(ec,context); - } - -private: - virtual bool do_error(std::error_code, - const ser_context& context) noexcept = 0; - - virtual void do_fatal_error(std::error_code, - const ser_context&) noexcept - { - } -}; - -class default_parse_error_handler : public parse_error_handler -{ -private: - bool do_error(std::error_code code, - const ser_context&) noexcept override - { - static const std::error_code illegal_comment = make_error_code(json_errc::illegal_comment); - - if (code == illegal_comment) - { - return true; // Recover, allow comments - } - else - { - return false; - } - } -}; - -class strict_parse_error_handler : public parse_error_handler -{ -private: - bool do_error(std::error_code, const ser_context&) noexcept override - { - return false; - } -}; - -} -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/pretty_print.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/pretty_print.hpp deleted file mode 100644 index 4ddf20eec2..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/pretty_print.hpp +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_PRETTY_PRINT_HPP -#define JSONCONS_PRETTY_PRINT_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace jsoncons { - -template -class json_printable -{ -public: - typedef typename Json::char_type char_type; - - json_printable(const Json& j, indenting line_indent) - : j_(&j), indenting_(line_indent) - { - } - - json_printable(const Json& j, - const basic_json_options& options, - indenting line_indent) - : j_(&j), options_(options), indenting_(line_indent) - { - } - - void dump(std::basic_ostream& os) const - { - j_->dump(os, options_, indenting_); - } - - friend std::basic_ostream& operator<<(std::basic_ostream& os, const json_printable& pr) - { - pr.dump(os); - return os; - } - - const Json *j_; - basic_json_options options_; - indenting indenting_; -private: - json_printable(); -}; - -template -json_printable print(const Json& j) -{ - return json_printable(j, indenting::no_indent); -} - -template -json_printable print(const Json& j, - const basic_json_options& options) -{ - return json_printable(j, options, indenting::no_indent); -} - -template -json_printable pretty_print(const Json& j) -{ - return json_printable(j, indenting::indent); -} - -template -json_printable pretty_print(const Json& j, - const basic_json_options& options) -{ - return json_printable(j, options, indenting::indent); -} - -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/result.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/result.hpp deleted file mode 100644 index d53341f2fe..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/result.hpp +++ /dev/null @@ -1,284 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_RESULT_HPP -#define JSONCONS_RESULT_HPP - -#include -#include -#include -#include -#include -#include -#include // std::addressof -#include // std::memcpy -#include -#include - -namespace jsoncons { - -// stream_result - -template -class stream_result -{ -public: - typedef CharT value_type; - typedef std::basic_ostream output_type; - -private: - static const size_t default_buffer_length = 16384; - - std::basic_ostream* os_; - std::vector buffer_; - CharT * begin_buffer_; - const CharT* end_buffer_; - CharT* p_; - - // Noncopyable - stream_result(const stream_result&) = delete; - stream_result& operator=(const stream_result&) = delete; - -public: - stream_result(stream_result&&) = default; - - stream_result(std::basic_ostream& os) - : os_(std::addressof(os)), buffer_(default_buffer_length), begin_buffer_(buffer_.data()), end_buffer_(begin_buffer_+buffer_.size()), p_(begin_buffer_) - { - } - stream_result(std::basic_ostream& os, size_t buflen) - : os_(std::addressof(os)), buffer_(buflen), begin_buffer_(buffer_.data()), end_buffer_(begin_buffer_+buffer_.size()), p_(begin_buffer_) - { - } - ~stream_result() - { - os_->write(begin_buffer_, buffer_length()); - os_->flush(); - } - - stream_result& operator=(stream_result&&) = default; - - void flush() - { - os_->write(begin_buffer_, buffer_length()); - os_->flush(); - p_ = buffer_.data(); - } - - void append(const CharT* s, size_t length) - { - size_t diff = end_buffer_ - p_; - if (diff >= length) - { - std::memcpy(p_, s, length*sizeof(CharT)); - p_ += length; - } - else - { - os_->write(begin_buffer_, buffer_length()); - os_->write(s,length); - p_ = begin_buffer_; - } - } - - void push_back(CharT ch) - { - if (p_ < end_buffer_) - { - *p_++ = ch; - } - else - { - os_->write(begin_buffer_, buffer_length()); - p_ = begin_buffer_; - push_back(ch); - } - } -private: - - size_t buffer_length() const - { - return p_ - begin_buffer_; - } -}; - -// binary_stream_result - -class binary_stream_result -{ -public: - typedef uint8_t value_type; - typedef std::basic_ostream output_type; -private: - static const size_t default_buffer_length = 16384; - - std::basic_ostream* os_; - std::vector buffer_; - uint8_t * begin_buffer_; - const uint8_t* end_buffer_; - uint8_t* p_; - - // Noncopyable - binary_stream_result(const binary_stream_result&) = delete; - binary_stream_result& operator=(const binary_stream_result&) = delete; - -public: - binary_stream_result(binary_stream_result&&) = default; - - binary_stream_result(std::basic_ostream& os) - : os_(std::addressof(os)), - buffer_(default_buffer_length), - begin_buffer_(buffer_.data()), - end_buffer_(begin_buffer_+buffer_.size()), - p_(begin_buffer_) - { - } - binary_stream_result(std::basic_ostream& os, size_t buflen) - : os_(std::addressof(os)), - buffer_(buflen), - begin_buffer_(buffer_.data()), - end_buffer_(begin_buffer_+buffer_.size()), - p_(begin_buffer_) - { - } - ~binary_stream_result() - { - os_->write((char*)begin_buffer_, buffer_length()); - os_->flush(); - } - - binary_stream_result& operator=(binary_stream_result&&) = default; - - void flush() - { - os_->write((char*)begin_buffer_, buffer_length()); - p_ = buffer_.data(); - } - - void append(const uint8_t* s, size_t length) - { - size_t diff = end_buffer_ - p_; - if (diff >= length) - { - std::memcpy(p_, s, length*sizeof(uint8_t)); - p_ += length; - } - else - { - os_->write((char*)begin_buffer_, buffer_length()); - os_->write((const char*)s,length); - p_ = begin_buffer_; - } - } - - void push_back(uint8_t ch) - { - if (p_ < end_buffer_) - { - *p_++ = ch; - } - else - { - os_->write((char*)begin_buffer_, buffer_length()); - p_ = begin_buffer_; - push_back(ch); - } - } -private: - - size_t buffer_length() const - { - return p_ - begin_buffer_; - } -}; - -// string_result - -template -class string_result -{ -public: - typedef typename StringT::value_type value_type; - typedef StringT output_type; -private: - output_type* s_; - - // Noncopyable - string_result(const string_result&) = delete; - string_result& operator=(const string_result&) = delete; -public: - string_result(string_result&& val) - : s_(nullptr) - { - std::swap(s_,val.s_); - } - - string_result(output_type& s) - : s_(std::addressof(s)) - { - } - - string_result& operator=(string_result&& val) - { - std::swap(s_, val.s_); - } - - void flush() - { - } - - void append(const value_type* s, size_t length) - { - s_->insert(s_->end(), s, s+length); - } - - void push_back(value_type ch) - { - s_->push_back(ch); - } -}; - -// bytes_result - -class bytes_result -{ -public: - typedef uint8_t value_type; - typedef std::vector output_type; -private: - output_type& s_; - - // Noncopyable - bytes_result(const bytes_result&) = delete; - bytes_result& operator=(const bytes_result&) = delete; -public: - bytes_result(bytes_result&&) = default; - - bytes_result(output_type& s) - : s_(s) - { - } - - bytes_result& operator=(bytes_result&&) = default; - - void flush() - { - } - - void append(const uint8_t* s, size_t length) - { - s_.insert(s_.end(), s, s+length); - } - - void push_back(uint8_t ch) - { - s_.push_back(ch); - } -}; - -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/ser_context.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/ser_context.hpp deleted file mode 100644 index 12397d0134..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/ser_context.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/// Copyright 2013-2018 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_SER_CONTEXT_HPP -#define JSONCONS_SER_CONTEXT_HPP - -namespace jsoncons { - -class ser_context -{ -public: - virtual ~ser_context() = default; - - virtual size_t line() const = 0; - - virtual size_t column() const = 0; - -#if !defined(JSONCONS_NO_DEPRECATED) - size_t line_number() const - { - return line(); - } - - size_t column_number() const - { - return column(); - } -#endif -}; - -class null_ser_context : public ser_context -{ -private: - size_t line() const override { return 0; } - - size_t column() const override { return 0; } -}; - -#if !defined(JSONCONS_NO_DEPRECATED) -typedef ser_context parsing_context; -typedef ser_context serializing_context; -typedef null_ser_context null_parsing_context; -typedef null_ser_context null_serializing_context; -#endif - -} -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/source.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/source.hpp deleted file mode 100644 index f1feec5706..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/source.hpp +++ /dev/null @@ -1,746 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_SOURCE_HPP -#define JSONCONS_SOURCE_HPP - -#include -#include -#include -#include -#include // std::addressof -#include // std::memcpy -#include -#include // std::enable_if -#include -#include // jsoncons::byte_traits - -namespace jsoncons { - -template -class basic_null_istream : public std::basic_istream -{ - class null_buffer : public std::basic_streambuf - { - public: - using typename std::basic_streambuf::int_type; - using typename std::basic_streambuf::traits_type; - - null_buffer() = default; - null_buffer(const null_buffer&) = default; - null_buffer& operator=(const null_buffer&) = default; - - int_type overflow( int_type ch = traits_type::eof() ) override - { - return ch; - } - } nb_; -public: - basic_null_istream() - : std::basic_istream(&nb_) - { - } - basic_null_istream(basic_null_istream&&) = default; - - basic_null_istream& operator=(basic_null_istream&&) = default; -}; - -// text sources - -template -class stream_source -{ -public: - typedef CharT value_type; - typedef std::char_traits traits_type; -private: - basic_null_istream null_is_; - std::basic_istream* is_; - std::basic_streambuf* sbuf_; - size_t position_; - - // Noncopyable - stream_source(const stream_source&) = delete; - stream_source& operator=(const stream_source&) = delete; -public: - stream_source() - : is_(&null_is_), sbuf_(null_is_.rdbuf()), position_(0) - { - } - - stream_source(std::basic_istream& is) - : is_(std::addressof(is)), sbuf_(is.rdbuf()), position_(0) - { - } - - stream_source(stream_source&& other) - { - std::swap(is_,other.is_); - std::swap(sbuf_,other.sbuf_); - std::swap(position_,other.position_); - } - - ~stream_source() - { - } - - stream_source& operator=(stream_source&& other) - { - std::swap(is_,other.is_); - std::swap(sbuf_,other.sbuf_); - std::swap(position_,other.position_); - return *this; - } - - bool eof() const - { - return is_->eof(); - } - - bool is_error() const - { - return is_->bad(); - } - - size_t position() const - { - return position_; - } - - size_t get(value_type& c) - { - try - { - int val = sbuf_->sbumpc(); - if (!(val == traits_type::eof())) - { - c = (value_type)val; - ++position_; - return 1; - } - else - { - is_->clear(is_->rdstate() | std::ios::eofbit); - return 0; - } - } - catch (const std::exception&) - { - is_->clear(is_->rdstate() | std::ios::badbit | std::ios::eofbit); - return 0; - } - } - - int get() - { - try - { - int c = sbuf_->sbumpc(); - if (c == traits_type::eof()) - { - is_->clear(is_->rdstate() | std::ios::eofbit); - } - else - { - ++position_; - } - return c; - } - catch (const std::exception&) - { - is_->clear(is_->rdstate() | std::ios::badbit | std::ios::eofbit); - return traits_type::eof(); - } - } - - void ignore(size_t count) - { - try - { - for (size_t i = 0; i < count; ++i) - { - int c = sbuf_->sbumpc(); - if (c == traits_type::eof()) - { - is_->clear(is_->rdstate() | std::ios::eofbit); - return; - } - else - { - ++position_; - } - } - } - catch (const std::exception&) - { - is_->clear(is_->rdstate() | std::ios::badbit | std::ios::eofbit); - } - } - - int peek() - { - try - { - int c = sbuf_->sgetc(); - if (c == traits_type::eof()) - { - is_->clear(is_->rdstate() | std::ios::eofbit); - } - return c; - } - catch (const std::exception&) - { - is_->clear(is_->rdstate() | std::ios::badbit); - return traits_type::eof(); - } - } - - size_t read(value_type* p, size_t length) - { - try - { - std::streamsize count = sbuf_->sgetn(p, length); // never negative - if (static_cast(count) < length) - { - is_->clear(is_->rdstate() | std::ios::eofbit); - } - position_ += length; - return static_cast(count); - } - catch (const std::exception&) - { - is_->clear(is_->rdstate() | std::ios::badbit | std::ios::eofbit); - return 0; - } - } - - template - typename std::enable_if::value,size_t>::type - read(OutputIt p, size_t length) - { - size_t count = 0; - try - { - for (count = 0; count < length; ++count) - { - int c = sbuf_->sbumpc(); - if (c == traits_type::eof()) - { - is_->clear(is_->rdstate() | std::ios::eofbit); - return count; - } - else - { - ++position_; - } - *p++ = (value_type)c; - } - return count; - } - catch (const std::exception&) - { - is_->clear(is_->rdstate() | std::ios::badbit | std::ios::eofbit); - return count; - } - } -}; - -// string_source - -template -struct is_string_sourceable : std::false_type {}; - -template -struct is_string_sourceable::value>::type> : std::true_type {}; - -template -class string_source -{ -public: - typedef CharT value_type; - typedef std::char_traits traits_type; - typedef jsoncons::basic_string_view string_view_type; -private: - const value_type* data_; - const value_type* input_ptr_; - const value_type* input_end_; - bool eof_; - - // Noncopyable - string_source(const string_source&) = delete; - string_source& operator=(const string_source&) = delete; -public: - string_source() - : data_(nullptr), input_ptr_(nullptr), input_end_(nullptr), eof_(true) - { - } - - template - string_source(const Source& s, - typename std::enable_if::type>::value>::type* = 0) - : data_(s.data()), input_ptr_(s.data()), input_end_(s.data()+s.size()), eof_(s.size() == 0) - { - } - - string_source(const value_type* data, size_t size) - : data_(data), input_ptr_(data), input_end_(data+size), eof_(size == 0) - { - } - - string_source(string_source&& val) - : data_(nullptr), input_ptr_(nullptr), input_end_(nullptr), eof_(true) - { - std::swap(data_,val.data_); - std::swap(input_ptr_,val.input_ptr_); - std::swap(input_end_,val.input_end_); - std::swap(eof_,val.eof_); - } - - string_source& operator=(string_source&& val) - { - std::swap(data_,val.data_); - std::swap(input_ptr_,val.input_ptr_); - std::swap(input_end_,val.input_end_); - std::swap(eof_,val.eof_); - return *this; - } - - bool eof() const - { - return eof_; - } - - bool is_error() const - { - return false; - } - - size_t position() const - { - return (input_ptr_ - data_)/sizeof(value_type) + 1; - } - - size_t get(value_type& c) - { - if (input_ptr_ < input_end_) - { - c = *input_ptr_++; - return 1; - } - else - { - eof_ = true; - input_ptr_ = input_end_; - return 0; - } - } - - int get() - { - if (input_ptr_ < input_end_) - { - return *input_ptr_++; - } - else - { - eof_ = true; - input_ptr_ = input_end_; - return traits_type::eof(); - } - } - - void ignore(size_t count) - { - size_t len; - if ((size_t)(input_end_ - input_ptr_) < count) - { - len = input_end_ - input_ptr_; - eof_ = true; - } - else - { - len = count; - } - input_ptr_ += len; - } - - int peek() - { - return input_ptr_ < input_end_ ? *input_ptr_ : traits_type::eof(); - } - - size_t read(value_type* p, size_t length) - { - size_t len; - if ((size_t)(input_end_ - input_ptr_) < length) - { - len = input_end_ - input_ptr_; - eof_ = true; - } - else - { - len = length; - } - std::memcpy(p, input_ptr_, len*sizeof(value_type)); - input_ptr_ += len; - return len; - } - - template - typename std::enable_if::value,size_t>::type - read(OutputIt d_first, size_t count) - { - size_t len; - if ((size_t)(input_end_ - input_ptr_) < count) - { - len = input_end_ - input_ptr_; - eof_ = true; - } - else - { - len = count; - } - for (size_t i = 0; i < len; ++i) - { - *d_first++ = *input_ptr_++; - } - return len; - } -}; - -// binary sources - -class binary_stream_source -{ -public: - typedef uint8_t value_type; - typedef byte_traits traits_type; -private: - basic_null_istream null_is_; - std::istream* is_; - std::streambuf* sbuf_; - size_t position_; - - // Noncopyable - binary_stream_source(const binary_stream_source&) = delete; - binary_stream_source& operator=(const binary_stream_source&) = delete; -public: - binary_stream_source() - : is_(&null_is_), sbuf_(null_is_.rdbuf()), position_(0) - { - } - - binary_stream_source(std::istream& is) - : is_(std::addressof(is)), sbuf_(is.rdbuf()), position_(0) - { - } - - binary_stream_source(binary_stream_source&& other) - { - std::swap(is_,other.is_); - std::swap(sbuf_,other.sbuf_); - std::swap(position_,other.position_); - } - - ~binary_stream_source() - { - } - - binary_stream_source& operator=(binary_stream_source&& other) - { - std::swap(is_,other.is_); - std::swap(sbuf_,other.sbuf_); - std::swap(position_,other.position_); - return *this; - } - - bool eof() const - { - return is_->eof(); - } - - bool is_error() const - { - return is_->bad(); - } - - size_t position() const - { - return position_; - } - - size_t get(value_type& c) - { - try - { - int val = sbuf_->sbumpc(); - if (!(val == traits_type::eof())) - { - c = (value_type)val; - ++position_; - return 1; - } - else - { - is_->clear(is_->rdstate() | std::ios::eofbit); - return 0; - } - } - catch (const std::exception&) - { - is_->clear(is_->rdstate() | std::ios::badbit | std::ios::eofbit); - return 0; - } - } - - int get() - { - try - { - int c = sbuf_->sbumpc(); - if (c == traits_type::eof()) - { - is_->clear(is_->rdstate() | std::ios::eofbit); - } - else - { - ++position_; - } - return c; - } - catch (const std::exception&) - { - is_->clear(is_->rdstate() | std::ios::badbit | std::ios::eofbit); - return traits_type::eof(); - } - } - - void ignore(size_t count) - { - try - { - for (size_t i = 0; i < count; ++i) - { - int c = sbuf_->sbumpc(); - if (c == traits_type::eof()) - { - is_->clear(is_->rdstate() | std::ios::eofbit); - return; - } - else - { - ++position_; - } - } - } - catch (const std::exception&) - { - is_->clear(is_->rdstate() | std::ios::badbit | std::ios::eofbit); - } - } - - int peek() - { - try - { - int c = sbuf_->sgetc(); - if (c == traits_type::eof()) - { - is_->clear(is_->rdstate() | std::ios::eofbit); - } - return c; - } - catch (const std::exception&) - { - is_->clear(is_->rdstate() | std::ios::badbit); - return traits_type::eof(); - } - } - - template - size_t read(OutputIt p, size_t length) - { - size_t count = 0; - try - { - for (count = 0; count < length; ++count) - { - int c = sbuf_->sbumpc(); - if (c == traits_type::eof()) - { - is_->clear(is_->rdstate() | std::ios::eofbit); - return count; - } - else - { - ++position_; - } - *p++ = (value_type)c; - } - return count; - } - catch (const std::exception&) - { - is_->clear(is_->rdstate() | std::ios::badbit | std::ios::eofbit); - return count; - } - } -}; - -template -struct is_bytes_sourceable : std::false_type {}; - -template -struct is_bytes_sourceable::value>::type> : std::true_type {}; - -class bytes_source -{ -public: - typedef uint8_t value_type; - typedef byte_traits traits_type; -private: - const value_type* data_; - const value_type* input_ptr_; - const value_type* input_end_; - bool eof_; - - // Noncopyable - bytes_source(const bytes_source&) = delete; - bytes_source& operator=(const bytes_source&) = delete; -public: - bytes_source() - : data_(nullptr), input_ptr_(nullptr), input_end_(nullptr), eof_(true) - { - } - - template - bytes_source(const Source& s, - typename std::enable_if::type>::value>::type* = 0) - : data_(s.data()), - input_ptr_(s.data()), - input_end_(s.data()+s.size()), - eof_(s.size() == 0) - { - } - - bytes_source(const value_type* data, size_t size) - : data_(data), - input_ptr_(data), - input_end_(data+size), - eof_(size == 0) - { - } - - bytes_source(bytes_source&&) = default; - - bytes_source& operator=(bytes_source&&) = default; - - bool eof() const - { - return eof_; - } - - bool is_error() const - { - return false; - } - - size_t position() const - { - return input_ptr_ - data_ + 1; - } - - size_t get(value_type& c) - { - if (input_ptr_ < input_end_) - { - c = *input_ptr_++; - return 1; - } - else - { - eof_ = true; - input_ptr_ = input_end_; - return 0; - } - } - - int get() - { - if (input_ptr_ < input_end_) - { - return *input_ptr_++; - } - else - { - eof_ = true; - input_ptr_ = input_end_; - return traits_type::eof(); - } - } - - void ignore(size_t count) - { - size_t len; - if ((size_t)(input_end_ - input_ptr_) < count) - { - len = input_end_ - input_ptr_; - eof_ = true; - } - else - { - len = count; - } - input_ptr_ += len; - } - - int peek() - { - return input_ptr_ < input_end_ ? *input_ptr_ : traits_type::eof(); - } - - size_t read(value_type* p, size_t length) - { - size_t len; - if ((size_t)(input_end_ - input_ptr_) < length) - { - len = input_end_ - input_ptr_; - eof_ = true; - } - else - { - len = length; - } - std::memcpy(p, input_ptr_, len); - input_ptr_ += len; - return len; - } - - template - typename std::enable_if::value,size_t>::type - read(OutputIt d_first, size_t count) - { - size_t len; - if ((size_t)(input_end_ - input_ptr_) < count) - { - len = input_end_ - input_ptr_; - eof_ = true; - } - else - { - len = count; - } - for (size_t i = 0; i < len; ++i) - { - *d_first++ = *input_ptr_++; - } - return len; - } -}; - -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/staj_iterator.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/staj_iterator.hpp deleted file mode 100644 index 5937972952..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/staj_iterator.hpp +++ /dev/null @@ -1,354 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_STAJ_ITERATOR_HPP -#define JSONCONS_STAJ_ITERATOR_HPP - -#include -#include -#include -#include -#include -#include // std::input_iterator_tag -#include -#include -#include - -namespace jsoncons { - -template> -class basic_staj_array_iterator -{ - typedef CharT char_type; - - basic_staj_reader* reader_; - T value_; -public: - typedef T value_type; - typedef std::ptrdiff_t difference_type; - typedef T* pointer; - typedef T& reference; - typedef std::input_iterator_tag iterator_category; - - basic_staj_array_iterator() noexcept - : reader_(nullptr) - { - } - - basic_staj_array_iterator(basic_staj_reader& reader) - : reader_(std::addressof(reader)) - { - if (reader_->current().event_type() == staj_event_type::begin_array) - { - next(); - } - else - { - reader_ = nullptr; - } - } - - basic_staj_array_iterator(basic_staj_reader& reader, - std::error_code& ec) - : reader_(std::addressof(reader)) - { - if (reader_->current().event_type() == staj_event_type::begin_array) - { - next(ec); - if (ec) - { - reader_ = nullptr; - } - } - else - { - reader_ = nullptr; - } - } - - const T& operator*() const - { - return value_; - } - - const T* operator->() const - { - return &value_; - } - - basic_staj_array_iterator& operator++() - { - next(); - return *this; - } - - basic_staj_array_iterator& increment(std::error_code& ec) - { - next(ec); - if (ec) - { - reader_ = nullptr; - } - return *this; - } - - basic_staj_array_iterator operator++(int) // postfix increment - { - basic_staj_array_iterator temp(*this); - next(); - return temp; - } - - friend bool operator==(const basic_staj_array_iterator& a, const basic_staj_array_iterator& b) - { - return (!a.reader_ && !b.reader_) - || (!a.reader_ && b.done()) - || (!b.reader_ && a.done()); - } - - friend bool operator!=(const basic_staj_array_iterator& a, const basic_staj_array_iterator& b) - { - return !(a == b); - } - -private: - - bool done() const - { - return reader_->done() || reader_->current().event_type() == staj_event_type::end_array; - } - - void next(); - - void next(std::error_code& ec); -}; - -template -basic_staj_array_iterator begin(basic_staj_array_iterator iter) noexcept -{ - return iter; -} - -template -basic_staj_array_iterator end(const basic_staj_array_iterator&) noexcept -{ - return basic_staj_array_iterator(); -} - -template -class basic_staj_object_iterator -{ -public: - typedef CharT char_type; - typedef std::basic_string key_type; - typedef std::pair value_type; - typedef std::ptrdiff_t difference_type; - typedef value_type* pointer; - typedef value_type& reference; - typedef std::input_iterator_tag iterator_category; - -private: - basic_staj_reader* reader_; - value_type kv_; -public: - - basic_staj_object_iterator() noexcept - : reader_(nullptr) - { - } - - basic_staj_object_iterator(basic_staj_reader& reader) - : reader_(std::addressof(reader)) - { - if (reader_->current().event_type() == staj_event_type::begin_object) - { - next(); - } - else - { - reader_ = nullptr; - } - } - - basic_staj_object_iterator(basic_staj_reader& reader, - std::error_code& ec) - : reader_(std::addressof(reader)) - { - if (reader_->current().event_type() == staj_event_type::begin_object) - { - next(ec); - if (ec) - { - reader_ = nullptr; - } - } - else - { - reader_ = nullptr; - } - } - - const value_type& operator*() const - { - return kv_; - } - - const value_type* operator->() const - { - return &kv_; - } - - basic_staj_object_iterator& operator++() - { - next(); - return *this; - } - - basic_staj_object_iterator& increment(std::error_code& ec) - { - next(ec); - if (ec) - { - reader_ = nullptr; - } - return *this; - } - - basic_staj_object_iterator operator++(int) // postfix increment - { - basic_staj_object_iterator temp(*this); - next(); - return temp; - } - - friend bool operator==(const basic_staj_object_iterator& a, const basic_staj_object_iterator& b) - { - return (!a.reader_ && !b.reader_) - || (!a.reader_ && b.done()) - || (!b.reader_ && a.done()); - } - - friend bool operator!=(const basic_staj_object_iterator& a, const basic_staj_object_iterator& b) - { - return !(a == b); - } - -private: - - bool done() const - { - return reader_->done() || reader_->current().event_type() == staj_event_type::end_object; - } - - void next(); - - void next(std::error_code& ec); -}; - -template -basic_staj_object_iterator begin(basic_staj_object_iterator iter) noexcept -{ - return iter; -} - -template -basic_staj_object_iterator end(const basic_staj_object_iterator&) noexcept -{ - return basic_staj_object_iterator(); -} - -template -using staj_array_iterator = basic_staj_array_iterator; - -template -using wstaj_array_iterator = basic_staj_array_iterator; - -template -using staj_object_iterator = basic_staj_object_iterator; - -template -using wstaj_object_iterator = basic_staj_object_iterator; - -} - -#include - -namespace jsoncons { - -template -void basic_staj_array_iterator::next() -{ - if (!done()) - { - reader_->next(); - if (!done()) - { - read_from(Json(), *reader_, value_); - } - } -} - -template -void basic_staj_array_iterator::next(std::error_code& ec) -{ - if (!done()) - { - reader_->next(ec); - if (ec) - { - return; - } - if (!done()) - { - read_from(Json(), *reader_, value_, ec); - } - } -} - -template -void basic_staj_object_iterator::next() -{ - reader_->next(); - if (!done()) - { - JSONCONS_ASSERT(reader_->current().event_type() == staj_event_type::name); - kv_.first =reader_->current(). template as(); - reader_->next(); - if (!done()) - { - read_from(Json(), *reader_, kv_.second); - } - } -} - -template -void basic_staj_object_iterator::next(std::error_code& ec) -{ - reader_->next(ec); - if (ec) - { - return; - } - if (!done()) - { - JSONCONS_ASSERT(reader_->current().event_type() == staj_event_type::name); - kv_.first =reader_->current(). template as(); - reader_->next(ec); - if (ec) - { - return; - } - if (!done()) - { - read_from(Json(), *reader_, kv_.second, ec); - } - } -} - -} - -#endif - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/staj_reader.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/staj_reader.hpp deleted file mode 100644 index 269437361f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/staj_reader.hpp +++ /dev/null @@ -1,434 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_STAJ_READER_HPP -#define JSONCONS_STAJ_READER_HPP - -#include // std::allocator -#include -#include -#include -#include -#include // std::enable_if -#include // std::array -#include -#include -#include -#include -#include -#include -#include - -namespace jsoncons { - -enum class staj_event_type -{ - begin_array, - end_array, - begin_object, - end_object, - name, - string_value, - byte_string_value, - null_value, - bool_value, - int64_value, - uint64_value, - double_value -}; - -JSONCONS_STRING_LITERAL(null,'n','u','l','l') -JSONCONS_STRING_LITERAL(true,'t','r','u','e') -JSONCONS_STRING_LITERAL(false,'f','a','l','s','e') - -template -class basic_staj_event -{ - staj_event_type event_type_; - semantic_tag semantic_tag_; - union - { - bool bool_value_; - int64_t int64_value_; - uint64_t uint64_value_; - double double_value_; - const CharT* string_data_; - const uint8_t* byte_string_data_; - } value_; - size_t length_; -public: - basic_staj_event(staj_event_type event_type, semantic_tag semantic_tag = semantic_tag::none) - : event_type_(event_type), semantic_tag_(semantic_tag), length_(0) - { - } - - basic_staj_event(null_type) - : event_type_(staj_event_type::null_value), semantic_tag_(semantic_tag::none), length_(0) - { - } - - basic_staj_event(bool value) - : event_type_(staj_event_type::bool_value), semantic_tag_(semantic_tag::none), length_(0) - { - value_.bool_value_ = value; - } - - basic_staj_event(int64_t value, semantic_tag semantic_tag) - : event_type_(staj_event_type::int64_value), semantic_tag_(semantic_tag), length_(0) - { - value_.int64_value_ = value; - } - - basic_staj_event(uint64_t value, semantic_tag semantic_tag) - : event_type_(staj_event_type::uint64_value), semantic_tag_(semantic_tag), length_(0) - { - value_.uint64_value_ = value; - } - - basic_staj_event(double value, semantic_tag semantic_tag) - : event_type_(staj_event_type::double_value), semantic_tag_(semantic_tag), length_(0) - { - value_.double_value_ = value; - } - - basic_staj_event(const CharT* data, size_t length, - staj_event_type event_type, - semantic_tag semantic_tag = semantic_tag::none) - : event_type_(event_type), semantic_tag_(semantic_tag), length_(length) - { - value_.string_data_ = data; - } - - template - typename std::enable_if::value && std::is_same::value, T>::type - as() const - { - T s; - switch (event_type_) - { - case staj_event_type::name: - case staj_event_type::string_value: - s = T(value_.string_data_, length_); - break; - case staj_event_type::int64_value: - { - jsoncons::string_result result(s); - jsoncons::detail::print_integer(value_.int64_value_, result); - break; - } - case staj_event_type::uint64_value: - { - jsoncons::string_result result(s); - jsoncons::detail::print_uinteger(value_.uint64_value_, result); - break; - } - case staj_event_type::double_value: - { - jsoncons::string_result result(s); - jsoncons::detail::print_double f{ floating_point_options() }; - f(value_.double_value_, result); - break; - } - case staj_event_type::bool_value: - { - jsoncons::string_result result(s); - if (value_.bool_value_) - { - result.append(true_literal().data(),true_literal().size()); - } - else - { - result.append(false_literal().data(),false_literal().size()); - } - break; - } - case staj_event_type::null_value: - { - jsoncons::string_result result(s); - result.append(null_literal().data(),null_literal().size()); - break; - } - default: - JSONCONS_THROW(json_runtime_error("Not a string")); - } - return s; - } - - template - typename std::enable_if::value && std::is_same::value, T>::type - as() const - { - T s; - switch (event_type_) - { - case staj_event_type::name: - case staj_event_type::string_value: - s = T(value_.string_data_, length_); - break; - default: - JSONCONS_THROW(json_runtime_error("Not a string")); - } - return s; - } - - template - typename std::enable_if::value, T>::type - as() const - { - return static_cast(as_int64()); - } - - template - typename std::enable_if::value, T>::type - as() const - { - return static_cast(as_uint64()); - } - - template - typename std::enable_if::value, T>::type - as() const - { - return static_cast(as_double()); - } - - template> - typename std::enable_if>::value, T>::type - as() const - { - return as_bignum(); - } - - template - typename std::enable_if::value, T>::type - as() const - { - return as_bool(); - } - - staj_event_type event_type() const noexcept { return event_type_; } - - semantic_tag get_semantic_tag() const noexcept { return semantic_tag_; } -private: - - int64_t as_int64() const - { - int64_t value = 0; - switch (event_type_) - { - case staj_event_type::name: - case staj_event_type::string_value: - { - auto result = jsoncons::detail::to_integer(value_.string_data_, length_); - if (result.ec != jsoncons::detail::to_integer_errc()) - { - JSONCONS_THROW(json_runtime_error(make_error_code(result.ec).message())); - } - value = result.value; - break; - } - case staj_event_type::double_value: - value = static_cast(value_.double_value_); - break; - case staj_event_type::int64_value: - value = value_.int64_value_; - break; - case staj_event_type::uint64_value: - value = static_cast(value_.uint64_value_); - break; - case staj_event_type::bool_value: - value = value_.bool_value_ ? 1 : 0; - break; - default: - JSONCONS_THROW(json_runtime_error("Not an integer")); - } - return value; - } - - uint64_t as_uint64() const - { - uint64_t value = 0; - switch (event_type_) - { - case staj_event_type::name: - case staj_event_type::string_value: - { - auto result = jsoncons::detail::to_integer(value_.string_data_, length_); - if (result.ec != jsoncons::detail::to_integer_errc()) - { - JSONCONS_THROW(json_runtime_error(make_error_code(result.ec).message())); - } - value = result.value; - break; - } - case staj_event_type::double_value: - value = static_cast(value_.double_value_); - break; - case staj_event_type::int64_value: - value = static_cast(value_.int64_value_); - break; - case staj_event_type::uint64_value: - value = value_.uint64_value_; - break; - case staj_event_type::bool_value: - value = value_.bool_value_ ? 1 : 0; - break; - default: - JSONCONS_THROW(json_runtime_error("Not an unsigned integer")); - } - return value; - } - - double as_double() const - { - switch (event_type_) - { - case staj_event_type::name: - case staj_event_type::string_value: - { - std::string target; - auto result = unicons::convert( - value_.string_data_, value_.string_data_ + length_, std::back_inserter(target), unicons::conv_flags::strict); - if (result.ec != unicons::conv_errc()) - { - JSONCONS_THROW(json_runtime_error("Not a double")); - } - jsoncons::detail::string_to_double f; - return f(target.data(), target.length()); - } - case staj_event_type::double_value: - return value_.double_value_; - case staj_event_type::int64_value: - return static_cast(value_.int64_value_); - case staj_event_type::uint64_value: - return static_cast(value_.uint64_value_); - default: - JSONCONS_THROW(json_runtime_error("Not a double")); - } - } - - bool as_bool() const - { - switch (event_type_) - { - case staj_event_type::bool_value: - return value_.bool_value_; - case staj_event_type::double_value: - return value_.double_value_ != 0.0; - case staj_event_type::int64_value: - return value_.int64_value_ != 0; - case staj_event_type::uint64_value: - return value_.uint64_value_ != 0; - default: - JSONCONS_THROW(json_runtime_error("Not a bool")); - } - } - - template > - basic_bignum as_bignum() const - { - switch (event_type_) - { - case staj_event_type::string_value: - if (!jsoncons::detail::is_integer(value_.string_data_, length_)) - { - JSONCONS_THROW(json_runtime_error("Not a bignum")); - } - return basic_bignum(value_.string_data_, length_); - case staj_event_type::double_value: - return basic_bignum(value_.double_value_); - case staj_event_type::int64_value: - return basic_bignum(value_.int64_value_); - case staj_event_type::uint64_value: - return basic_bignum(value_.uint64_value_); - case staj_event_type::bool_value: - return basic_bignum(value_.bool_value_ ? 1 : 0); - default: - JSONCONS_THROW(json_runtime_error("Not a bignum")); - } - } - -}; - -template -class basic_staj_reader -{ -public: - virtual ~basic_staj_reader() = default; - - virtual bool done() const = 0; - - virtual const basic_staj_event& current() const = 0; - - virtual void accept(basic_json_content_handler& handler) = 0; - - virtual void accept(basic_json_content_handler& handler, - std::error_code& ec) = 0; - - virtual void next() = 0; - - virtual void next(std::error_code& ec) = 0; - - virtual const ser_context& context() const = 0; -}; - -template -class basic_staj_filter -{ -public: - - virtual ~basic_staj_filter() = default; - - virtual bool accept(const basic_staj_event& event, const ser_context& context) = 0; -}; - -template -class default_basic_staj_filter : public basic_staj_filter -{ -public: - bool accept(const basic_staj_event&, const ser_context&) override - { - return true; - } -}; - -typedef basic_staj_event staj_event; -typedef basic_staj_event wstaj_event; - -typedef basic_staj_reader staj_reader; -typedef basic_staj_reader wstaj_reader; - -typedef basic_staj_filter staj_filter; -typedef basic_staj_filter wstaj_filter; - -#if !defined(JSONCONS_NO_DEPRECATED) - -typedef staj_event_type stream_event_type; - -template -using basic_stream_event = basic_staj_event; - -template -using basic_stream_reader = basic_staj_reader; - -template -using basic_stream_filter = basic_staj_filter; - -typedef basic_staj_event stream_event; -typedef basic_staj_event wstream_event; - -typedef basic_staj_reader stream_reader; -typedef basic_staj_reader wstream_reader; - -typedef basic_staj_filter stream_filter; -typedef basic_staj_filter wstream_filter; - -#endif - -} - -#endif - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/unicode_traits.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/unicode_traits.hpp deleted file mode 100644 index 56bc9e7aa6..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons/unicode_traits.hpp +++ /dev/null @@ -1,1492 +0,0 @@ -// Copyright 2016 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/unicode_traits for latest version - -/* - * Includes code derived from Unicode, Inc decomposition code in ConvertUTF.h and ConvertUTF.c - * http://www.unicode.org/ - * - * "Unicode, Inc. hereby grants the right to freely use the information - * supplied in this file in the creation of products supporting the - * Unicode Standard." -*/ - -#ifndef JSONCONS_UNICONS_UNICODE_TRAITS_HPP -#define JSONCONS_UNICONS_UNICODE_TRAITS_HPP - -#if defined(__clang__) -# define UNICONS_FALLTHROUGH [[clang::fallthrough]] -#elif defined(__GNUC__) && ((__GNUC__ >= 7)) -# define UNICONS_FALLTHROUGH __attribute__((fallthrough)) -#elif defined (__GNUC__) -# define UNICONS_FALLTHROUGH // FALLTHRU -#else -# define UNICONS_FALLTHROUGH -#endif - -#if defined (__clang__) -#if defined(_GLIBCXX_USE_NOEXCEPT) -#define UNICONS_NOEXCEPT _GLIBCXX_USE_NOEXCEPT -#else -#define UNICONS_NOEXCEPT noexcept -#endif -#elif defined(__GNUC__) -#define UNICONS_NOEXCEPT _GLIBCXX_USE_NOEXCEPT -#elif defined(_MSC_VER) -#if _MSC_VER >= 1900 -#define UNICONS_NOEXCEPT noexcept -#else -#define UNICONS_NOEXCEPT -#endif -#else -#define UNICONS_NOEXCEPT -#endif - -#include -#include -#include -#include - -namespace jsoncons { namespace unicons { - -/* - * Magic values subtracted from a buffer value during UTF8 conversion. - * This table contains as many values as there might be trailing bytes - * in a UTF-8 sequence. Source: ConvertUTF.c - */ -const uint32_t offsets_from_utf8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, - 0x03C82080UL, 0xFA082080UL, 0x82082080UL }; - -/* - * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed - * into the first byte, depending on how many bytes follow. There are - * as many entries in this table as there are UTF-8 sequence types. - * (I.e., one byte sequence, two byte... etc.). Remember that sequencs - * for *legal* UTF-8 will be 4 or fewer bytes total. Source: ConvertUTF.c - */ -const uint8_t first_byte_mark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; - -/* - * Index into the table below with the first byte of a UTF-8 sequence to - * get the number of trailing bytes that are supposed to follow it. - * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is - * left as-is for anyone who may want to do such conversion, which was - * allowed in earlier algorithms. Source: ConvertUTF.c - */ -const uint8_t trailing_bytes_for_utf8[256] = { - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 -}; - -// Some fundamental constants. Source: ConvertUTF.h -const uint32_t replacement_char = 0x0000FFFD; -const uint32_t max_bmp = 0x0000FFFF; -const uint32_t max_utf16 = 0x0010FFFF; -const uint32_t max_utf32 = 0x7FFFFFFF; -const uint32_t max_legal_utf32 = 0x0010FFFF; - -const int half_shift = 10; // used for shifting by 10 bits -const uint32_t half_base = 0x0010000UL; -const uint32_t half_mask = 0x3FFUL; - -const uint16_t sur_high_start = 0xD800; -const uint16_t sur_high_end = 0xDBFF; -const uint16_t sur_low_start = 0xDC00; -const uint16_t sur_low_end = 0xDFFF; - -inline -static bool is_continuation_byte(unsigned char ch) -{ - return (ch & 0xC0) == 0x80; -} - -inline -bool is_high_surrogate(uint32_t ch) UNICONS_NOEXCEPT -{ - return (ch >= sur_high_start && ch <= sur_high_end); -} - -inline -bool is_low_surrogate(uint32_t ch) UNICONS_NOEXCEPT -{ - return (ch >= sur_low_start && ch <= sur_low_end); -} - -inline -bool is_surrogate(uint32_t ch) UNICONS_NOEXCEPT -{ - return (ch >= sur_high_start && ch <= sur_low_end); -} - -enum class conv_flags -{ - strict = 0, - lenient -}; - -// conv_errc - -enum class conv_errc -{ - ok = 0, - over_long_utf8_sequence = 1, // over long utf8 sequence - expected_continuation_byte, // expected continuation byte - unpaired_high_surrogate, // unpaired high surrogate UTF-16 - illegal_surrogate_value, // UTF-16 surrogate values are illegal in UTF-32 - source_exhausted, // partial character in source, but hit end - source_illegal // source sequence is illegal/malformed -}; - -class Unicode_traits_error_category_impl_ - : public std::error_category -{ -public: - virtual const char* name() const UNICONS_NOEXCEPT - { - return "unicons conversion error"; - } - virtual std::string message(int ev) const - { - switch (static_cast(ev)) - { - case conv_errc::over_long_utf8_sequence: - return "Over long utf8 sequence"; - case conv_errc::expected_continuation_byte: - return "Expected continuation byte"; - case conv_errc::unpaired_high_surrogate: - return "Unpaired high surrogate UTF-16"; - case conv_errc::illegal_surrogate_value: - return "UTF-16 surrogate values are illegal in UTF-32"; - case conv_errc::source_exhausted: - return "Partial character in source, but hit end"; - case conv_errc::source_illegal: - return "Source sequence is illegal/malformed"; - default: - return ""; - break; - } - } -}; - -inline -const std::error_category& unicode_traits_error_category() -{ - static Unicode_traits_error_category_impl_ instance; - return instance; -} - -inline -std::error_code make_error_code(conv_errc result) -{ - return std::error_code(static_cast(result),unicode_traits_error_category()); -} - -// encoding_errc - -enum class encoding_errc -{ - ok = 0, - expected_u8_found_u16 = 1, - expected_u8_found_u32, - expected_u16_found_fffe, - expected_u32_found_fffe -}; - -class Encoding_errc_impl_ - : public std::error_category -{ -public: - virtual const char* name() const UNICONS_NOEXCEPT - { - return "unicons encoding error"; - } - virtual std::string message(int ev) const - { - switch (static_cast(ev)) - { - case encoding_errc::expected_u8_found_u16: - return "Expected UTF-8, found UTF-16"; - case encoding_errc::expected_u8_found_u32: - return "Expected UTF-8, found UTF-32"; - case encoding_errc::expected_u16_found_fffe: - return "Expected UTF-16, found non character"; - case encoding_errc::expected_u32_found_fffe: - return "Expected UTF-32, found non character"; - default: - return ""; - break; - } - } -}; - -inline -const std::error_category& encoding_error_category() -{ - static Encoding_errc_impl_ instance; - return instance; -} - -inline -std::error_code make_error_code(encoding_errc result) -{ - return std::error_code(static_cast(result),encoding_error_category()); -} - -// utf8 - -template -typename std::enable_if::value_type>::value - && sizeof(typename std::iterator_traits::value_type) == sizeof(uint8_t), - conv_errc >::type -is_legal_utf8(Iterator first, size_t length) -{ - uint8_t a; - Iterator srcptr = first+length; - switch (length) { - default: - return conv_errc::over_long_utf8_sequence; - case 4: - if (((a = (*--srcptr))& 0xC0) != 0x80) - return conv_errc::expected_continuation_byte; - UNICONS_FALLTHROUGH; - case 3: - if (((a = (*--srcptr))& 0xC0) != 0x80) - return conv_errc::expected_continuation_byte; - UNICONS_FALLTHROUGH; - case 2: - if (((a = (*--srcptr))& 0xC0) != 0x80) - return conv_errc::expected_continuation_byte; - - switch (static_cast(*first)) - { - /* no fall-through in this inner switch */ - case 0xE0: if (a < 0xA0) return conv_errc::source_illegal; break; - case 0xED: if (a > 0x9F) return conv_errc::source_illegal; break; - case 0xF0: if (a < 0x90) return conv_errc::source_illegal; break; - case 0xF4: if (a > 0x8F) return conv_errc::source_illegal; break; - default: if (a < 0x80) return conv_errc::source_illegal; - } - - UNICONS_FALLTHROUGH; - case 1: - if (static_cast(*first) >= 0x80 && static_cast(*first) < 0xC2) - return conv_errc::source_illegal; - break; - } - if (static_cast(*first) > 0xF4) - return conv_errc::source_illegal; - - return conv_errc(); -} - -template using void_t = void; - -template -struct is_output_iterator : std::false_type {}; - -template -struct is_output_iterator::iterator_category, - decltype(*std::declval() = std::declval())>> : std::true_type {}; - -// is_same_size fixes issue with vs2013 - -// primary template -template -struct is_same_size : std::false_type -{ -}; - -// specialization for non void types -template -struct is_same_size::value && !std::is_void::value>::type> -{ - static const bool value = (sizeof(T1) == sizeof(T2)); -}; - -template -struct is_compatible_output_iterator : std::false_type {}; - -template -struct is_compatible_output_iterator::value - && std::is_void::value_type>::value - && std::is_integral::value - && !std::is_void::value - && is_same_size::value>::type -> : std::true_type {}; - -template -struct is_compatible_output_iterator::value - && std::is_integral::value_type>::value - && is_same_size::value_type,CharT>::value>::type -> : std::true_type {}; - -template -struct is_compatible_output_iterator::value - && std::is_void::value_type>::value - && is_same_size::value>::type -> : std::true_type {}; - -// convert - -template -struct convert_result -{ - Iterator it; - conv_errc ec; -}; - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint8_t) - && is_compatible_output_iterator::value,convert_result>::type -convert(InputIt first, InputIt last, OutputIt target, conv_flags flags=conv_flags::strict) -{ - (void)flags; - - conv_errc result = conv_errc(); - while (first != last) - { - size_t length = trailing_bytes_for_utf8[static_cast(*first)] + 1; - if (length > (size_t)(last - first)) - { - return convert_result{first, conv_errc::source_exhausted}; - } - if ((result=is_legal_utf8(first, length)) != conv_errc()) - { - return convert_result{first,result}; - } - - switch (length) { - case 4: *target++ = (static_cast(*first++)); - UNICONS_FALLTHROUGH; - case 3: *target++ = (static_cast(*first++)); - UNICONS_FALLTHROUGH; - case 2: *target++ = (static_cast(*first++)); - UNICONS_FALLTHROUGH; - case 1: *target++ = (static_cast(*first++)); - } - } - return convert_result{first,result} ; -} - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint8_t) - && is_compatible_output_iterator::value,convert_result>::type -convert(InputIt first, InputIt last, - OutputIt target, - conv_flags flags = conv_flags::strict) -{ - conv_errc result = conv_errc(); - - while (first != last) - { - uint32_t ch = 0; - unsigned short extra_bytes_to_read = trailing_bytes_for_utf8[static_cast(*first)]; - if (extra_bytes_to_read >= last - first) - { - result = conv_errc::source_exhausted; - break; - } - /* Do this check whether lenient or strict */ - if ((result=is_legal_utf8(first, extra_bytes_to_read+1)) != conv_errc()) - { - break; - } - /* - * The cases all fall through. See "Note A" below. - */ - switch (extra_bytes_to_read) { - case 5: ch += static_cast(*first++); ch <<= 6; /* remember, illegal UTF-8 */ - case 4: ch += static_cast(*first++); ch <<= 6; /* remember, illegal UTF-8 */ - case 3: ch += static_cast(*first++); ch <<= 6; - case 2: ch += static_cast(*first++); ch <<= 6; - case 1: ch += static_cast(*first++); ch <<= 6; - case 0: ch += static_cast(*first++); - } - ch -= offsets_from_utf8[extra_bytes_to_read]; - - if (ch <= max_bmp) { /* Target is a character <= 0xFFFF */ - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (is_surrogate(ch) ) { - if (flags == conv_flags::strict) { - first -= (extra_bytes_to_read+1); /* return to the illegal value itself */ - result = conv_errc::source_illegal; - break; - } else { - *target++ = (replacement_char); - } - } else { - *target++ = ((uint16_t)ch); /* normal case */ - } - } else if (ch > max_utf16) { - if (flags == conv_flags::strict) { - result = conv_errc::source_illegal; - first -= (extra_bytes_to_read+1); /* return to the start */ - break; /* Bail out; shouldn't continue */ - } else { - *target++ = (replacement_char); - } - } else { - /* target is a character in range 0xFFFF - 0x10FFFF. */ - ch -= half_base; - *target++ = ((uint16_t)((ch >> half_shift) + sur_high_start)); - *target++ = ((uint16_t)((ch & half_mask) + sur_low_start)); - } - } - return convert_result{first,result} ; -} - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint8_t) - && is_compatible_output_iterator::value,convert_result>::type -convert(InputIt first, InputIt last, - OutputIt target, - conv_flags flags = conv_flags::strict) -{ - conv_errc result = conv_errc(); - - while (first < last) - { - uint32_t ch = 0; - unsigned short extra_bytes_to_read = trailing_bytes_for_utf8[static_cast(*first)]; - if (extra_bytes_to_read >= last - first) - { - result = conv_errc::source_exhausted; - break; - } - /* Do this check whether lenient or strict */ - if ((result=is_legal_utf8(first, extra_bytes_to_read+1)) != conv_errc()) { - break; - } - /* - * The cases all fall through. See "Note A" below. - */ - switch (extra_bytes_to_read) - { - case 5: - ch += static_cast(*first++); - ch <<= 6; - UNICONS_FALLTHROUGH; - case 4: - ch += static_cast(*first++); - ch <<= 6; - UNICONS_FALLTHROUGH; - case 3: - ch += static_cast(*first++); - ch <<= 6; - UNICONS_FALLTHROUGH; - case 2: - ch += static_cast(*first++); - ch <<= 6; - UNICONS_FALLTHROUGH; - case 1: - ch += static_cast(*first++); - ch <<= 6; - UNICONS_FALLTHROUGH; - case 0: - ch += static_cast(*first++); - break; - } - ch -= offsets_from_utf8[extra_bytes_to_read]; - - if (ch <= max_legal_utf32) { - /* - * UTF-16 surrogate values are illegal in UTF-32, and anything - * over Plane 17 (> 0x10FFFF) is illegal. - */ - if (is_surrogate(ch) ) { - if (flags == conv_flags::strict) { - first -= (extra_bytes_to_read+1); /* return to the illegal value itself */ - result = conv_errc::source_illegal; - break; - } else { - *target++ = (replacement_char); - } - } else { - *target++ = (ch); - } - } else { /* i.e., ch > max_legal_utf32 */ - result = conv_errc::source_illegal; - *target++ = (replacement_char); - } - } - return convert_result{first,result} ; -} - -// utf16 - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint16_t) - && is_compatible_output_iterator::value,convert_result>::type -convert(InputIt first, InputIt last, - OutputIt target, - conv_flags flags = conv_flags::strict) { - conv_errc result = conv_errc(); - while (first < last) { - unsigned short bytes_to_write = 0; - const uint32_t byteMask = 0xBF; - const uint32_t byteMark = 0x80; - uint32_t ch = *first++; - /* If we have a surrogate pair, convert to uint32_t first. */ - if (is_high_surrogate(ch)) { - /* If the 16 bits following the high surrogate are in the first buffer... */ - if (first < last) { - uint32_t ch2 = *first; - /* If it's a low surrogate, convert to uint32_t. */ - if (ch2 >= sur_low_start && ch2 <= sur_low_end) { - ch = ((ch - sur_high_start) << half_shift) - + (ch2 - sur_low_start) + half_base; - ++first; - } else if (flags == conv_flags::strict) { /* it's an unpaired high surrogate */ - --first; /* return to the illegal value itself */ - result = conv_errc::unpaired_high_surrogate; - break; - } - } else { /* We don't have the 16 bits following the high surrogate. */ - --first; /* return to the high surrogate */ - result = conv_errc::source_exhausted; - break; - } - } else if (flags == conv_flags::strict) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (is_low_surrogate(ch)) { - --first; /* return to the illegal value itself */ - result = conv_errc::source_illegal; - break; - } - } - /* Figure out how many bytes the result will require */ - if (ch < (uint32_t)0x80) { - bytes_to_write = 1; - } else if (ch < (uint32_t)0x800) { - bytes_to_write = 2; - } else if (ch < (uint32_t)0x10000) { - bytes_to_write = 3; - } else if (ch < (uint32_t)0x110000) { - bytes_to_write = 4; - } else { - bytes_to_write = 3; - ch = replacement_char; - } - - uint8_t byte1 = 0; - uint8_t byte2 = 0; - uint8_t byte3 = 0; - uint8_t byte4 = 0; - - switch (bytes_to_write) { // note: everything falls through - case 4: byte4 = (uint8_t)((ch | byteMark) & byteMask); ch >>= 6; - case 3: byte3 = (uint8_t)((ch | byteMark) & byteMask); ch >>= 6; - case 2: byte2 = (uint8_t)((ch | byteMark) & byteMask); ch >>= 6; - case 1: byte1 = (uint8_t)(ch | first_byte_mark[bytes_to_write]); - } - switch (bytes_to_write) - { - case 4: - *target++ = (byte1); - *target++ = (byte2); - *target++ = (byte3); - *target++ = (byte4); - break; - case 3: - *target++ = (byte1); - *target++ = (byte2); - *target++ = (byte3); - break; - case 2: - *target++ = (byte1); - *target++ = (byte2); - break; - case 1: - *target++ = (byte1); - break; - } - } - return convert_result{first,result} ; -} - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint16_t) - && is_compatible_output_iterator::value,convert_result>::type -convert(InputIt first, InputIt last, - OutputIt target, - conv_flags flags = conv_flags::strict) -{ - conv_errc result = conv_errc(); - - while (first != last) - { - uint32_t ch = *first++; - /* If we have a surrogate pair, convert to uint32_t first. */ - if (is_high_surrogate(ch)) - { - /* If the 16 bits following the high surrogate are in the first buffer... */ - if (first < last) { - uint32_t ch2 = *first; - /* If it's a low surrogate, */ - if (ch2 >= sur_low_start && ch2 <= sur_low_end) { - *target++ = ((uint16_t)ch); - *target++ = ((uint16_t)ch2); - ++first; - } else if (flags == conv_flags::strict) { /* it's an unpaired high surrogate */ - --first; /* return to the illegal value itself */ - result = conv_errc::unpaired_high_surrogate; - break; - } - } else { /* We don't have the 16 bits following the high surrogate. */ - --first; /* return to the high surrogate */ - result = conv_errc::source_exhausted; - break; - } - } else if (is_low_surrogate(ch)) - { - // illegal leading low surrogate - if (flags == conv_flags::strict) { - --first; /* return to the illegal value itself */ - result = conv_errc::source_illegal; - break; - } - else - { - *target++ = ((uint16_t)ch); - } - } - else - { - *target++ = ((uint16_t)ch); - } - } - return convert_result{first,result} ; -} - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint16_t) - && is_compatible_output_iterator::value,convert_result>::type -convert(InputIt first, InputIt last, - OutputIt target, - conv_flags flags = conv_flags::strict) -{ - conv_errc result = conv_errc(); - - while (first != last) - { - uint32_t ch = *first++; - /* If we have a surrogate pair, convert to UTF32 first. */ - if (is_high_surrogate(ch)) { - /* If the 16 bits following the high surrogate are in the first buffer... */ - if (first < last) { - uint32_t ch2 = *first; - /* If it's a low surrogate, convert to UTF32. */ - if (ch2 >= sur_low_start && ch2 <= sur_low_end ) { - ch = ((ch - sur_high_start) << half_shift) - + (ch2 - sur_low_start) + half_base; - ++first; - } else if (flags == conv_flags::strict) { /* it's an unpaired high surrogate */ - --first; /* return to the illegal value itself */ - result = conv_errc::source_illegal; - break; - } - } else { /* We don't have the 16 bits following the high surrogate. */ - --first; /* return to the high surrogate */ - result = conv_errc::source_exhausted; - break; - } - } else if (flags == conv_flags::strict) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (is_low_surrogate(ch) ) { - --first; /* return to the illegal value itself */ - result = conv_errc::source_illegal; - break; - } - } - *target++ = (ch); - } - return convert_result{first,result} ; -} - -// utf32 - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint32_t) - && is_compatible_output_iterator::value,convert_result>::type -convert(InputIt first, InputIt last, - OutputIt target, - conv_flags flags = conv_flags::strict) -{ - conv_errc result = conv_errc(); - while (first < last) { - unsigned short bytes_to_write = 0; - const uint32_t byteMask = 0xBF; - const uint32_t byteMark = 0x80; - uint32_t ch = *first++; - if (flags == conv_flags::strict ) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (is_surrogate(ch)) { - --first; /* return to the illegal value itself */ - result = conv_errc::illegal_surrogate_value; - break; - } - } - /* - * Figure out how many bytes the result will require. Turn any - * illegally large UTF32 things (> Plane 17) into replacement chars. - */ - if (ch < (uint32_t)0x80) { bytes_to_write = 1; - } else if (ch < (uint32_t)0x800) { bytes_to_write = 2; - } else if (ch < (uint32_t)0x10000) { bytes_to_write = 3; - } else if (ch <= max_legal_utf32) { bytes_to_write = 4; - } else { - bytes_to_write = 3; - ch = replacement_char; - result = conv_errc::source_illegal; - } - - uint8_t byte1 = 0; - uint8_t byte2 = 0; - uint8_t byte3 = 0; - uint8_t byte4 = 0; - - switch (bytes_to_write) { - case 4: - byte4 = (uint8_t)((ch | byteMark) & byteMask); ch >>= 6; - UNICONS_FALLTHROUGH; - case 3: - byte3 = (uint8_t)((ch | byteMark) & byteMask); ch >>= 6; - UNICONS_FALLTHROUGH; - case 2: - byte2 = (uint8_t)((ch | byteMark) & byteMask); ch >>= 6; - UNICONS_FALLTHROUGH; - case 1: - byte1 = (uint8_t) (ch | first_byte_mark[bytes_to_write]); - } - - switch (bytes_to_write) - { - case 4: - *target++ = (byte1); - *target++ = (byte2); - *target++ = (byte3); - *target++ = (byte4); - break; - case 3: - *target++ = (byte1); - *target++ = (byte2); - *target++ = (byte3); - break; - case 2: - *target++ = (byte1); - *target++ = (byte2); - break; - case 1: - *target++ = (byte1); - break; - } - } - return convert_result{first,result} ; -} - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint32_t) - && is_compatible_output_iterator::value,convert_result>::type -convert(InputIt first, InputIt last, - OutputIt target, - conv_flags flags = conv_flags::strict) -{ - conv_errc result = conv_errc(); - - while (first != last) - { - uint32_t ch = *first++; - if (ch <= max_bmp) { /* Target is a character <= 0xFFFF */ - /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */ - if (is_surrogate(ch) ) { - if (flags == conv_flags::strict) { - --first; /* return to the illegal value itself */ - result = conv_errc::source_illegal; - break; - } else { - *target++ = (replacement_char); - } - } else { - *target++ = ((uint16_t)ch); /* normal case */ - } - } else if (ch > max_legal_utf32) { - if (flags == conv_flags::strict) { - result = conv_errc::source_illegal; - } else { - *target++ = (replacement_char); - } - } else { - /* target is a character in range 0xFFFF - 0x10FFFF. */ - ch -= half_base; - *target++ = ((uint16_t)((ch >> half_shift) + sur_high_start)); - *target++ = ((uint16_t)((ch & half_mask) + sur_low_start)); - } - } - return convert_result{first,result} ; -} - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint32_t) - && is_compatible_output_iterator::value,convert_result>::type -convert(InputIt first, InputIt last, - OutputIt target, - conv_flags flags = conv_flags::strict) -{ - conv_errc result = conv_errc(); - - while (first != last) - { - uint32_t ch = *first++; - if (flags == conv_flags::strict ) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (is_surrogate(ch)) { - --first; /* return to the illegal value itself */ - result = conv_errc::illegal_surrogate_value; - break; - } - } - if (ch <= max_legal_utf32) - { - *target++ = (ch); - } - else - { - *target++ = (replacement_char); - result = conv_errc::source_illegal; - } - } - return convert_result{first,result} ; -} - -// validate - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint8_t) - ,convert_result>::type -validate(InputIt first, InputIt last) UNICONS_NOEXCEPT -{ - conv_errc result = conv_errc(); - while (first != last) - { - size_t length = trailing_bytes_for_utf8[static_cast(*first)] + 1; - if (length > (size_t)(last - first)) - { - return convert_result{first, conv_errc::source_exhausted}; - } - if ((result=is_legal_utf8(first, length)) != conv_errc()) - { - return convert_result{first,result} ; - } - first += length; - } - return convert_result{first,result} ; -} - -// utf16 - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint16_t) - ,convert_result>::type -validate(InputIt first, InputIt last) UNICONS_NOEXCEPT -{ - conv_errc result = conv_errc(); - - while (first != last) - { - uint32_t ch = *first++; - /* If we have a surrogate pair, validate to uint32_t first. */ - if (is_high_surrogate(ch)) - { - /* If the 16 bits following the high surrogate are in the first buffer... */ - if (first < last) { - uint32_t ch2 = *first; - /* If it's a low surrogate, */ - if (ch2 >= sur_low_start && ch2 <= sur_low_end) { - ++first; - } else { - --first; /* return to the illegal value itself */ - result = conv_errc::unpaired_high_surrogate; - break; - } - } else { /* We don't have the 16 bits following the high surrogate. */ - --first; /* return to the high surrogate */ - result = conv_errc::source_exhausted; - break; - } - } else if (is_low_surrogate(ch)) - { - /* UTF-16 surrogate values are illegal in UTF-32 */ - --first; /* return to the illegal value itself */ - result = conv_errc::source_illegal; - break; - } - } - return convert_result{first,result} ; -} - - -// utf32 - - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint32_t) - ,convert_result>::type -validate(InputIt first, InputIt last) UNICONS_NOEXCEPT -{ - conv_errc result = conv_errc(); - - while (first != last) - { - uint32_t ch = *first++; - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (is_surrogate(ch)) { - --first; /* return to the illegal value itself */ - result = conv_errc::illegal_surrogate_value; - break; - } - if (!(ch <= max_legal_utf32)) - { - result = conv_errc::source_illegal; - } - } - return convert_result{first,result} ; -} - -// sequence - -template -class sequence -{ - Iterator first_; - size_t length_; -public: - sequence(Iterator first, size_t length) - : first_(first), length_(length) - { - } - - Iterator begin() const - { - return first_; - } - - size_t length() const - { - return length_; - } - - template ::value_type> - typename std::enable_if::type - codepoint() const UNICONS_NOEXCEPT - { - uint32_t ch = 0; - Iterator it = first_; - switch (length_) - { - default: - return replacement_char; - break; - case 4: - ch += static_cast(*it++); ch <<= 6; - UNICONS_FALLTHROUGH; - case 3: - ch += static_cast(*it++); ch <<= 6; - UNICONS_FALLTHROUGH; - case 2: - ch += static_cast(*it++); ch <<= 6; - UNICONS_FALLTHROUGH; - case 1: - ch += static_cast(*it++); - ch -= offsets_from_utf8[length_ - 1]; - break; - } - if (ch <= max_legal_utf32) - { - if (is_surrogate(ch)) - { - ch = replacement_char; - } - } - else // ch > max_legal_utf32 - { - ch = replacement_char; - } - return ch; - } - - template ::value_type> - typename std::enable_if::type - codepoint() const UNICONS_NOEXCEPT - { - if (length_ == 0) - { - return replacement_char; - } - if (length_ == 2) - { - uint32_t ch = *first_; - uint32_t ch2 = *(first_+ 1); - ch = ((ch - sur_high_start) << half_shift) - + (ch2 - sur_low_start) + half_base; - return ch; - } - else - { - return *first_; - } - } - - template ::value_type> - typename std::enable_if::type - codepoint() const UNICONS_NOEXCEPT - { - if (length_ == 0) - { - return replacement_char; - } - return *(first_); - } -}; - -// sequence_generator - -template -class sequence_generator -{ - Iterator begin_; - Iterator last_; - conv_flags flags_; - size_t length_; - conv_errc err_cd_; -public: - typedef sequence sequence_type; - - sequence_generator(Iterator first, Iterator last, - conv_flags flags = conv_flags::strict) UNICONS_NOEXCEPT - : begin_(first), last_(last), flags_(flags), - length_(0), err_cd_(conv_errc()) - { - next(); - } - - bool done() const UNICONS_NOEXCEPT - { - return err_cd_ != conv_errc() || begin_ == last_; - } - - conv_errc status() const UNICONS_NOEXCEPT - { - return err_cd_; - } - - sequence_type get() const UNICONS_NOEXCEPT - { - return sequence(begin_,length_); - } - - template ::value_type> - typename std::enable_if::type - next() UNICONS_NOEXCEPT - { - begin_ += length_; - if (begin_ != last_) - { - size_t length = trailing_bytes_for_utf8[static_cast(*begin_)] + 1; - if (length > (size_t)(last_ - begin_)) - { - err_cd_ = conv_errc::source_exhausted; - } - else if ((err_cd_ = is_legal_utf8(begin_, length)) != conv_errc()) - { - } - else - { - length_ = length; - } - } - } - - template ::value_type> - typename std::enable_if::type - next() UNICONS_NOEXCEPT - { - begin_ += length_; - if (begin_ != last_) - { - if (begin_ != last_) - { - - Iterator it = begin_; - - uint32_t ch = *it++; - /* If we have a surrogate pair, validate to uint32_t it. */ - if (is_high_surrogate(ch)) - { - /* If the 16 bits following the high surrogate are in the it buffer... */ - if (it < last_) { - uint32_t ch2 = *it; - /* If it's a low surrogate, */ - if (ch2 >= sur_low_start && ch2 <= sur_low_end) - { - ++it; - length_ = 2; - } - else - { - err_cd_ = conv_errc::unpaired_high_surrogate; - } - } - else - { - // We don't have the 16 bits following the high surrogate. - err_cd_ = conv_errc::source_exhausted; - } - } - else if (is_low_surrogate(ch)) - { - /* leading low surrogate */ - err_cd_ = conv_errc::source_illegal; - } - else - { - length_ = 1; - } - } - } - } - - template ::value_type> - typename std::enable_if::type - next() UNICONS_NOEXCEPT - { - begin_ += length_; - length_ = 1; - } -}; - -template -sequence_generator make_sequence_generator(Iterator first, Iterator last, - conv_flags flags = conv_flags::strict) -{ - return sequence_generator(first, last, flags); -} - -template -typename std::enable_if::value_type>::value - && (sizeof(typename std::iterator_traits::value_type) == sizeof(uint8_t) || sizeof(typename std::iterator_traits::value_type) == sizeof(uint16_t)), - sequence>::type -sequence_at(InputIt first, InputIt last, size_t index) -{ - sequence_generator g(first, last, unicons::conv_flags::strict); - - size_t count = 0; - while (!g.done() && count < index) - { - g.next(); - ++count; - } - return (!g.done() && count == index) ? g.get() : sequence(last,0); -} - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint32_t), - sequence>::type -sequence_at(InputIt first, InputIt last, size_t index) -{ - size_t size = std::distance(first,last); - return index < size ? sequence(first+index,1) : sequence(last,0); -} - -// u8_length - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint8_t),size_t>::type -u8_length(InputIt first, InputIt last) UNICONS_NOEXCEPT -{ - return std::distance(first,last); -} - -// utf16 - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint16_t),size_t>::type -u8_length(InputIt first, InputIt last) UNICONS_NOEXCEPT -{ - conv_flags flags = conv_flags::strict; - size_t count = 0; - for (InputIt p = first; p != last; ++p) - { - uint32_t ch = *p; - if (is_high_surrogate(ch)) { - /* If the 16 bits following the high surrogate are in the p buffer... */ - if (p < last) { - uint32_t ch2 = *(++p); - /* If it's a low surrogate, convert to uint32_t. */ - if (ch2 >= sur_low_start && ch2 <= sur_low_end) { - ch = ((ch - sur_high_start) << half_shift) - + (ch2 - sur_low_start) + half_base; - - } else if (flags == conv_flags::strict) { /* it's an unpaired high surrogate */ - break; - } - } else { /* We don't have the 16 bits following the high surrogate. */ - break; - } - } else if (flags == conv_flags::strict) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (is_low_surrogate(ch)) { - break; - } - } - if (ch < (uint32_t)0x80) { - ++count; - } else if (ch < (uint32_t)0x800) { - count += 2; - } else if (ch < (uint32_t)0x10000) { - count += 3; - } else if (ch < (uint32_t)0x110000) { - count += 4; - } else { - count += 3; - } - } - return count; -} - - -// utf32 - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint32_t),size_t>::type -u8_length(InputIt first, InputIt last) UNICONS_NOEXCEPT -{ - size_t count = 0; - for (InputIt p = first; p < last; ++p) - { - uint32_t ch = *p; - if (ch < (uint32_t)0x80) { - ++count; - } else if (ch < (uint32_t)0x800) { - count += 2; - } else if (ch < (uint32_t)0x10000) { - count += 3; - } else if (ch <= max_legal_utf32) { - count += 4; - } else { - count += 3; - } - } - return count; -} - -// u32_length - -template -typename std::enable_if::value_type>::value - && (sizeof(typename std::iterator_traits::value_type) == sizeof(uint8_t) || sizeof(typename std::iterator_traits::value_type) == sizeof(uint16_t)), - size_t>::type -u32_length(InputIt first, InputIt last) UNICONS_NOEXCEPT -{ - sequence_generator g(first, last, unicons::conv_flags::strict); - - size_t count = 0; - while (!g.done()) - { - g.next(); - ++count; - } - return count; -} - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint32_t), - size_t>::type -u32_length(InputIt first, InputIt last) UNICONS_NOEXCEPT -{ - return std::distance(first,last); -} - -enum class encoding {u8,u16le,u16be,u32le,u32be,undetected}; - -template -struct detect_encoding_result -{ - Iterator it; - encoding ec; -}; - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint8_t), - detect_encoding_result>::type -detect_encoding(Iterator first, Iterator last) UNICONS_NOEXCEPT -{ - Iterator it1 = first; - if (std::distance(first,last) < 4) - { - if (std::distance(first,last) == 3) - { - Iterator it2 = ++first; - Iterator it3 = ++first; - if (static_cast(*it1) == 0xEF && static_cast(*it2) == 0xBB && static_cast(*it3) == 0xBF) - { - return detect_encoding_result{last,encoding::u8}; - } - } - return detect_encoding_result{it1,encoding::undetected}; - } - else - { - Iterator it2 = ++first; - Iterator it3 = ++first; - Iterator it4 = ++first; - - uint32_t bom = static_cast(*it1) | (static_cast(*it2) << 8) | (static_cast(*it3) << 16) | (static_cast(*it4) << 24); - if (bom == 0xFFFE0000) - { - return detect_encoding_result{it4++,encoding::u32be}; - } - else if (bom == 0x0000FEFF) - { - return detect_encoding_result{first,encoding::u32le}; - } - else if ((bom & 0xFFFF) == 0xFFFE) - { - return detect_encoding_result{it3,encoding::u16be}; - } - else if ((bom & 0xFFFF) == 0xFEFF) - { - return detect_encoding_result{it3,encoding::u16le}; - } - else if ((bom & 0xFFFFFF) == 0xBFBBEF) - { - return detect_encoding_result{it4,encoding::u8}; - } - else - { - uint32_t pattern = (static_cast(*it1) ? 1 : 0) | (static_cast(*it2) ? 2 : 0) | (static_cast(*it3) ? 4 : 0) | (static_cast(*it4) ? 8 : 0); - switch (pattern) { - case 0x08: - return detect_encoding_result{it1,encoding::u32be}; - case 0x0A: - return detect_encoding_result{it1,encoding::u16be}; - case 0x01: - return detect_encoding_result{it1,encoding::u32le}; - case 0x05: - return detect_encoding_result{it1,encoding::u16le}; - case 0x0F: - return detect_encoding_result{it1,encoding::u8}; - default: - return detect_encoding_result{it1,encoding::undetected}; - } - } - } -} - -template -struct skip_bom_result -{ - Iterator it; - encoding_errc ec; -}; - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint8_t), - skip_bom_result>::type -skip_bom(Iterator first, Iterator last) UNICONS_NOEXCEPT -{ - auto result = unicons::detect_encoding(first,last); - switch (result.ec) - { - case unicons::encoding::u8: - return skip_bom_result{result.it,encoding_errc()}; - break; - case unicons::encoding::u16le: - case unicons::encoding::u16be: - return skip_bom_result{result.it,encoding_errc::expected_u8_found_u16}; - break; - case unicons::encoding::u32le: - case unicons::encoding::u32be: - return skip_bom_result{result.it,encoding_errc::expected_u8_found_u32}; - break; - default: - return skip_bom_result{result.it,encoding_errc()}; - break; - } -} - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint16_t), - skip_bom_result>::type -skip_bom(Iterator first, Iterator last) UNICONS_NOEXCEPT -{ - if (first == last) - { - return skip_bom_result{first,encoding_errc()}; - } - uint16_t bom = static_cast(*first); - if (bom == 0xFEFF) - { - return skip_bom_result{++first,encoding_errc()}; - } - else if (bom == 0xFFFE) - { - return skip_bom_result{last,encoding_errc::expected_u16_found_fffe}; - } - else - { - return skip_bom_result{first,encoding_errc()}; - } -} - -template -typename std::enable_if::value_type>::value && sizeof(typename std::iterator_traits::value_type) == sizeof(uint32_t), - skip_bom_result>::type -skip_bom(Iterator first, Iterator last) UNICONS_NOEXCEPT -{ - if (first == last) - { - return skip_bom_result{first,encoding_errc()}; - } - uint32_t bom = static_cast(*first); - if (bom == 0xFEFF0000) - { - return skip_bom_result{++first,encoding_errc()}; - } - else if (bom == 0xFFFE0000) - { - return skip_bom_result{last,encoding_errc::expected_u32_found_fffe}; - } - else - { - return skip_bom_result{first,encoding_errc()}; - } -} - -} // unicons -} // jsoncons - -namespace std { - template<> - struct is_error_code_enum : public true_type - { - }; - template<> - struct is_error_code_enum : public true_type - { - }; -} - -#endif - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson.hpp deleted file mode 100644 index 7d14eda0c4..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson.hpp +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_BSON_BSON_HPP -#define JSONCONS_BSON_BSON_HPP - -#include -#include -#include -#include // std::enable_if -#include // std::basic_istream -#include -#include -#include -#include - -namespace jsoncons { namespace bson { - -// encode_bson - -template -typename std::enable_if::value,void>::type -encode_bson(const T& j, std::vector& v) -{ - typedef typename T::char_type char_type; - bson_bytes_encoder encoder(v); - auto adaptor = make_json_content_handler_adaptor>(encoder); - j.dump(adaptor); -} - -template -typename std::enable_if::value,void>::type -encode_bson(const T& val, std::vector& v) -{ - bson_bytes_encoder encoder(v); - write_to(json(), val, encoder); -} - -template -typename std::enable_if::value,void>::type -encode_bson(const T& j, std::ostream& os) -{ - typedef typename T::char_type char_type; - bson_encoder encoder(os); - auto adaptor = make_json_content_handler_adaptor>(encoder); - j.dump(adaptor); -} - -template -typename std::enable_if::value,void>::type -encode_bson(const T& val, std::ostream& os) -{ - bson_encoder encoder(os); - write_to(json(), val, encoder); -} - -// decode_bson - -template -typename std::enable_if::value,T>::type -decode_bson(const std::vector& v) -{ - jsoncons::json_decoder decoder; - auto adaptor = make_json_content_handler_adaptor(decoder); - basic_bson_reader reader(v, adaptor); - reader.read(); - return decoder.get_result(); -} - -template -typename std::enable_if::value,T>::type -decode_bson(const std::vector& v) -{ - jsoncons::json_decoder decoder; - basic_bson_reader reader(v, decoder); - reader.read(); - return decoder.get_result().template as(); -} - -template -typename std::enable_if::value,T>::type -decode_bson(std::istream& is) -{ - jsoncons::json_decoder decoder; - auto adaptor = make_json_content_handler_adaptor(decoder); - bson_reader reader(is, adaptor); - reader.read(); - return decoder.get_result(); -} - -template -typename std::enable_if::value,T>::type -decode_bson(std::istream& is) -{ - jsoncons::json_decoder decoder; - bson_reader reader(is, decoder); - reader.read(); - return decoder.get_result(); -} - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson_detail.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson_detail.hpp deleted file mode 100644 index a1b50025eb..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson_detail.hpp +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_BSON_BSON_DETAIL_HPP -#define JSONCONS_BSON_BSON_DETAIL_HPP - -#include -#include -#include - -namespace jsoncons { namespace bson { namespace detail { - -namespace bson_format -{ - const uint8_t double_cd = 0x01; - const uint8_t string_cd = 0x02; // UTF-8 string - const uint8_t document_cd = 0x03; - const uint8_t array_cd = 0x04; - const uint8_t binary_cd = 0x05; - const uint8_t object_id_cd = 0x07; - const uint8_t bool_cd = 0x08; - const uint8_t datetime_cd = 0x09; - const uint8_t null_cd = 0x0a; - const uint8_t regex_cd = 0x0b; - const uint8_t javascript_cd = 0x0d; - const uint8_t javascript_with_scope_cd = 0x0f; - const uint8_t int32_cd = 0x10; - const uint8_t timestamp_cd = 0x11; // MongoDB internal Timestamp, uint64 - const uint8_t int64_cd = 0x12; - const uint8_t decimal128_cd = 0x13; - const uint8_t min_key_cd = 0xff; - const uint8_t max_key_cd = 0x7f; -} - -enum class bson_container_type {document, array}; - -}}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson_encoder.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson_encoder.hpp deleted file mode 100644 index 4161bb2505..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson_encoder.hpp +++ /dev/null @@ -1,342 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_BSON_BSON_ENCODER_HPP -#define JSONCONS_BSON_BSON_ENCODER_HPP - -#include -#include -#include // std::numeric_limits -#include -#include // std::move -#include -#include -#include -#include -#include -#include - -namespace jsoncons { namespace bson { - -template -class basic_bson_encoder final : public basic_json_content_handler -{ - enum class decimal_parse_state { start, integer, exp1, exp2, fraction1 }; -public: - typedef char char_type; - using typename basic_json_content_handler::string_view_type; - typedef Result result_type; - -private: - struct stack_item - { - jsoncons::bson::detail::bson_container_type type_; - size_t offset_; - size_t name_offset_; - size_t index_; - - stack_item(jsoncons::bson::detail::bson_container_type type, size_t offset) - : type_(type), offset_(offset), name_offset_(0), index_(0) - { - } - - size_t offset() const - { - return offset_; - } - - size_t member_offset() const - { - return name_offset_; - } - - void member_offset(size_t offset) - { - name_offset_ = offset; - } - - size_t next_index() - { - return index_++; - } - - bool is_object() const - { - return type_ == jsoncons::bson::detail::bson_container_type::document; - } - - - }; - - std::vector stack_; - std::vector buffer_; - result_type result_; - - // Noncopyable and nonmoveable - basic_bson_encoder(const basic_bson_encoder&) = delete; - basic_bson_encoder& operator=(const basic_bson_encoder&) = delete; -public: - explicit basic_bson_encoder(result_type result) - : result_(std::move(result)) - { - } - - ~basic_bson_encoder() - { - try - { - result_.flush(); - } - catch (...) - { - } - } - -private: - // Implementing methods - - void do_flush() override - { - result_.flush(); - } - - bool do_begin_object(semantic_tag, const ser_context&) override - { - if (buffer_.size() > 0) - { - before_value(jsoncons::bson::detail::bson_format::document_cd); - } - stack_.emplace_back(jsoncons::bson::detail::bson_container_type::document, buffer_.size()); - buffer_.insert(buffer_.end(), sizeof(int32_t), 0); - - return true; - } - - bool do_end_object(const ser_context&) override - { - JSONCONS_ASSERT(!stack_.empty()); - - buffer_.push_back(0x00); - - size_t length = buffer_.size() - stack_.back().offset(); - jsoncons::detail::to_little_endian(static_cast(length), buffer_.begin()+stack_.back().offset()); - - stack_.pop_back(); - if (stack_.empty()) - { - for (auto c : buffer_) - { - result_.push_back(c); - } - } - return true; - } - - bool do_begin_array(semantic_tag, const ser_context&) override - { - if (buffer_.size() > 0) - { - before_value(jsoncons::bson::detail::bson_format::array_cd); - } - stack_.emplace_back(jsoncons::bson::detail::bson_container_type::array, buffer_.size()); - buffer_.insert(buffer_.end(), sizeof(int32_t), 0); - return true; - } - - bool do_end_array(const ser_context&) override - { - JSONCONS_ASSERT(!stack_.empty()); - - buffer_.push_back(0x00); - - size_t length = buffer_.size() - stack_.back().offset(); - jsoncons::detail::to_little_endian(static_cast(length), buffer_.begin()+stack_.back().offset()); - - stack_.pop_back(); - if (stack_.empty()) - { - for (auto c : buffer_) - { - result_.push_back(c); - } - } - return true; - } - - bool do_name(const string_view_type& name, const ser_context&) override - { - stack_.back().member_offset(buffer_.size()); - buffer_.push_back(0x00); // reserve space for code - for (auto c : name) - { - buffer_.push_back(c); - } - buffer_.push_back(0x00); - return true; - } - - bool do_null_value(semantic_tag, const ser_context&) override - { - before_value(jsoncons::bson::detail::bson_format::null_cd); - return true; - } - - bool do_bool_value(bool val, semantic_tag, const ser_context&) override - { - before_value(jsoncons::bson::detail::bson_format::bool_cd); - if (val) - { - buffer_.push_back(0x01); - } - else - { - buffer_.push_back(0x00); - } - - return true; - } - - bool do_string_value(const string_view_type& sv, semantic_tag, const ser_context&) override - { - before_value(jsoncons::bson::detail::bson_format::string_cd); - - size_t offset = buffer_.size(); - buffer_.insert(buffer_.end(), sizeof(int32_t), 0); - size_t string_offset = buffer_.size(); - - auto result = unicons::validate(sv.begin(), sv.end()); - if (result.ec != unicons::conv_errc()) - { - JSONCONS_THROW(json_runtime_error("Illegal unicode")); - } - for (auto c : sv) - { - buffer_.push_back(c); - } - buffer_.push_back(0x00); - size_t length = buffer_.size() - string_offset; - jsoncons::detail::to_little_endian(static_cast(length), buffer_.begin()+offset); - - return true; - } - - bool do_byte_string_value(const byte_string_view& b, - semantic_tag, - const ser_context&) override - { - before_value(jsoncons::bson::detail::bson_format::binary_cd); - - size_t offset = buffer_.size(); - buffer_.insert(buffer_.end(), sizeof(int32_t), 0); - size_t string_offset = buffer_.size(); - - for (auto c : b) - { - buffer_.push_back(c); - } - size_t length = buffer_.size() - string_offset; - jsoncons::detail::to_little_endian(static_cast(length), buffer_.begin()+offset); - - return true; - } - - bool do_int64_value(int64_t val, - semantic_tag tag, - const ser_context&) override - { - if (tag == semantic_tag::timestamp) - { - before_value(jsoncons::bson::detail::bson_format::datetime_cd); - } - else - { - before_value(jsoncons::bson::detail::bson_format::int64_cd); - } - if (val >= (std::numeric_limits::lowest)() && val <= (std::numeric_limits::max)()) - { - jsoncons::detail::to_little_endian(static_cast(val),std::back_inserter(buffer_)); - } - else if (val >= (std::numeric_limits::lowest)() && val <= (std::numeric_limits::max)()) - { - jsoncons::detail::to_little_endian(static_cast(val),std::back_inserter(buffer_)); - } - else - { - // error - } - - return true; - } - - bool do_uint64_value(uint64_t val, - semantic_tag tag, - const ser_context&) override - { - if (tag == semantic_tag::timestamp) - { - before_value(jsoncons::bson::detail::bson_format::datetime_cd); - } - else - { - before_value(jsoncons::bson::detail::bson_format::int64_cd); - } - if (val <= (std::numeric_limits::max)()) - { - jsoncons::detail::to_little_endian(static_cast(val),std::back_inserter(buffer_)); - } - else if (val <= (uint64_t)(std::numeric_limits::max)()) - { - jsoncons::detail::to_little_endian(static_cast(val),std::back_inserter(buffer_)); - } - else - { - // error - } - - return true; - } - - bool do_double_value(double val, - semantic_tag, - const ser_context&) override - { - before_value(jsoncons::bson::detail::bson_format::double_cd); - - jsoncons::detail::to_little_endian(val,std::back_inserter(buffer_)); - - return true; - } - - void before_value(uint8_t code) - { - if (stack_.back().is_object()) - { - buffer_[stack_.back().member_offset()] = code; - } - else - { - buffer_.push_back(code); - std::string name = std::to_string(stack_.back().next_index()); - buffer_.insert(buffer_.end(), name.begin(), name.end()); - buffer_.push_back(0x00); - } - } -}; - -typedef basic_bson_encoder bson_encoder; -typedef basic_bson_encoder bson_bytes_encoder; - -#if !defined(JSONCONS_NO_DEPRECATED) -template -using basic_bson_serializer = basic_bson_encoder; - -typedef basic_bson_encoder bson_serializer; -typedef basic_bson_encoder bson_buffer_serializer; - -#endif - -}} -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson_error.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson_error.hpp deleted file mode 100644 index 47204f67e9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson_error.hpp +++ /dev/null @@ -1,70 +0,0 @@ -/// Copyright 2018 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_BSON_BSON_ERROR_HPP -#define JSONCONS_BSON_BSON_ERROR_HPP - -#include -#include - -namespace jsoncons { namespace bson { - -enum class bson_errc -{ - ok = 0, - unexpected_eof = 1, - source_error, - invalid_utf8_text_string -}; - -class bson_error_category_impl - : public std::error_category -{ -public: - const char* name() const noexcept override - { - return "jsoncons/bson"; - } - std::string message(int ev) const override - { - switch (static_cast(ev)) - { - case bson_errc::unexpected_eof: - return "Unexpected end of file"; - case bson_errc::source_error: - return "Source error"; - case bson_errc::invalid_utf8_text_string: - return "Illegal UTF-8 encoding in text string"; - default: - return "Unknown BSON parser error"; - } - } -}; - -inline -const std::error_category& bson_error_category() -{ - static bson_error_category_impl instance; - return instance; -} - -inline -std::error_code make_error_code(bson_errc result) -{ - return std::error_code(static_cast(result),bson_error_category()); -} - - -}} - -namespace std { - template<> - struct is_error_code_enum : public true_type - { - }; -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson_reader.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson_reader.hpp deleted file mode 100644 index f0d3d1b458..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/bson/bson_reader.hpp +++ /dev/null @@ -1,297 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_BSON_BSON_READER_HPP -#define JSONCONS_BSON_BSON_READER_HPP - -#include -#include -#include -#include // std::move -#include -#include -#include -#include -#include -#include - -namespace jsoncons { namespace bson { - -template -class basic_bson_reader : public ser_context -{ - Src source_; - json_content_handler& handler_; - size_t nesting_depth_; -public: - template - basic_bson_reader(Source&& source, json_content_handler& handler) - : source_(std::forward(source)), - handler_(handler), - nesting_depth_(0) - { - } - - void read() - { - std::error_code ec; - read(ec); - if (ec) - { - throw ser_error(ec,line(),column()); - } - } - - void read(std::error_code& ec) - { - try - { - if (source_.is_error()) - { - ec = bson_errc::source_error; - return; - } - uint8_t buf[sizeof(int32_t)]; - if (source_.read(buf, sizeof(int32_t)) != sizeof(int32_t)) - { - ec = bson_errc::unexpected_eof; - return; - } - const uint8_t* endp; - /* auto len = */jsoncons::detail::from_little_endian(buf, buf+sizeof(int32_t),&endp); - - handler_.begin_object(semantic_tag::none, *this); - ++nesting_depth_; - read_e_list(jsoncons::bson::detail::bson_container_type::document, ec); - handler_.end_object(*this); - --nesting_depth_; - } - catch (const ser_error& e) - { - ec = e.code(); - } - } - - size_t line() const override - { - return 0; - } - - size_t column() const override - { - return source_.position(); - } -private: - - void read_e_list(jsoncons::bson::detail::bson_container_type type, std::error_code& ec) - { - uint8_t t{}; - while (source_.get(t) > 0 && t != 0x00) - { - std::basic_string s; - uint8_t c{}; - while (source_.get(c) > 0 && c != 0) - { - s.push_back(c); - } - - if (type == jsoncons::bson::detail::bson_container_type::document) - { - auto result = unicons::validate(s.begin(),s.end()); - if (result.ec != unicons::conv_errc()) - { - ec = bson_errc::invalid_utf8_text_string; - return; - } - handler_.name(basic_string_view(s.data(),s.length()), *this); - } - read_internal(t, ec); - } - } - - void read_internal(uint8_t type, std::error_code& ec) - { - switch (type) - { - case jsoncons::bson::detail::bson_format::double_cd: - { - uint8_t buf[sizeof(double)]; - if (source_.read(buf, sizeof(double)) != sizeof(double)) - { - ec = bson_errc::unexpected_eof; - return; - } - const uint8_t* endp; - double res = jsoncons::detail::from_little_endian(buf,buf+sizeof(buf),&endp); - handler_.double_value(res, semantic_tag::none, *this); - break; - } - case jsoncons::bson::detail::bson_format::string_cd: - { - uint8_t buf[sizeof(int32_t)]; - if (source_.read(buf, sizeof(int32_t)) != sizeof(int32_t)) - { - ec = bson_errc::unexpected_eof; - return; - } - const uint8_t* endp; - auto len = jsoncons::detail::from_little_endian(buf, buf+sizeof(buf),&endp); - - std::basic_string s; - s.reserve(len - 1); - if ((int32_t)source_.read(std::back_inserter(s), len-1) != len-1) - { - ec = bson_errc::unexpected_eof; - return; - } - uint8_t c{}; - source_.get(c); // discard 0 - auto result = unicons::validate(s.begin(),s.end()); - if (result.ec != unicons::conv_errc()) - { - ec = bson_errc::invalid_utf8_text_string; - return; - } - handler_.string_value(basic_string_view(s.data(),s.length()), semantic_tag::none, *this); - break; - } - case jsoncons::bson::detail::bson_format::document_cd: - { - read(ec); - if (ec) - { - return; - } - break; - } - - case jsoncons::bson::detail::bson_format::array_cd: - { - uint8_t buf[sizeof(int32_t)]; - if (source_.read(buf, sizeof(int32_t)) != sizeof(int32_t)) - { - ec = bson_errc::unexpected_eof; - return; - } - const uint8_t* endp; - /* auto len = */ jsoncons::detail::from_little_endian(buf, buf+sizeof(int32_t),&endp); - - handler_.begin_array(semantic_tag::none, *this); - ++nesting_depth_; - read_e_list(jsoncons::bson::detail::bson_container_type::document, ec); - handler_.end_array(*this); - --nesting_depth_; - break; - } - case jsoncons::bson::detail::bson_format::null_cd: - { - handler_.null_value(semantic_tag::none, *this); - break; - } - case jsoncons::bson::detail::bson_format::bool_cd: - { - uint8_t val{}; - if (source_.get(val) == 0) - { - ec = bson_errc::unexpected_eof; - return; - } - handler_.bool_value(val != 0, semantic_tag::none, *this); - break; - } - case jsoncons::bson::detail::bson_format::int32_cd: - { - uint8_t buf[sizeof(int32_t)]; - if (source_.read(buf, sizeof(int32_t)) != sizeof(int32_t)) - { - ec = bson_errc::unexpected_eof; - return; - } - const uint8_t* endp; - auto val = jsoncons::detail::from_little_endian(buf, buf+sizeof(int32_t),&endp); - handler_.int64_value(val, semantic_tag::none, *this); - break; - } - - case jsoncons::bson::detail::bson_format::timestamp_cd: - { - uint8_t buf[sizeof(uint64_t)]; - if (source_.read(buf, sizeof(uint64_t)) != sizeof(uint64_t)) - { - ec = bson_errc::unexpected_eof; - return; - } - const uint8_t* endp; - auto val = jsoncons::detail::from_little_endian(buf, buf+sizeof(uint64_t),&endp); - handler_.uint64_value(val, semantic_tag::timestamp, *this); - break; - } - - case jsoncons::bson::detail::bson_format::int64_cd: - { - uint8_t buf[sizeof(int64_t)]; - if (source_.read(buf, sizeof(int64_t)) != sizeof(int64_t)) - { - ec = bson_errc::unexpected_eof; - return; - } - const uint8_t* endp; - auto val = jsoncons::detail::from_little_endian(buf, buf+sizeof(int64_t),&endp); - handler_.int64_value(val, semantic_tag::none, *this); - break; - } - - case jsoncons::bson::detail::bson_format::datetime_cd: - { - uint8_t buf[sizeof(int64_t)]; - if (source_.read(buf, sizeof(int64_t)) != sizeof(int64_t)) - { - ec = bson_errc::unexpected_eof; - return; - } - const uint8_t* endp; - auto val = jsoncons::detail::from_little_endian(buf, buf+sizeof(int64_t),&endp); - handler_.int64_value(val, semantic_tag::timestamp, *this); - break; - } - case jsoncons::bson::detail::bson_format::binary_cd: - { - uint8_t buf[sizeof(int32_t)]; - if (source_.read(buf, sizeof(int32_t)) != sizeof(int32_t)) - { - ec = bson_errc::unexpected_eof; - return; - } - const uint8_t* endp; - const auto len = jsoncons::detail::from_little_endian(buf, buf+sizeof(int32_t),&endp); - - std::vector v(len, 0); - if (source_.read(v.data(), v.size()) != v.size()) - { - ec = bson_errc::unexpected_eof; - return; - } - - handler_.byte_string_value(byte_string_view(v.data(),v.size()), - semantic_tag::none, - *this); - break; - } - } - - } -}; - -typedef basic_bson_reader bson_reader; -typedef basic_bson_reader bson_bytes_reader; - -#if !defined(JSONCONS_NO_DEPRECATED) -typedef bson_bytes_reader bson_buffer_reader; -#endif - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor.hpp deleted file mode 100644 index c05b3c2439..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor.hpp +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_CBOR_CBOR_HPP -#define JSONCONS_CBOR_CBOR_HPP - -#include -#include -#include -#include // std::enable_if -#include // std::basic_istream -#include -#include -#include -#include -#include - -namespace jsoncons { namespace cbor { - -// encode_cbor - -template -void encode_cbor(const T& j, std::vector& v) -{ - encode_cbor(j,v,cbor_options::default_options()); -} - -template -void encode_cbor(const T& j, std::ostream& os) -{ - encode_cbor(j,os,cbor_options::default_options()); -} - -template -typename std::enable_if::value,void>::type -encode_cbor(const T& j, std::vector& v, const cbor_encode_options& options) -{ - typedef typename T::char_type char_type; - cbor_bytes_encoder encoder(v, options); - auto adaptor = make_json_content_handler_adaptor>(encoder); - j.dump(adaptor); -} - -template -typename std::enable_if::value,void>::type -encode_cbor(const T& val, std::vector& v, const cbor_encode_options& options) -{ - cbor_bytes_encoder encoder(v, options); - write_to(json(), val, encoder); -} - -template -typename std::enable_if::value,void>::type -encode_cbor(const T& j, std::ostream& os, const cbor_encode_options& options) -{ - typedef typename T::char_type char_type; - cbor_encoder encoder(os, options); - auto adaptor = make_json_content_handler_adaptor>(encoder); - j.dump(adaptor); -} - -template -typename std::enable_if::value,void>::type -encode_cbor(const T& val, std::ostream& os, const cbor_encode_options& options) -{ - cbor_encoder encoder(os, options); - write_to(json(), val, encoder); -} - -// decode_cbor - -template -typename std::enable_if::value,T>::type -decode_cbor(const std::vector& v) -{ - jsoncons::json_decoder decoder; - auto adaptor = make_json_content_handler_adaptor(decoder); - basic_cbor_reader reader(v, adaptor); - reader.read(); - return decoder.get_result(); -} - -template -typename std::enable_if::value,T>::type -decode_cbor(const std::vector& v) -{ - jsoncons::json_decoder decoder; - basic_cbor_reader reader(v, decoder); - reader.read(); - return decoder.get_result().template as(); -} - -template -typename std::enable_if::value,T>::type -decode_cbor(std::istream& is) -{ - jsoncons::json_decoder decoder; - auto adaptor = make_json_content_handler_adaptor(decoder); - cbor_reader reader(is, adaptor); - reader.read(); - return decoder.get_result(); -} - -template -typename std::enable_if::value,T>::type -decode_cbor(std::istream& is) -{ - jsoncons::json_decoder decoder; - cbor_reader reader(is, decoder); - reader.read(); - return decoder.get_result().template as(); -} - - -#if !defined(JSONCONS_NO_DEPRECATED) -template -std::vector encode_cbor(const Json& j) -{ - std::vector v; - encode_cbor(j, v); - return v; -} -#endif - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_detail.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_detail.hpp deleted file mode 100644 index d59397bd27..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_detail.hpp +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_CBOR_CBOR_DETAIL_HPP -#define JSONCONS_CBOR_CBOR_DETAIL_HPP - -#include -#include -#include -#include // std::forward_iterator_tag -#include // std::numeric_limits -#include // std::move -#include -#include -#include - -namespace jsoncons { namespace cbor { namespace detail { - -// 0x00..0x17 (0..23) -#define JSONCONS_CBOR_0x00_0x17 \ - 0x00:case 0x01:case 0x02:case 0x03:case 0x04:case 0x05:case 0x06:case 0x07:case 0x08:case 0x09:case 0x0a:case 0x0b:case 0x0c:case 0x0d:case 0x0e:case 0x0f:case 0x10:case 0x11:case 0x12:case 0x13:case 0x14:case 0x15:case 0x16:case 0x17 - -enum class cbor_major_type : uint8_t -{ - unsigned_integer = 0x00, - negative_integer = 0x01, - byte_string = 0x02, - text_string = 0x03, - array = 0x04, - map = 0x05, - semantic_tag = 0x06, - simple = 0x7 -}; - -namespace additional_info -{ - const uint8_t indefinite_length = 0x1f; -} - -inline -size_t min_length_for_stringref(uint64_t index) -{ - size_t n; - if (index <= 23) - { - n = 3; - } - else if (index <= 255) - { - n = 4; - } - else if (index <= 65535) - { - n = 5; - } - else if (index <= 4294967295) - { - n = 7; - } - else - { - n = 11; - } - return n; -} - -}}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_encoder.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_encoder.hpp deleted file mode 100644 index 6fb3ad185a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_encoder.hpp +++ /dev/null @@ -1,1062 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_CBOR_CBOR_ENCODER_HPP -#define JSONCONS_CBOR_CBOR_ENCODER_HPP - -#include -#include -#include // std::numeric_limits -#include -#include // std::move -#include // jsoncons::ser_error -#include -#include -#include -#include -#include -#include - -namespace jsoncons { namespace cbor { - -enum class cbor_container_type {object, indefinite_length_object, array, indefinite_length_array}; - -template -class basic_cbor_encoder final : public basic_json_content_handler -{ - enum class decimal_parse_state { start, integer, exp1, exp2, fraction1 }; - enum class hexfloat_parse_state { start, expect_0, expect_x, integer, exp1, exp2, fraction1 }; - -public: - typedef char char_type; - typedef Result result_type; - using typename basic_json_content_handler::string_view_type; - -private: - struct stack_item - { - cbor_container_type type_; - size_t length_; - size_t count_; - - stack_item(cbor_container_type type, size_t length = 0) - : type_(type), length_(length), count_(0) - { - } - - size_t length() const - { - return length_; - } - - size_t count() const - { - return count_; - } - - bool is_object() const - { - return type_ == cbor_container_type::object || type_ == cbor_container_type::indefinite_length_object; - } - - bool is_indefinite_length() const - { - return type_ == cbor_container_type::indefinite_length_array || type_ == cbor_container_type::indefinite_length_object; - } - - }; - std::vector stack_; - Result result_; - const cbor_encode_options& options_; - - // Noncopyable and nonmoveable - basic_cbor_encoder(const basic_cbor_encoder&) = delete; - basic_cbor_encoder& operator=(const basic_cbor_encoder&) = delete; - - std::map stringref_map_; - std::map bytestringref_map_; - size_t next_stringref_ = 0; -public: - explicit basic_cbor_encoder(result_type result) - : result_(std::move(result)), options_(cbor_options::default_options()) - { - } - basic_cbor_encoder(result_type result, const cbor_encode_options& options) - : result_(std::move(result)), options_(options) - { - if (options.pack_strings()) - { - // tag(256) - result_.push_back(0xd9); - result_.push_back(0x01); - result_.push_back(0x00); - } - } - - ~basic_cbor_encoder() - { - try - { - result_.flush(); - } - catch (...) - { - } - } - -private: - // Implementing methods - - void do_flush() override - { - result_.flush(); - } - - bool do_begin_object(semantic_tag, const ser_context&) override - { - stack_.push_back(stack_item(cbor_container_type::indefinite_length_object)); - - result_.push_back(0xbf); - return true; - } - - bool do_begin_object(size_t length, semantic_tag, const ser_context&) override - { - stack_.push_back(stack_item(cbor_container_type::object, length)); - - if (length <= 0x17) - { - jsoncons::detail::to_big_endian(static_cast(0xa0 + length), - std::back_inserter(result_)); - } - else if (length <= 0xff) - { - jsoncons::detail::to_big_endian(static_cast(0xb8), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), - std::back_inserter(result_)); - } - else if (length <= 0xffff) - { - jsoncons::detail::to_big_endian(static_cast(0xb9), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), - std::back_inserter(result_)); - } - else if (length <= 0xffffffff) - { - jsoncons::detail::to_big_endian(static_cast(0xba), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), - std::back_inserter(result_)); - } - else if (length <= 0xffffffffffffffff) - { - jsoncons::detail::to_big_endian(static_cast(0xbb), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), - std::back_inserter(result_)); - } - - return true; - } - - bool do_end_object(const ser_context&) override - { - JSONCONS_ASSERT(!stack_.empty()); - if (stack_.back().is_indefinite_length()) - { - result_.push_back(0xff); - } - else - { - if (stack_.back().count() < stack_.back().length()) - { - throw ser_error(cbor_errc::too_few_items); - } - if (stack_.back().count() > stack_.back().length()) - { - throw ser_error(cbor_errc::too_many_items); - } - } - - stack_.pop_back(); - end_value(); - - return true; - } - - bool do_begin_array(semantic_tag, const ser_context&) override - { - stack_.push_back(stack_item(cbor_container_type::indefinite_length_array)); - result_.push_back(0x9f); - return true; - } - - bool do_begin_array(size_t length, semantic_tag tag, const ser_context&) override - { -#if !defined(JSONCONS_NO_DEPRECATED) - if (length == 2 && tag == semantic_tag::bigfloat) - { - result_.push_back(0xc5); - } -#endif - stack_.push_back(stack_item(cbor_container_type::array, length)); - if (length <= 0x17) - { - jsoncons::detail::to_big_endian(static_cast(0x80 + length), - std::back_inserter(result_)); - } - else if (length <= 0xff) - { - jsoncons::detail::to_big_endian(static_cast(0x98), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), - std::back_inserter(result_)); - } - else if (length <= 0xffff) - { - jsoncons::detail::to_big_endian(static_cast(0x99), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), - std::back_inserter(result_)); - } - else if (length <= 0xffffffff) - { - jsoncons::detail::to_big_endian(static_cast(0x9a), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), - std::back_inserter(result_)); - } - else if (length <= 0xffffffffffffffff) - { - jsoncons::detail::to_big_endian(static_cast(0x9b), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), - std::back_inserter(result_)); - } - return true; - } - - bool do_end_array(const ser_context&) override - { - JSONCONS_ASSERT(!stack_.empty()); - - if (stack_.back().is_indefinite_length()) - { - result_.push_back(0xff); - } - else - { - if (stack_.back().count() < stack_.back().length()) - { - throw ser_error(cbor_errc::too_few_items); - } - if (stack_.back().count() > stack_.back().length()) - { - throw ser_error(cbor_errc::too_many_items); - } - } - - stack_.pop_back(); - end_value(); - - return true; - } - - bool do_name(const string_view_type& name, const ser_context&) override - { - write_string(name); - return true; - } - - bool do_null_value(semantic_tag tag, const ser_context&) override - { - if (tag == semantic_tag::undefined) - { - result_.push_back(0xf7); - } - else - { - result_.push_back(0xf6); - } - - end_value(); - return true; - } - - void write_string(const string_view& sv) - { - auto result = unicons::validate(sv.begin(), sv.end()); - if (result.ec != unicons::conv_errc()) - { - JSONCONS_THROW(json_runtime_error("Illegal unicode")); - } - - if (options_.pack_strings() && sv.size() >= jsoncons::cbor::detail::min_length_for_stringref(next_stringref_)) - { - std::string s(sv); - auto it = stringref_map_.find(s); - if (it == stringref_map_.end()) - { - stringref_map_.insert(std::make_pair(std::move(s), next_stringref_++)); - write_utf8_string(sv); - } - else - { - // tag(25) - result_.push_back(0xd8); - result_.push_back(0x19); - write_uint64_value(it->second); - } - } - else - { - write_utf8_string(sv); - } - } - - void write_utf8_string(const string_view& sv) - { - const size_t length = sv.size(); - - if (length <= 0x17) - { - // fixstr stores a byte array whose length is upto 31 bytes - jsoncons::detail::to_big_endian(static_cast(0x60 + length), - std::back_inserter(result_)); - } - else if (length <= 0xff) - { - jsoncons::detail::to_big_endian(static_cast(0x78), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), - std::back_inserter(result_)); - } - else if (length <= 0xffff) - { - jsoncons::detail::to_big_endian(static_cast(0x79), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), - std::back_inserter(result_)); - } - else if (length <= 0xffffffff) - { - jsoncons::detail::to_big_endian(static_cast(0x7a), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), - std::back_inserter(result_)); - } - else if (length <= 0xffffffffffffffff) - { - jsoncons::detail::to_big_endian(static_cast(0x7b), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), - std::back_inserter(result_)); - } - - for (auto c : sv) - { - result_.push_back(c); - } - } - - void write_bignum(const bignum& n) - { - int signum; - std::vector data; - n.dump(signum, data); - size_t length = data.size(); - - if (signum == -1) - { - result_.push_back(0xc3); - } - else - { - result_.push_back(0xc2); - } - - if (length <= 0x17) - { - // fixstr stores a byte array whose length is upto 31 bytes - jsoncons::detail::to_big_endian(static_cast(0x40 + length), - std::back_inserter(result_)); - } - else if (length <= 0xff) - { - jsoncons::detail::to_big_endian(static_cast(0x58), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), - std::back_inserter(result_)); - } - else if (length <= 0xffff) - { - jsoncons::detail::to_big_endian(static_cast(0x59), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), - std::back_inserter(result_)); - } - else if (length <= 0xffffffff) - { - jsoncons::detail::to_big_endian(static_cast(0x5a), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), - std::back_inserter(result_)); - } - else if (length <= 0xffffffffffffffff) - { - jsoncons::detail::to_big_endian(static_cast(0x5b), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), - std::back_inserter(result_)); - } - - for (auto c : data) - { - result_.push_back(c); - } - } - - void write_decimal_value(const string_view_type& sv, const ser_context& context) - { - decimal_parse_state state = decimal_parse_state::start; - std::basic_string s; - std::basic_string exponent; - int64_t scale = 0; - for (auto c : sv) - { - switch (state) - { - case decimal_parse_state::start: - { - switch (c) - { - case '-': - s.push_back(c); - state = decimal_parse_state::integer; - break; - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - s.push_back(c); - state = decimal_parse_state::integer; - break; - default: - throw std::invalid_argument("Invalid decimal string"); - } - break; - } - case decimal_parse_state::integer: - { - switch (c) - { - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - s.push_back(c); - state = decimal_parse_state::integer; - break; - case 'e': case 'E': - state = decimal_parse_state::exp1; - break; - case '.': - state = decimal_parse_state::fraction1; - break; - default: - throw std::invalid_argument("Invalid decimal string"); - } - break; - } - case decimal_parse_state::exp1: - { - switch (c) - { - case '+': - state = decimal_parse_state::exp2; - break; - case '-': - exponent.push_back(c); - state = decimal_parse_state::exp2; - break; - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - exponent.push_back(c); - state = decimal_parse_state::exp2; - break; - default: - throw std::invalid_argument("Invalid decimal string"); - } - break; - } - case decimal_parse_state::exp2: - { - switch (c) - { - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - exponent.push_back(c); - break; - default: - throw std::invalid_argument("Invalid decimal string"); - } - break; - } - case decimal_parse_state::fraction1: - { - switch (c) - { - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9': - s.push_back(c); - --scale; - break; - default: - throw std::invalid_argument("Invalid decimal string"); - } - break; - } - } - } - - result_.push_back(0xc4); - do_begin_array((size_t)2, semantic_tag::none, context); - if (exponent.length() > 0) - { - auto result = jsoncons::detail::to_integer(exponent.data(), exponent.length()); - if (result.ec != jsoncons::detail::to_integer_errc()) - { - JSONCONS_THROW(json_runtime_error(make_error_code(result.ec).message())); - } - scale += result.value; - } - do_int64_value(scale, semantic_tag::none, context); - - auto result = jsoncons::detail::to_integer(s.data(),s.length()); - if (result.ec == jsoncons::detail::to_integer_errc()) - { - do_int64_value(result.value, semantic_tag::none, context); - } - else if (result.ec == jsoncons::detail::to_integer_errc::overflow) - { - bignum n(s.data(), s.length()); - write_bignum(n); - end_value(); - } - else - { - JSONCONS_THROW(json_runtime_error(make_error_code(result.ec).message())); - } - do_end_array(context); - } - - void write_hexfloat_value(const string_view_type& sv, const ser_context& context) - { - hexfloat_parse_state state = hexfloat_parse_state::start; - std::basic_string s; - std::basic_string exponent; - int64_t scale = 0; - - for (auto c : sv) - { - switch (state) - { - case hexfloat_parse_state::start: - { - switch (c) - { - case '-': - s.push_back(c); - state = hexfloat_parse_state::expect_0; - break; - case '0': - state = hexfloat_parse_state::expect_x; - break; - default: - throw std::invalid_argument("Invalid hexfloat string"); - } - break; - } - case hexfloat_parse_state::expect_0: - { - switch (c) - { - case '0': - state = hexfloat_parse_state::expect_x; - break; - default: - throw std::invalid_argument("Invalid hexfloat string"); - } - break; - } - case hexfloat_parse_state::expect_x: - { - switch (c) - { - case 'x': - case 'X': - state = hexfloat_parse_state::integer; - break; - default: - throw std::invalid_argument("Invalid hexfloat string"); - } - break; - } - case hexfloat_parse_state::integer: - { - switch (c) - { - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9':case 'a':case 'b':case 'c':case 'd':case 'e':case 'f':case 'A':case 'B':case 'C':case 'D':case 'E':case 'F': - s.push_back(c); - state = hexfloat_parse_state::integer; - break; - case 'p': case 'P': - state = hexfloat_parse_state::exp1; - break; - case '.': - state = hexfloat_parse_state::fraction1; - break; - default: - throw std::invalid_argument("Invalid hexfloat string"); - } - break; - } - case hexfloat_parse_state::exp1: - { - switch (c) - { - case '+': - state = hexfloat_parse_state::exp2; - break; - case '-': - exponent.push_back(c); - state = hexfloat_parse_state::exp2; - break; - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9':case 'a':case 'b':case 'c':case 'd':case 'e':case 'f':case 'A':case 'B':case 'C':case 'D':case 'E':case 'F': - exponent.push_back(c); - state = hexfloat_parse_state::exp2; - break; - default: - throw std::invalid_argument("Invalid hexfloat string"); - } - break; - } - case hexfloat_parse_state::exp2: - { - switch (c) - { - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9':case 'a':case 'b':case 'c':case 'd':case 'e':case 'f':case 'A':case 'B':case 'C':case 'D':case 'E':case 'F': - exponent.push_back(c); - break; - default: - throw std::invalid_argument("Invalid hexfloat string"); - } - break; - } - case hexfloat_parse_state::fraction1: - { - switch (c) - { - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8': case '9':case 'a':case 'b':case 'c':case 'd':case 'e':case 'f':case 'A':case 'B':case 'C':case 'D':case 'E':case 'F': - s.push_back(c); - scale -= 4; - break; - default: - throw std::invalid_argument("Invalid hexfloat string"); - } - break; - } - } - } - - result_.push_back(0xc5); - do_begin_array((size_t)2, semantic_tag::none, context); - if (exponent.length() > 0) - { - auto result = jsoncons::detail::base16_to_integer(exponent.data(), exponent.length()); - if (result.ec != jsoncons::detail::to_integer_errc()) - { - JSONCONS_THROW(json_runtime_error(make_error_code(result.ec).message())); - } - scale += result.value; - } - do_int64_value(scale, semantic_tag::none, context); - - auto result = jsoncons::detail::base16_to_integer(s.data(),s.length()); - if (result.ec == jsoncons::detail::to_integer_errc()) - { - do_int64_value(result.value, semantic_tag::none, context); - } - else if (result.ec == jsoncons::detail::to_integer_errc::overflow) - { - bignum n(s.data(), s.length(), 16); - write_bignum(n); - end_value(); - } - else - { - JSONCONS_THROW(json_runtime_error(make_error_code(result.ec).message())); - } - do_end_array(context); - } - - bool do_string_value(const string_view_type& sv, semantic_tag tag, const ser_context& context) override - { - switch (tag) - { - case semantic_tag::bigint: - { - bignum n(sv.data(), sv.length()); - write_bignum(n); - end_value(); - break; - } - case semantic_tag::bigdec: - { - write_decimal_value(sv, context); - break; - } - case semantic_tag::bigfloat: - { - write_hexfloat_value(sv, context); - break; - } - case semantic_tag::datetime: - { - result_.push_back(0xc0); - write_string(sv); - end_value(); - break; - } - case semantic_tag::uri: - { - result_.push_back(32); - write_string(sv); - end_value(); - break; - } - case semantic_tag::base64url: - { - result_.push_back(33); - write_string(sv); - end_value(); - break; - } - case semantic_tag::base64: - { - result_.push_back(34); - write_string(sv); - end_value(); - break; - } - default: - { - write_string(sv); - end_value(); - break; - } - } - return true; - } - - bool do_byte_string_value(const byte_string_view& b, - semantic_tag tag, - const ser_context&) override - { - byte_string_chars_format encoding_hint; - switch (tag) - { - case semantic_tag::base16: - encoding_hint = byte_string_chars_format::base16; - break; - case semantic_tag::base64: - encoding_hint = byte_string_chars_format::base64; - break; - case semantic_tag::base64url: - encoding_hint = byte_string_chars_format::base64url; - break; - default: - encoding_hint = byte_string_chars_format::none; - break; - } - switch (encoding_hint) - { - case byte_string_chars_format::base64url: - result_.push_back(0xd5); - break; - case byte_string_chars_format::base64: - result_.push_back(0xd6); - break; - case byte_string_chars_format::base16: - result_.push_back(0xd7); - break; - default: - break; - } - if (options_.pack_strings() && b.length() >= jsoncons::cbor::detail::min_length_for_stringref(next_stringref_)) - { - auto it = bytestringref_map_.find(byte_string(b)); - if (it == bytestringref_map_.end()) - { - bytestringref_map_.insert(std::make_pair(byte_string(b), next_stringref_++)); - write_byte_string_value(b); - } - else - { - // tag(25) - result_.push_back(0xd8); - result_.push_back(0x19); - write_uint64_value(it->second); - } - } - else - { - write_byte_string_value(b); - } - - end_value(); - return true; - } - - void write_byte_string_value(const byte_string_view& b) - { - if (b.length() <= 0x17) - { - // fixstr stores a byte array whose length is upto 31 bytes - jsoncons::detail::to_big_endian(static_cast(0x40 + b.length()), - std::back_inserter(result_)); - } - else if (b.length() <= 0xff) - { - jsoncons::detail::to_big_endian(static_cast(0x58), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(b.length()), - std::back_inserter(result_)); - } - else if (b.length() <= 0xffff) - { - jsoncons::detail::to_big_endian(static_cast(0x59), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(b.length()), - std::back_inserter(result_)); - } - else if (b.length() <= 0xffffffff) - { - jsoncons::detail::to_big_endian(static_cast(0x5a), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(b.length()), - std::back_inserter(result_)); - } - else if (b.length() <= 0xffffffffffffffff) - { - jsoncons::detail::to_big_endian(static_cast(0x5b), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(b.length()), - std::back_inserter(result_)); - } - - for (auto c : b) - { - result_.push_back(c); - } - } - - bool do_double_value(double val, - semantic_tag tag, - const ser_context&) override - { - if (tag == semantic_tag::timestamp) - { - result_.push_back(0xc1); - } - - float valf = (float)val; - if ((double)valf == val) - { - jsoncons::detail::to_big_endian(static_cast(0xfa), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(valf, std::back_inserter(result_)); - } - else - { - jsoncons::detail::to_big_endian(static_cast(0xfb), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(val, std::back_inserter(result_)); - } - - // write double - - end_value(); - return true; - } - - bool do_int64_value(int64_t value, - semantic_tag tag, - const ser_context&) override - { - if (tag == semantic_tag::timestamp) - { - result_.push_back(0xc1); - } - if (value >= 0) - { - if (value <= 0x17) - { - jsoncons::detail::to_big_endian(static_cast(value), - std::back_inserter(result_)); - } - else if (value <= (std::numeric_limits::max)()) - { - jsoncons::detail::to_big_endian(static_cast(0x18), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(value), - std::back_inserter(result_)); - } - else if (value <= (std::numeric_limits::max)()) - { - jsoncons::detail::to_big_endian(static_cast(0x19), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(value), - std::back_inserter(result_)); - } - else if (value <= (std::numeric_limits::max)()) - { - jsoncons::detail::to_big_endian(static_cast(0x1a), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(value), - std::back_inserter(result_)); - } - else if (value <= (std::numeric_limits::max)()) - { - jsoncons::detail::to_big_endian(static_cast(0x1b), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(value), - std::back_inserter(result_)); - } - } else - { - const auto posnum = -1 - value; - if (value >= -24) - { - jsoncons::detail::to_big_endian(static_cast(0x20 + posnum), - std::back_inserter(result_)); - } - else if (posnum <= (std::numeric_limits::max)()) - { - jsoncons::detail::to_big_endian(static_cast(0x38), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(posnum), - std::back_inserter(result_)); - } - else if (posnum <= (std::numeric_limits::max)()) - { - jsoncons::detail::to_big_endian(static_cast(0x39), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(posnum), - std::back_inserter(result_)); - } - else if (posnum <= (std::numeric_limits::max)()) - { - jsoncons::detail::to_big_endian(static_cast(0x3a), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(posnum), - std::back_inserter(result_)); - } - else if (posnum <= (std::numeric_limits::max)()) - { - jsoncons::detail::to_big_endian(static_cast(0x3b), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(posnum), - std::back_inserter(result_)); - } - } - end_value(); - return true; - } - - bool do_uint64_value(uint64_t value, - semantic_tag tag, - const ser_context&) override - { - if (tag == semantic_tag::timestamp) - { - result_.push_back(0xc1); - } - - write_uint64_value(value); - end_value(); - return true; - } - - void write_uint64_value(uint64_t value) - { - if (value <= 0x17) - { - jsoncons::detail::to_big_endian(static_cast(value), - std::back_inserter(result_)); - } - else if (value <=(std::numeric_limits::max)()) - { - jsoncons::detail::to_big_endian(static_cast(0x18), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(value), - std::back_inserter(result_)); - } - else if (value <=(std::numeric_limits::max)()) - { - jsoncons::detail::to_big_endian(static_cast(0x19), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(value), - std::back_inserter(result_)); - } - else if (value <=(std::numeric_limits::max)()) - { - jsoncons::detail::to_big_endian(static_cast(0x1a), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(value), - std::back_inserter(result_)); - } - else if (value <=(std::numeric_limits::max)()) - { - jsoncons::detail::to_big_endian(static_cast(0x1b), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(value), - std::back_inserter(result_)); - } - } - - bool do_bool_value(bool value, semantic_tag, const ser_context&) override - { - if (value) - { - result_.push_back(0xf5); - } - else - { - result_.push_back(0xf4); - } - - end_value(); - return true; - } - - void end_value() - { - if (!stack_.empty()) - { - ++stack_.back().count_; - } - } -}; - -typedef basic_cbor_encoder cbor_encoder; -typedef basic_cbor_encoder cbor_bytes_encoder; - -#if !defined(JSONCONS_NO_DEPRECATED) -typedef basic_cbor_encoder cbor_bytes_serializer; - -template -using basic_cbor_serializer = basic_cbor_encoder; - -typedef basic_cbor_encoder cbor_serializer; -typedef basic_cbor_encoder cbor_buffer_serializer; -#endif - -}} -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_error.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_error.hpp deleted file mode 100644 index 79c3de976f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_error.hpp +++ /dev/null @@ -1,96 +0,0 @@ -/// Copyright 2018 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_CBOR_CBOR_ERROR_HPP -#define JSONCONS_CBOR_CBOR_ERROR_HPP - -#include -#include -#include // jsoncons::ser_error - -namespace jsoncons { namespace cbor { - -enum class cbor_errc -{ - ok = 0, - unexpected_eof, - source_error, - invalid_decimal, - invalid_bigfloat, - invalid_utf8_text_string, - too_many_items, - too_few_items, - number_too_large, - stringref_too_large -}; - -class cbor_error_category_impl - : public std::error_category -{ -public: - const char* name() const noexcept override - { - return "jsoncons/cbor"; - } - std::string message(int ev) const override - { - switch (static_cast(ev)) - { - case cbor_errc::unexpected_eof: - return "Unexpected end of file"; - case cbor_errc::source_error: - return "Source error"; - case cbor_errc::invalid_decimal: - return "Invalid decimal"; - case cbor_errc::invalid_bigfloat: - return "Invalid bigfloat"; - case cbor_errc::invalid_utf8_text_string: - return "Illegal UTF-8 encoding in text string"; - case cbor_errc::too_many_items: - return "Too many items were added to a CBOR map or array of known length"; - case cbor_errc::too_few_items: - return "Too few items were added to a CBOR map or array of known length"; - case cbor_errc::number_too_large: - return "Number exceeds implementation limits"; - case cbor_errc::stringref_too_large: - return "stringref exceeds stringref map size"; - default: - return "Unknown CBOR parser error"; - } - } -}; - -inline -const std::error_category& cbor_error_category() -{ - static cbor_error_category_impl instance; - return instance; -} - -inline -std::error_code make_error_code(cbor_errc e) -{ - return std::error_code(static_cast(e),cbor_error_category()); -} - - -#if !defined(JSONCONS_NO_DEPRECATED) - -typedef ser_error cbor_error; -typedef ser_error cbor_decode_error; -typedef cbor_errc cbor_reader_errc; -#endif - -}} - -namespace std { - template<> - struct is_error_code_enum : public true_type - { - }; -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_options.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_options.hpp deleted file mode 100644 index cc780ca80a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_options.hpp +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2019 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_CBOR_CBOR_OPTIONS_HPP -#define JSONCONS_CBOR_CBOR_OPTIONS_HPP - -#include -#include // std::numeric_limits -#include -#include -#include - -namespace jsoncons { namespace cbor { - -class cbor_decode_options -{ -public: - virtual ~cbor_decode_options() = default; -}; - -class cbor_encode_options -{ -public: - virtual ~cbor_encode_options() = default; - - virtual bool pack_strings() const = 0; -}; - -class cbor_options : public virtual cbor_decode_options, - public virtual cbor_encode_options -{ -private: - bool pack_strings_; -public: - - static const cbor_options& default_options() - { - static cbor_options options{}; - return options; - } - - -// Constructors - - cbor_options() - : pack_strings_(false) - { - } - - bool pack_strings() const override - { - return pack_strings_; - } - - cbor_options& pack_strings(bool value) - { - pack_strings_ = value; - return *this; - } -}; - -}} -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_reader.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_reader.hpp deleted file mode 100644 index cb63bcafd1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/cbor/cbor_reader.hpp +++ /dev/null @@ -1,1491 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_CBOR_CBOR_READER_HPP -#define JSONCONS_CBOR_CBOR_READER_HPP - -#include -#include -#include -#include // std::move -#include -#include -#include -#include -#include -#include -#include - -namespace jsoncons { namespace cbor { - -enum class parse_mode {root,array,indefinite_array,map,indefinite_map}; - -struct mapped_string -{ - jsoncons::cbor::detail::cbor_major_type type; - std::string s; - std::vector bs; - - mapped_string(const std::string& s) - : type(jsoncons::cbor::detail::cbor_major_type::text_string), s(s) - { - } - - mapped_string(std::string&& s) - : type(jsoncons::cbor::detail::cbor_major_type::text_string), s(std::move(s)) - { - } - - mapped_string(const std::vector& bs) - : type(jsoncons::cbor::detail::cbor_major_type::byte_string), bs(bs) - { - } - - mapped_string(std::vector&& bs) - : type(jsoncons::cbor::detail::cbor_major_type::byte_string), bs(std::move(bs)) - { - } - - mapped_string(const mapped_string&) = default; - - mapped_string(mapped_string&&) = default; - - mapped_string& operator=(const mapped_string&) = default; - - mapped_string& operator=(mapped_string&&) = default; -}; - -typedef std::vector stringref_map_type; - -struct parse_state -{ - parse_mode mode; - size_t length; - size_t index; - std::shared_ptr stringref_map; - - parse_state(parse_mode mode, size_t length) - : mode(mode), length(length), index(0) - { - } - - parse_state(parse_mode mode, size_t length, std::shared_ptr stringref_map) - : mode(mode), length(length), index(0), stringref_map(stringref_map) - { - } - - parse_state(const parse_state&) = default; - parse_state(parse_state&&) = default; -}; - -template -class basic_cbor_reader : public ser_context -{ - Src source_; - json_content_handler& handler_; - std::string buffer_; - std::vector tags_; - std::vector state_stack_; -public: - template - basic_cbor_reader(Source&& source, json_content_handler& handler) - : source_(std::forward(source)), - handler_(handler) - { - } - - void read() - { - std::error_code ec; - read(ec); - if (ec) - { - throw ser_error(ec,line(),column()); - } - } - - void read(std::error_code& ec) - { - if (source_.is_error()) - { - ec = cbor_errc::source_error; - return; - } - try - { - state_stack_.emplace_back(parse_mode::root,0); - read_internal(ec); - if (ec) - { - return; - } - } - catch (const ser_error& e) - { - ec = e.code(); - } - } - - size_t line() const override - { - return 0; - } - - size_t column() const override - { - return source_.position(); - } -private: - - void read_internal(std::error_code& ec) - { - while (!state_stack_.empty()) - { - switch (state_stack_.back().mode) - { - case parse_mode::array: - { - if (state_stack_.back().index < state_stack_.back().length) - { - ++state_stack_.back().index; - read_item(ec); - if (ec) - { - return; - } - } - else - { - end_array(ec); - } - break; - } - case parse_mode::indefinite_array: - { - int c = source_.peek(); - switch (c) - { - case Src::traits_type::eof(): - ec = cbor_errc::unexpected_eof; - return; - case 0xff: - end_array(ec); - if (ec) - { - return; - } - break; - default: - read_item(ec); - if (ec) - { - return; - } - break; - } - break; - } - case parse_mode::map: - { - if (state_stack_.back().index < state_stack_.back().length) - { - ++state_stack_.back().index; - read_name(ec); - if (ec) - { - return; - } - read_item(ec); - if (ec) - { - return; - } - } - else - { - end_map(ec); - } - break; - } - case parse_mode::indefinite_map: - { - int c = source_.peek(); - switch (c) - { - case Src::traits_type::eof(): - ec = cbor_errc::unexpected_eof; - return; - case 0xff: - end_map(ec); - if (ec) - { - return; - } - break; - default: - read_name(ec); - if (ec) - { - return; - } - read_item(ec); - if (ec) - { - return; - } - break; - } - break; - } - case parse_mode::root: - { - read_item(ec); - if (ec) - { - return; - } - } - break; - } - - JSONCONS_ASSERT(!state_stack_.empty()); - if (state_stack_.back().mode == parse_mode::root) - { - state_stack_.pop_back(); - handler_.flush(); - } - } - } - - void read_item(std::error_code& ec) - { - read_tags(ec); - if (ec) - { - return; - } - int c = source_.peek(); - if (c == Src::traits_type::eof()) - { - ec = cbor_errc::unexpected_eof; - return; - } - jsoncons::cbor::detail::cbor_major_type major_type = get_major_type((uint8_t)c); - - uint8_t info = get_additional_information_value((uint8_t)c); - switch (major_type) - { - case jsoncons::cbor::detail::cbor_major_type::unsigned_integer: - { - uint64_t val = get_uint64_value(source_, ec); - if (ec) - { - return; - } - if (state_stack_.back().stringref_map && !tags_.empty() && tags_.back() == 25) - { - tags_.pop_back(); - if (val >= state_stack_.back().stringref_map->size()) - { - ec = cbor_errc::stringref_too_large; - return; - } - stringref_map_type::size_type index = (stringref_map_type::size_type)val; - if (index != val) - { - ec = cbor_errc::number_too_large; - return; - } - auto& str = state_stack_.back().stringref_map->at(index); - switch (str.type) - { - case jsoncons::cbor::detail::cbor_major_type::text_string: - { - handle_string(basic_string_view(str.s.data(),str.s.length()),ec); - if (ec) - { - return; - } - break; - } - case jsoncons::cbor::detail::cbor_major_type::byte_string: - { - handle_byte_string(byte_string_view(str.bs.data(),str.bs.size()), ec); - if (ec) - { - return; - } - break; - } - default: - JSONCONS_UNREACHABLE(); - break; - } - } - else - { - semantic_tag tag = semantic_tag::none; - if (!tags_.empty()) - { - if (tags_.back() == 1) - { - tag = semantic_tag::timestamp; - } - tags_.clear(); - } - handler_.uint64_value(val, tag, *this); - } - break; - } - case jsoncons::cbor::detail::cbor_major_type::negative_integer: - { - int64_t val = get_int64_value(source_, ec); - if (ec) - { - return; - } - semantic_tag tag = semantic_tag::none; - if (!tags_.empty()) - { - if (tags_.back() == 1) - { - tag = semantic_tag::timestamp; - } - tags_.clear(); - } - handler_.int64_value(val, tag, *this); - break; - } - case jsoncons::cbor::detail::cbor_major_type::byte_string: - { - std::vector v = get_byte_string(ec); - if (ec) - { - return; - } - handle_byte_string(byte_string_view(v.data(), v.size()), ec); - if (ec) - { - return; - } - break; - } - case jsoncons::cbor::detail::cbor_major_type::text_string: - { - std::string s = get_text_string(ec); - auto result = unicons::validate(s.begin(),s.end()); - if (result.ec != unicons::conv_errc()) - { - ec = cbor_errc::invalid_utf8_text_string; - return; - } - handle_string(basic_string_view(s.data(),s.length()),ec); - if (ec) - { - return; - } - break; - } - case jsoncons::cbor::detail::cbor_major_type::semantic_tag: - { - JSONCONS_UNREACHABLE(); - break; - } - case jsoncons::cbor::detail::cbor_major_type::simple: - { - switch (info) - { - case 0x14: - handler_.bool_value(false, semantic_tag::none, *this); - source_.ignore(1); - break; - case 0x15: - handler_.bool_value(true, semantic_tag::none, *this); - source_.ignore(1); - break; - case 0x16: - handler_.null_value(semantic_tag::none, *this); - source_.ignore(1); - break; - case 0x17: - handler_.null_value(semantic_tag::undefined, *this); - source_.ignore(1); - break; - case 0x19: // Half-Precision Float (two-byte IEEE 754) - case 0x1a: // Single-Precision Float (four-byte IEEE 754) - case 0x1b: // Double-Precision Float (eight-byte IEEE 754) - double val = get_double(source_, ec); - if (ec) - { - return; - } - semantic_tag tag = semantic_tag::none; - if (!tags_.empty()) - { - if (tags_.back() == 1) - { - tag = semantic_tag::timestamp; - } - tags_.clear(); - } - handler_.double_value(val, tag, *this); - break; - } - break; - } - case jsoncons::cbor::detail::cbor_major_type::array: - { - if (!tags_.empty() && tags_.back() == 0x04) - { - std::string s = get_array_as_decimal_string(ec); - if (ec) - { - return; - } - handler_.string_value(s, semantic_tag::bigdec); - tags_.pop_back(); - } - else if (!tags_.empty() && tags_.back() == 0x05) - { - std::string s = get_array_as_hexfloat_string(ec); - if (ec) - { - return; - } - handler_.string_value(s, semantic_tag::bigfloat); - tags_.pop_back(); - } - else - { - begin_array(info, ec); - } - break; - } - case jsoncons::cbor::detail::cbor_major_type::map: - { - begin_map(info,ec); - break; - } - default: - break; - } - tags_.clear(); - } - - void begin_array(uint8_t info, std::error_code& ec) - { - semantic_tag tag = semantic_tag::none; - auto stringref_map = state_stack_.back().stringref_map; - for (auto t : tags_) - { - switch (t) - { - case 0x05: - tag = semantic_tag::bigfloat; - break; - case 0x100: // 256 (stringref-namespace) - stringref_map = std::make_shared(); - break; - default: - break; - } - } - tags_.clear(); - switch (info) - { - case jsoncons::cbor::detail::additional_info::indefinite_length: - { - state_stack_.push_back(parse_state(parse_mode::indefinite_array,0,stringref_map)); - handler_.begin_array(tag, *this); - source_.ignore(1); - break; - } - default: // definite length - { - size_t len = get_definite_length(source_,ec); - if (ec) - { - return; - } - state_stack_.push_back(parse_state(parse_mode::array,len,stringref_map)); - handler_.begin_array(len, tag, *this); - break; - } - } - } - - void end_array(std::error_code&) - { - switch (state_stack_.back().mode) - { - case parse_mode::indefinite_array: - { - source_.ignore(1); - break; - } - default: - break; - } - handler_.end_array(*this); - state_stack_.pop_back(); - } - - void begin_map(uint8_t info, std::error_code& ec) - { - auto stringref_map = state_stack_.back().stringref_map; - for (auto t : tags_) - { - switch (t) - { - case 0x100: // 256 (stringref-namespace) - stringref_map = std::make_shared(); - break; - default: - break; - } - } - tags_.clear(); - switch (info) - { - case jsoncons::cbor::detail::additional_info::indefinite_length: - { - state_stack_.push_back(parse_state(parse_mode::indefinite_map,0,stringref_map)); - handler_.begin_object(semantic_tag::none, *this); - source_.ignore(1); - break; - } - default: // definite_length - { - size_t len = get_definite_length(source_, ec); - if (ec) - { - return; - } - state_stack_.push_back(parse_state(parse_mode::map,len,stringref_map)); - handler_.begin_object(len, semantic_tag::none, *this); - break; - } - } - } - - void end_map(std::error_code&) - { - switch (state_stack_.back().mode) - { - case parse_mode::indefinite_map: - { - source_.ignore(1); - break; - } - default: - break; - } - handler_.end_object(*this); - state_stack_.pop_back(); - } - - void read_name(std::error_code& ec) - { - read_tags(ec); - if (ec) - { - return; - } - jsoncons::cbor::detail::cbor_major_type major_type; - int c = source_.peek(); - switch (c) - { - case Src::traits_type::eof(): - ec = cbor_errc::unexpected_eof; - return; - default: - major_type = get_major_type((uint8_t)c); - break; - } - switch (major_type) - { - case jsoncons::cbor::detail::cbor_major_type::text_string: - { - std::string s = get_text_string(ec); - if (ec) - { - return; - } - auto result = unicons::validate(s.begin(),s.end()); - if (result.ec != unicons::conv_errc()) - { - ec = cbor_errc::invalid_utf8_text_string; - return; - } - handler_.name(basic_string_view(s.data(),s.length()), *this); - break; - } - case jsoncons::cbor::detail::cbor_major_type::byte_string: - { - std::vector v = get_byte_string(ec); - if (ec) - { - return; - } - std::string s; - encode_base64url(v.begin(),v.end(),s); - handler_.name(basic_string_view(s.data(),s.length()), *this); - break; - } - case jsoncons::cbor::detail::cbor_major_type::unsigned_integer: - { - if (state_stack_.back().stringref_map && !tags_.empty() && tags_.back() == 25) - { - uint64_t ref = get_uint64_value(source_, ec); - if (ec) - { - return; - } - if (ref >= state_stack_.back().stringref_map->size()) - { - ec = cbor_errc::stringref_too_large; - return; - } - - stringref_map_type::size_type index = (stringref_map_type::size_type)ref; - if (index != ref) - { - ec = cbor_errc::number_too_large; - return; - } - auto& val = state_stack_.back().stringref_map->at(index); - switch (val.type) - { - case jsoncons::cbor::detail::cbor_major_type::text_string: - { - handler_.name(basic_string_view(val.s.data(),val.s.length()), *this); - break; - } - case jsoncons::cbor::detail::cbor_major_type::byte_string: - { - std::string s; - encode_base64url(val.bs.begin(),val.bs.end(),s); - handler_.name(basic_string_view(s.data(),s.length()), *this); - break; - } - default: - JSONCONS_UNREACHABLE(); - break; - } - tags_.pop_back(); - break; - } - } - JSONCONS_FALLTHROUGH; - default: - { - std::string s; - json_string_encoder encoder(s); - basic_cbor_reader reader(std::move(source_), encoder); - reader.read(ec); - source_ = std::move(reader.source_); - auto result = unicons::validate(s.begin(),s.end()); - if (result.ec != unicons::conv_errc()) - { - ec = cbor_errc::invalid_utf8_text_string; - return; - } - handler_.name(basic_string_view(s.data(),s.length()), *this); - } - } - } - - std::string get_text_string(std::error_code& ec) - { - std::string s; - - jsoncons::cbor::detail::cbor_major_type major_type; - uint8_t info; - int c = source_.peek(); - switch (c) - { - case Src::traits_type::eof(): - ec = cbor_errc::unexpected_eof; - return s; - default: - major_type = get_major_type((uint8_t)c); - info = get_additional_information_value((uint8_t)c); - break; - } - JSONCONS_ASSERT(major_type == jsoncons::cbor::detail::cbor_major_type::text_string); - auto func = [&s](Src& source_, size_t length, std::error_code& ec) - { - s.reserve(s.size()+length); - source_.read(std::back_inserter(s), length); - if (source_.eof()) - { - ec = cbor_errc::unexpected_eof; - return; - } - }; - iterate_string_chunks(source_, func, ec); - if (state_stack_.back().stringref_map && - info != jsoncons::cbor::detail::additional_info::indefinite_length && - s.length() >= jsoncons::cbor::detail::min_length_for_stringref(state_stack_.back().stringref_map->size())) - { - state_stack_.back().stringref_map->emplace_back(s); - } - - return s; - } - - static size_t get_definite_length(Src& source, std::error_code& ec) - { - if (JSONCONS_UNLIKELY(source.eof())) - { - ec = cbor_errc::unexpected_eof; - return 0; - } - switch (get_major_type((uint8_t)source.peek())) - { - case jsoncons::cbor::detail::cbor_major_type::byte_string: - case jsoncons::cbor::detail::cbor_major_type::text_string: - case jsoncons::cbor::detail::cbor_major_type::array: - case jsoncons::cbor::detail::cbor_major_type::map: - break; - default: - return 0; - } - - uint64_t u = get_uint64_value(source, ec); - size_t len = (size_t)u; - if (len != u) - { - ec = cbor_errc::number_too_large; - } - return len; - } - - std::vector get_byte_string(std::error_code& ec) - { - std::vector v; - - jsoncons::cbor::detail::cbor_major_type major_type; - uint8_t info; - int c = source_.peek(); - switch (c) - { - case Src::traits_type::eof(): - ec = cbor_errc::unexpected_eof; - return v; - default: - major_type = get_major_type((uint8_t)c); - info = get_additional_information_value((uint8_t)c); - break; - } - JSONCONS_ASSERT(major_type == jsoncons::cbor::detail::cbor_major_type::byte_string); - auto func = [&v](Src& source_, size_t length, std::error_code& ec) - { - v.reserve(v.size()+length); - source_.read(std::back_inserter(v), length); - if (source_.eof()) - { - ec = cbor_errc::unexpected_eof; - return; - } - }; - iterate_string_chunks(source_, func, ec); - if (state_stack_.back().stringref_map && - info != jsoncons::cbor::detail::additional_info::indefinite_length && - v.size() >= jsoncons::cbor::detail::min_length_for_stringref(state_stack_.back().stringref_map->size())) - { - state_stack_.back().stringref_map->emplace_back(v); - } - return v; - } - - template - static void iterate_string_chunks(Src& source, - Function func, - std::error_code& ec) - { - int c = source.peek(); - if (c == Src::traits_type::eof()) - { - ec = cbor_errc::unexpected_eof; - return; - } - - jsoncons::cbor::detail::cbor_major_type major_type = get_major_type((uint8_t)c); - JSONCONS_ASSERT(major_type == jsoncons::cbor::detail::cbor_major_type::text_string || major_type == jsoncons::cbor::detail::cbor_major_type::byte_string); - uint8_t info = get_additional_information_value((uint8_t)c); - - switch (info) - { - case jsoncons::cbor::detail::additional_info::indefinite_length: - { - source.ignore(1); - bool done = false; - while (!done) - { - int test = source.peek(); - switch (test) - { - case Src::traits_type::eof(): - ec = cbor_errc::unexpected_eof; - return; - case 0xff: - done = true; - break; - default: - iterate_string_chunks(source, func, ec); - if (ec) - { - return; - } - break; - } - } - source.ignore(1); - break; - } - default: // definite length - { - size_t length = get_definite_length(source, ec); - if (ec) - { - return; - } - func(source, length, ec); - if (ec) - { - return; - } - break; - } - } - } - - static uint64_t get_uint64_value(Src& source, std::error_code& ec) - { - uint64_t val = 0; - if (JSONCONS_UNLIKELY(source.eof())) - { - ec = cbor_errc::unexpected_eof; - return val; - } - const uint8_t* endp = nullptr; - - uint8_t type{}; - if (source.get(type) == 0) - { - ec = cbor_errc::unexpected_eof; - return 0; - } - uint8_t info = get_additional_information_value(type); - switch (info) - { - case JSONCONS_CBOR_0x00_0x17: // Integer 0x00..0x17 (0..23) - { - val = info; - break; - } - - case 0x18: // Unsigned integer (one-byte uint8_t follows) - { - uint8_t c{}; - source.get(c); - if (source.eof()) - { - ec = cbor_errc::unexpected_eof; - return val; - } - val = c; - break; - } - - case 0x19: // Unsigned integer (two-byte uint16_t follows) - { - uint8_t buf[sizeof(uint16_t)]; - source.read(buf, sizeof(uint16_t)); - val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - break; - } - - case 0x1a: // Unsigned integer (four-byte uint32_t follows) - { - uint8_t buf[sizeof(uint32_t)]; - source.read(buf, sizeof(uint32_t)); - val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - break; - } - - case 0x1b: // Unsigned integer (eight-byte uint64_t follows) - { - uint8_t buf[sizeof(uint64_t)]; - source.read(buf, sizeof(uint64_t)); - val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - break; - } - default: - break; - } - return val; - } - - static int64_t get_int64_value(Src& source, std::error_code& ec) - { - int64_t val = 0; - if (JSONCONS_UNLIKELY(source.eof())) - { - ec = cbor_errc::unexpected_eof; - return val; - } - const uint8_t* endp = nullptr; - - uint8_t info = get_additional_information_value((uint8_t)source.peek()); - switch (get_major_type((uint8_t)source.peek())) - { - case jsoncons::cbor::detail::cbor_major_type::negative_integer: - source.ignore(1); - switch (info) - { - case JSONCONS_CBOR_0x00_0x17: // 0x00..0x17 (0..23) - { - val = static_cast(- 1 - info); - break; - } - case 0x18: // Negative integer (one-byte uint8_t follows) - { - uint8_t c{}; - source.get(c); - if (source.eof()) - { - ec = cbor_errc::unexpected_eof; - return val; - } - val = static_cast(-1)- c; - break; - } - - case 0x19: // Negative integer -1-n (two-byte uint16_t follows) - { - uint8_t buf[sizeof(uint16_t)]; - if (source.read(buf, sizeof(uint16_t)) != sizeof(uint16_t)) - { - return val; - } - auto x = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - val = static_cast(-1)- x; - break; - } - - case 0x1a: // Negative integer -1-n (four-byte uint32_t follows) - { - uint8_t buf[sizeof(uint32_t)]; - if (source.read(buf, sizeof(uint32_t)) != sizeof(uint32_t)) - { - return val; - } - auto x = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - val = static_cast(-1)- x; - break; - } - - case 0x1b: // Negative integer -1-n (eight-byte uint64_t follows) - { - uint8_t buf[sizeof(uint64_t)]; - if (source.read(buf, sizeof(uint64_t)) != sizeof(uint64_t)) - { - return val; - } - auto x = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - val = static_cast(-1)- static_cast(x); - break; - } - } - break; - - case jsoncons::cbor::detail::cbor_major_type::unsigned_integer: - { - uint64_t x = get_uint64_value(source, ec); - if (ec) - { - return 0; - } - if (x <= static_cast((std::numeric_limits::max)())) - { - val = x; - } - else - { - // error; - } - - break; - } - break; - default: - break; - } - - return val; - } - - static double get_double(Src& source, std::error_code& ec) - { - double val = 0; - if (JSONCONS_UNLIKELY(source.eof())) - { - ec = cbor_errc::unexpected_eof; - return val; - } - const uint8_t* endp = nullptr; - - uint8_t type{}; - if (source.get(type) == 0) - { - ec = cbor_errc::unexpected_eof; - return 0; - } - uint8_t info = get_additional_information_value(type); - switch (info) - { - case 0x19: // Half-Precision Float (two-byte IEEE 754) - { - uint8_t buf[sizeof(uint16_t)]; - source.read(buf, sizeof(uint16_t)); - if (source.eof()) - { - ec = cbor_errc::unexpected_eof; - return 0; - } - uint16_t x = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - val = jsoncons::detail::decode_half(x); - break; - } - - - case 0x1a: // Single-Precision Float (four-byte IEEE 754) - { - uint8_t buf[sizeof(float)]; - source.read(buf, sizeof(float)); - if (source.eof()) - { - ec = cbor_errc::unexpected_eof; - return 0; - } - val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - break; - } - - case 0x1b: // Double-Precision Float (eight-byte IEEE 754) - { - uint8_t buf[sizeof(double)]; - source.read(buf, sizeof(double)); - if (source.eof()) - { - ec = cbor_errc::unexpected_eof; - return 0; - } - val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - break; - } - default: - break; - } - - return val; - } - - std::string get_array_as_decimal_string(std::error_code& ec) - { - std::string s; - - int c; - if ((c=source_.get()) == Src::traits_type::eof()) - { - ec = cbor_errc::unexpected_eof; - return s; - } - jsoncons::cbor::detail::cbor_major_type major_type = get_major_type((uint8_t)c); - uint8_t info = get_additional_information_value((uint8_t)c); - JSONCONS_ASSERT(major_type == jsoncons::cbor::detail::cbor_major_type::array); - JSONCONS_ASSERT(info == 2); - - if ((c=source_.peek()) == Src::traits_type::eof()) - { - ec = cbor_errc::unexpected_eof; - return s; - } - int64_t exponent = 0; - switch (get_major_type((uint8_t)c)) - { - case jsoncons::cbor::detail::cbor_major_type::unsigned_integer: - { - exponent = get_uint64_value(source_,ec); - if (ec) - { - return s; - } - break; - } - case jsoncons::cbor::detail::cbor_major_type::negative_integer: - { - exponent = get_int64_value(source_,ec); - if (ec) - { - return s; - } - break; - } - default: - { - ec = cbor_errc::invalid_decimal; - return s; - } - } - - switch (get_major_type((uint8_t)source_.peek())) - { - case jsoncons::cbor::detail::cbor_major_type::unsigned_integer: - { - uint64_t val = get_uint64_value(source_,ec); - if (ec) - { - return s; - } - jsoncons::detail::print_uinteger(val, s); - break; - } - case jsoncons::cbor::detail::cbor_major_type::negative_integer: - { - int64_t val = get_int64_value(source_,ec); - if (ec) - { - return s; - } - jsoncons::detail::print_integer(val, s); - break; - } - case jsoncons::cbor::detail::cbor_major_type::semantic_tag: - { - if ((c=source_.get()) == Src::traits_type::eof()) - { - ec = cbor_errc::unexpected_eof; - return s; - } - uint8_t tag = get_additional_information_value((uint8_t)c); - if ((c=source_.peek()) == Src::traits_type::eof()) - { - ec = cbor_errc::unexpected_eof; - return s; - } - - if (get_major_type((uint8_t)c) == jsoncons::cbor::detail::cbor_major_type::byte_string) - { - std::vector v = get_byte_string(ec); - if (ec) - { - return s; - } - if (tag == 2) - { - bignum n(1, v.data(), v.size()); - n.dump(s); - } - else if (tag == 3) - { - bignum n(-1, v.data(), v.size()); - n.dump(s); - } - } - break; - } - default: - { - ec = cbor_errc::invalid_decimal; - return s; - } - } - - std::string result; - if (s.size() > 0) - { - if (s[0] == '-') - { - result.push_back('-'); - jsoncons::detail::prettify_string(s.c_str()+1, s.size()-1, (int)exponent, -4, 17, result); - } - else - { - jsoncons::detail::prettify_string(s.c_str(), s.size(), (int)exponent, -4, 17, result); - } - } - //std::cout << "s: " << s << ", exponent: " << std::dec << exponent << ", result: " << result << "\n"; - return result; - } - - std::string get_array_as_hexfloat_string(std::error_code& ec) - { - std::string s; - - int c; - if ((c=source_.get()) == Src::traits_type::eof()) - { - ec = cbor_errc::unexpected_eof; - return s; - } - jsoncons::cbor::detail::cbor_major_type major_type = get_major_type((uint8_t)c); - uint8_t info = get_additional_information_value((uint8_t)c); - JSONCONS_ASSERT(major_type == jsoncons::cbor::detail::cbor_major_type::array); - JSONCONS_ASSERT(info == 2); - - if ((c=source_.peek()) == Src::traits_type::eof()) - { - ec = cbor_errc::unexpected_eof; - return s; - } - int64_t exponent = 0; - switch (get_major_type((uint8_t)c)) - { - case jsoncons::cbor::detail::cbor_major_type::unsigned_integer: - { - exponent = get_uint64_value(source_,ec); - if (ec) - { - return s; - } - break; - } - case jsoncons::cbor::detail::cbor_major_type::negative_integer: - { - exponent = get_int64_value(source_,ec); - if (ec) - { - return s; - } - break; - } - default: - { - ec = cbor_errc::invalid_bigfloat; - return s; - } - } - - switch (get_major_type((uint8_t)source_.peek())) - { - case jsoncons::cbor::detail::cbor_major_type::unsigned_integer: - { - uint64_t val = get_uint64_value(source_,ec); - if (ec) - { - return s; - } - s.push_back('0'); - s.push_back('x'); - jsoncons::detail::uinteger_to_hex_string(val, s); - break; - } - case jsoncons::cbor::detail::cbor_major_type::negative_integer: - { - int64_t val = get_int64_value(source_,ec); - if (ec) - { - return s; - } - s.push_back('-'); - s.push_back('0'); - s.push_back('x'); - jsoncons::detail::uinteger_to_hex_string(static_cast(-val), s); - break; - } - case jsoncons::cbor::detail::cbor_major_type::semantic_tag: - { - if ((c=source_.get()) == Src::traits_type::eof()) - { - ec = cbor_errc::unexpected_eof; - return s; - } - uint8_t tag = get_additional_information_value((uint8_t)c); - if ((c=source_.peek()) == Src::traits_type::eof()) - { - ec = cbor_errc::unexpected_eof; - return s; - } - - if (get_major_type((uint8_t)c) == jsoncons::cbor::detail::cbor_major_type::byte_string) - { - std::vector v = get_byte_string(ec); - if (ec) - { - return s; - } - if (tag == 2) - { - s.push_back('-'); - s.push_back('0'); - s.push_back('x'); - bignum n(1, v.data(), v.size()); - n.dump_hex_string(s); - } - else if (tag == 3) - { - s.push_back('-'); - s.push_back('0'); - bignum n(-1, v.data(), v.size()); - n.dump_hex_string(s); - s[2] = 'x'; - } - } - break; - } - default: - { - ec = cbor_errc::invalid_bigfloat; - return s; - } - } - - s.push_back('p'); - if (exponent >=0) - { - jsoncons::detail::uinteger_to_hex_string(static_cast(exponent), s); - } - else - { - s.push_back('-'); - jsoncons::detail::uinteger_to_hex_string(static_cast(-exponent), s); - } - return s; - } - - static jsoncons::cbor::detail::cbor_major_type get_major_type(uint8_t type) - { - static const uint8_t major_type_shift = 0x05; - uint8_t value = type >> major_type_shift; - return static_cast(value); - } - - static uint8_t get_additional_information_value(uint8_t type) - { - static const uint8_t additional_information_mask = (1U << 5) - 1; - uint8_t value = type & additional_information_mask; - return value; - } - - void read_tags(std::error_code& ec) - { - int c = source_.peek(); - if (c == Src::traits_type::eof()) - { - ec = cbor_errc::unexpected_eof; - return; - } - jsoncons::cbor::detail::cbor_major_type major_type = get_major_type((uint8_t)c); - while (major_type == jsoncons::cbor::detail::cbor_major_type::semantic_tag) - { - uint64_t val = get_uint64_value(source_, ec); - if (ec) - { - return; - } - tags_.push_back(val); - c = source_.peek(); - if (c == Src::traits_type::eof()) - { - ec = cbor_errc::unexpected_eof; - return; - } - major_type = get_major_type((uint8_t)c); - } - } - - void handle_string(const basic_string_view& v, std::error_code&) - { - semantic_tag tag = semantic_tag::none; - if (!tags_.empty()) - { - switch (tags_.back()) - { - case 0: - tag = semantic_tag::datetime; - break; - case 32: - tag = semantic_tag::uri; - break; - case 33: - tag = semantic_tag::base64url; - break; - case 34: - tag = semantic_tag::base64; - break; - default: - break; - } - tags_.clear(); - } - handler_.string_value(v, tag, *this); - } - - void handle_byte_string(const byte_string_view& v, std::error_code&) - { - if (!tags_.empty()) - { - switch (tags_.back()) - { - case 0x2: - { - bignum n(1, v.data(), v.size()); - buffer_.clear(); - n.dump(buffer_); - handler_.string_value(buffer_, semantic_tag::bigint, *this); - break; - } - case 0x3: - { - bignum n(-1, v.data(), v.size()); - buffer_.clear(); - n.dump(buffer_); - handler_.string_value(buffer_, semantic_tag::bigint, *this); - break; - } - case 0x15: - { - handler_.byte_string_value(byte_string_view(v.data(), v.size()), semantic_tag::base64url, *this); - break; - } - case 0x16: - { - handler_.byte_string_value(byte_string_view(v.data(), v.size()), semantic_tag::base64, *this); - break; - } - case 0x17: - { - handler_.byte_string_value(byte_string_view(v.data(), v.size()), semantic_tag::base16, *this); - break; - } - default: - handler_.byte_string_value(byte_string_view(v.data(), v.size()), semantic_tag::none, *this); - break; - } - tags_.clear(); - } - else - { - handler_.byte_string_value(byte_string_view(v.data(), v.size()), semantic_tag::none, *this); - } - } - -}; - -typedef basic_cbor_reader cbor_reader; - -typedef basic_cbor_reader cbor_bytes_reader; - -#if !defined(JSONCONS_NO_DEPRECATED) -typedef cbor_bytes_reader cbor_buffer_reader; -#endif - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv.hpp deleted file mode 100644 index d0d5597c98..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv.hpp +++ /dev/null @@ -1,14 +0,0 @@ -/// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_CSV_CSV_HPP -#define JSONCONS_CSV_CSV_HPP - -#include -#include -#include - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_encoder.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_encoder.hpp deleted file mode 100644 index 1bc98d9830..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_encoder.hpp +++ /dev/null @@ -1,619 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_CSV_CSV_ENCODER_HPP -#define JSONCONS_CSV_CSV_ENCODER_HPP - -#include // std::array -#include -#include -#include -#include -#include // std::move -#include // std::unordered_map -#include // std::allocator -#include // std::numeric_limits -#include -#include -#include -#include -#include - -namespace jsoncons { namespace csv { - -template,class Allocator=std::allocator> -class basic_csv_encoder final : public basic_json_content_handler -{ - static const std::array& null_k() - { - static constexpr std::array k{'n','u','l','l'}; - return k; - } - static const std::array& true_k() - { - static constexpr std::array k{'t','r','u','e'}; - return k; - } - static const std::array& false_k() - { - static constexpr std::array k{'f','a','l','s','e'}; - return k; - } - -public: - typedef CharT char_type; - using typename basic_json_content_handler::string_view_type; - typedef Result result_type; - - typedef Allocator allocator_type; - typedef typename std::allocator_traits:: template rebind_alloc char_allocator_type; - typedef std::basic_string, char_allocator_type> string_type; - typedef typename std::allocator_traits:: template rebind_alloc string_allocator_type; - -private: - struct stack_item - { - stack_item(bool is_object) - : is_object_(is_object), count_(0) - { - } - bool is_object() const - { - return is_object_; - } - - bool is_object_; - size_t count_; - string_type name_; - }; - Result result_; - const basic_csv_encode_options& options_; - std::vector stack_; - jsoncons::detail::print_double fp_; - std::vector column_names_; - - typedef typename std::allocator_traits:: template rebind_alloc> string_string_allocator_type; - std::unordered_map,std::equal_to,string_string_allocator_type> buffered_line_; - - // Noncopyable and nonmoveable - basic_csv_encoder(const basic_csv_encoder&) = delete; - basic_csv_encoder& operator=(const basic_csv_encoder&) = delete; -public: - basic_csv_encoder(result_type result) - : basic_csv_encoder(std::move(result), basic_csv_options::default_options()) - { - } - - basic_csv_encoder(result_type result, - const basic_csv_encode_options& options) - : - result_(std::move(result)), - options_(options), - stack_(), - fp_(floating_point_options(options.floating_point_format(), - options.precision(), - 0)), - column_names_(options_.column_names()) - { - } - - ~basic_csv_encoder() - { - try - { - result_.flush(); - } - catch (...) - { - } - } - -private: - - template - void escape_string(const CharT* s, - size_t length, - CharT quote_char, CharT quote_escape_char, - AnyWriter& result) - { - const CharT* begin = s; - const CharT* end = s + length; - for (const CharT* it = begin; it != end; ++it) - { - CharT c = *it; - if (c == quote_char) - { - result.push_back(quote_escape_char); - result.push_back(quote_char); - } - else - { - result.push_back(c); - } - } - } - - void do_flush() override - { - result_.flush(); - } - - bool do_begin_object(semantic_tag, const ser_context&) override - { - stack_.push_back(stack_item(true)); - return true; - } - - bool do_end_object(const ser_context&) override - { - if (stack_.size() == 2) - { - if (stack_[0].count_ == 0) - { - for (size_t i = 0; i < column_names_.size(); ++i) - { - if (i > 0) - { - result_.push_back(options_.field_delimiter()); - } - result_.append(column_names_[i].data(), - column_names_[i].length()); - } - result_.append(options_.line_delimiter().data(), - options_.line_delimiter().length()); - } - for (size_t i = 0; i < column_names_.size(); ++i) - { - if (i > 0) - { - result_.push_back(options_.field_delimiter()); - } - auto it = buffered_line_.find(column_names_[i]); - if (it != buffered_line_.end()) - { - result_.append(it->second.data(),it->second.length()); - it->second.clear(); - } - } - result_.append(options_.line_delimiter().data(), options_.line_delimiter().length()); - } - stack_.pop_back(); - - end_value(); - return true; - } - - bool do_begin_array(semantic_tag, const ser_context&) override - { - stack_.push_back(stack_item(false)); - if (stack_.size() == 2) - { - if (stack_[0].count_ == 0) - { - for (size_t i = 0; i < column_names_.size(); ++i) - { - if (i > 0) - { - result_.push_back(options_.field_delimiter()); - } - result_.append(column_names_[i].data(),column_names_[i].length()); - } - if (column_names_.size() > 0) - { - result_.append(options_.line_delimiter().data(), - options_.line_delimiter().length()); - } - } - } - return true; - } - - bool do_end_array(const ser_context&) override - { - if (stack_.size() == 2) - { - result_.append(options_.line_delimiter().data(), - options_.line_delimiter().length()); - } - stack_.pop_back(); - - end_value(); - return true; - } - - bool do_name(const string_view_type& name, const ser_context&) override - { - if (stack_.size() == 2) - { - stack_.back().name_ = string_type(name); - buffered_line_[string_type(name)] = std::basic_string(); - if (stack_[0].count_ == 0 && options_.column_names().size() == 0) - { - column_names_.push_back(string_type(name)); - } - } - return true; - } - - template - bool string_value(const CharT* s, size_t length, AnyWriter& result) - { - bool quote = false; - if (options_.quote_style() == quote_style_type::all || options_.quote_style() == quote_style_type::nonnumeric || - (options_.quote_style() == quote_style_type::minimal && - (std::char_traits::find(s, length, options_.field_delimiter()) != nullptr || std::char_traits::find(s, length, options_.quote_char()) != nullptr))) - { - quote = true; - result.push_back(options_.quote_char()); - } - escape_string(s, length, options_.quote_char(), options_.quote_escape_char(), result); - if (quote) - { - result.push_back(options_.quote_char()); - } - - return true; - } - - bool do_null_value(semantic_tag, const ser_context&) override - { - if (stack_.size() == 2) - { - if (stack_.back().is_object()) - { - auto it = buffered_line_.find(stack_.back().name_); - if (it != buffered_line_.end()) - { - std::basic_string s; - jsoncons::string_result> bo(s); - accept_null_value(bo); - bo.flush(); - it->second = s; - } - } - else - { - accept_null_value(result_); - } - } - return true; - } - - bool do_string_value(const string_view_type& sv, semantic_tag, const ser_context&) override - { - if (stack_.size() == 2) - { - if (stack_.back().is_object()) - { - auto it = buffered_line_.find(stack_.back().name_); - if (it != buffered_line_.end()) - { - std::basic_string s; - jsoncons::string_result> bo(s); - value(sv,bo); - bo.flush(); - it->second = s; - } - } - else - { - value(sv,result_); - } - } - return true; - } - - bool do_byte_string_value(const byte_string_view& b, - semantic_tag tag, - const ser_context& context) override - { - byte_string_chars_format encoding_hint; - switch (tag) - { - case semantic_tag::base16: - encoding_hint = byte_string_chars_format::base16; - break; - case semantic_tag::base64: - encoding_hint = byte_string_chars_format::base64; - break; - case semantic_tag::base64url: - encoding_hint = byte_string_chars_format::base64url; - break; - default: - encoding_hint = byte_string_chars_format::none; - break; - } - byte_string_chars_format format = jsoncons::detail::resolve_byte_string_chars_format(encoding_hint,byte_string_chars_format::none,byte_string_chars_format::base64url); - - std::basic_string s; - switch (format) - { - case byte_string_chars_format::base16: - { - encode_base16(b.begin(),b.end(),s); - do_string_value(s, semantic_tag::none, context); - break; - } - case byte_string_chars_format::base64: - { - encode_base64(b.begin(),b.end(),s); - do_string_value(s, semantic_tag::none, context); - break; - } - case byte_string_chars_format::base64url: - { - encode_base64url(b.begin(),b.end(),s); - do_string_value(s, semantic_tag::none, context); - break; - } - default: - { - JSONCONS_UNREACHABLE(); - } - } - - return true; - } - - bool do_double_value(double val, - semantic_tag, - const ser_context&) override - { - if (stack_.size() == 2) - { - if (stack_.back().is_object()) - { - auto it = buffered_line_.find(stack_.back().name_); - if (it != buffered_line_.end()) - { - std::basic_string s; - jsoncons::string_result> bo(s); - value(val, bo); - bo.flush(); - it->second = s; - } - } - else - { - value(val, result_); - } - } - return true; - } - - bool do_int64_value(int64_t val, - semantic_tag, - const ser_context&) override - { - if (stack_.size() == 2) - { - if (stack_.back().is_object()) - { - auto it = buffered_line_.find(stack_.back().name_); - if (it != buffered_line_.end()) - { - std::basic_string s; - jsoncons::string_result> bo(s); - value(val,bo); - bo.flush(); - it->second = s; - } - } - else - { - value(val,result_); - } - } - return true; - } - - bool do_uint64_value(uint64_t val, - semantic_tag, - const ser_context&) override - { - if (stack_.size() == 2) - { - if (stack_.back().is_object()) - { - auto it = buffered_line_.find(stack_.back().name_); - if (it != buffered_line_.end()) - { - std::basic_string s; - jsoncons::string_result> bo(s); - value(val,bo); - bo.flush(); - it->second = s; - } - } - else - { - value(val,result_); - } - } - return true; - } - - bool do_bool_value(bool val, semantic_tag, const ser_context&) override - { - if (stack_.size() == 2) - { - if (stack_.back().is_object()) - { - auto it = buffered_line_.find(stack_.back().name_); - if (it != buffered_line_.end()) - { - std::basic_string s; - jsoncons::string_result> bo(s); - value(val,bo); - bo.flush(); - it->second = s; - } - } - else - { - value(val,result_); - } - } - return true; - } - - template - void value(const string_view_type& value, AnyWriter& result) - { - begin_value(result); - string_value(value.data(),value.length(),result); - end_value(); - } - - template - void value(double val, AnyWriter& result) - { - begin_value(result); - - if ((std::isnan)(val)) - { - result.append(null_k().data(), null_k().size()); - } - else if (val == std::numeric_limits::infinity()) - { - result.append(null_k().data(), null_k().size()); - } - else if (!(std::isfinite)(val)) - { - result.append(null_k().data(), null_k().size()); - } - else - { - fp_(val, result); - } - - end_value(); - - } - - template - void value(int64_t val, AnyWriter& result) - { - begin_value(result); - - std::basic_ostringstream ss; - ss << val; - result.append(ss.str().data(),ss.str().length()); - - end_value(); - } - - template - void value(uint64_t val, AnyWriter& result) - { - begin_value(result); - - std::basic_ostringstream ss; - ss << val; - result.append(ss.str().data(),ss.str().length()); - - end_value(); - } - - template - void value(bool val, AnyWriter& result) - { - begin_value(result); - - if (val) - { - result.append(true_k().data(), true_k().size()); - } - else - { - result.append(false_k().data(), false_k().size()); - } - - end_value(); - } - - template - bool accept_null_value(AnyWriter& result) - { - begin_value(result); - result.append(null_k().data(), null_k().size()); - end_value(); - return true; - } - - template - void begin_value(AnyWriter& result) - { - if (!stack_.empty()) - { - if (!stack_.back().is_object_ && stack_.back().count_ > 0) - { - result.push_back(options_.field_delimiter()); - } - } - } - - void end_value() - { - if (!stack_.empty()) - { - ++stack_.back().count_; - } - } -}; - -// encode_csv - -template -typename std::enable_if::value,void>::type -encode_csv(const T& j, std::basic_string& s, const basic_csv_options& options = basic_csv_options::default_options()) -{ - typedef CharT char_type; - basic_csv_encoder>> encoder(s,options); - j.dump(encoder); -} - -template -typename std::enable_if::value,void>::type -encode_csv(const T& val, std::basic_string& s, const basic_csv_options& options = basic_csv_options::default_options()) -{ - typedef CharT char_type; - basic_csv_encoder>> encoder(s,options); - write_to(basic_json(), val, encoder); -} - -template -typename std::enable_if::value,void>::type -encode_csv(const T& j, std::basic_ostream& os, const basic_csv_options& options = basic_csv_options::default_options()) -{ - typedef CharT char_type; - basic_csv_encoder> encoder(os,options); - j.dump(encoder); -} - -template -typename std::enable_if::value,void>::type -encode_csv(const T& val, std::basic_ostream& os, const basic_csv_options& options = basic_csv_options::default_options()) -{ - typedef CharT char_type; - basic_csv_encoder> encoder(os,options); - write_to(basic_json(), val, encoder); -} - -typedef basic_csv_encoder csv_encoder; -typedef basic_csv_encoder> csv_string_encoder; - -#if !defined(JSONCONS_NO_DEPRECATED) -template, class Allocator = std::allocator> -using basic_csv_serializer = basic_csv_encoder; - -typedef basic_csv_serializer csv_serializer; -typedef basic_csv_serializer> csv_string_serializer; -#endif - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_error.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_error.hpp deleted file mode 100644 index 49a09a0d6d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_error.hpp +++ /dev/null @@ -1,77 +0,0 @@ -/// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_CSV_CSV_ERROR_HPP -#define JSONCONS_CSV_CSV_ERROR_HPP - -#include -#include - -namespace jsoncons { namespace csv { - - enum class csv_errc : int - { - ok = 0, - unexpected_eof = 1, - source_error, - expected_quote, - invalid_csv_text, - invalid_state - }; - -#if !defined(JSONCONS_NO_DEPRECATED) -typedef csv_errc csv_parser_errc; -#endif - -class csv_error_category_impl - : public std::error_category -{ -public: - const char* name() const noexcept override - { - return "jsoncons/csv"; - } - std::string message(int ev) const override - { - switch (static_cast(ev)) - { - case csv_errc::unexpected_eof: - return "Unexpected end of file"; - case csv_errc::source_error: - return "Source error"; - case csv_errc::expected_quote: - return "Expected quote character"; - case csv_errc::invalid_csv_text: - return "Invalid CSV text"; - default: - return "Unknown JSON parser error"; - } - } -}; - -inline -const std::error_category& csv_error_category() -{ - static csv_error_category_impl instance; - return instance; -} - -inline -std::error_code make_error_code(csv_errc result) -{ - return std::error_code(static_cast(result),csv_error_category()); -} - -}} - -namespace std { - template<> - struct is_error_code_enum : public true_type - { - }; -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_options.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_options.hpp deleted file mode 100644 index 5cae9e1610..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_options.hpp +++ /dev/null @@ -1,771 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_CSV_CSV_OPTIONS_HPP -#define JSONCONS_CSV_CSV_OPTIONS_HPP - -#include -#include -#include // std::pair -#include // std::unordered_map -#include // std::numeric_limits -#include - -namespace jsoncons { namespace csv { - -namespace detail { - JSONCONS_STRING_LITERAL(string,'s','t','r','i','n','g') - JSONCONS_STRING_LITERAL(integer,'i','n','t','e','g','e','r') - JSONCONS_STRING_LITERAL(float,'f','l','o','a','t') - JSONCONS_STRING_LITERAL(boolean,'b','o','o','l','e','a','n') -} - -enum class csv_column_type -{ - string_t,integer_t,float_t,boolean_t,repeat_t -}; - -enum class quote_style_type -{ - all,minimal,none,nonnumeric -}; - -typedef quote_style_type quote_styles; - -enum class mapping_type -{ - n_rows, - n_objects, - m_columns -}; - -enum class column_state {sequence,label}; - -struct csv_type_info -{ - csv_type_info() = default; - csv_type_info(const csv_type_info&) = default; - csv_type_info(csv_type_info&&) = default; - - csv_type_info(csv_column_type ctype, size_t lev, size_t repcount = 0) - { - col_type = ctype; - level = lev; - rep_count = repcount; - } - - csv_column_type col_type; - size_t level; - size_t rep_count; -}; - -template -class basic_csv_decode_options -{ -public: - typedef std::basic_string string_type; - - virtual size_t header_lines() const = 0; - - virtual bool assume_header() const = 0; - - virtual bool ignore_empty_values() const = 0; - - virtual bool ignore_empty_lines() const = 0; - - virtual bool trim_leading() const = 0; - - virtual bool trim_trailing() const = 0; - - virtual bool trim_leading_inside_quotes() const = 0; - - virtual bool trim_trailing_inside_quotes() const = 0; - - virtual bool trim() const = 0; - - virtual bool trim_inside_quotes() const = 0; - - virtual bool unquoted_empty_value_is_null() const = 0; - - virtual std::vector column_names() const = 0; - - virtual std::vector column_types() const = 0; - - virtual std::vector column_defaults() const = 0; - - virtual CharT field_delimiter() const = 0; - - virtual const std::pair& subfield_delimiter() const = 0; - - virtual string_type line_delimiter() const = 0; - - virtual CharT quote_char() const = 0; - - virtual bool infer_types() const = 0; - - virtual bool lossless_number() const = 0; - - virtual CharT quote_escape_char() const = 0; - - virtual CharT comment_starter() const = 0; - - virtual mapping_type mapping() const = 0; - - virtual unsigned long max_lines() const = 0; -}; - -template -class basic_csv_encode_options -{ -public: - typedef std::basic_string string_type; - - virtual chars_format floating_point_format() const = 0; - - virtual int precision() const = 0; - - virtual std::vector column_names() const = 0; - - virtual CharT field_delimiter() const = 0; - - virtual const std::pair& subfield_delimiter() const = 0; - - virtual string_type line_delimiter() const = 0; - - virtual CharT quote_char() const = 0; - - virtual CharT quote_escape_char() const = 0; - - virtual quote_style_type quote_style() const = 0; -}; - -template -class basic_csv_options : public virtual basic_csv_decode_options, - public virtual basic_csv_encode_options -{ - typedef CharT char_type; - typedef std::basic_string string_type; - - chars_format floating_point_format_; - int precision_; - bool assume_header_; - bool ignore_empty_values_; - bool ignore_empty_lines_; - bool trim_leading_; - bool trim_trailing_; - bool trim_leading_inside_quotes_; - bool trim_trailing_inside_quotes_; - bool unquoted_empty_value_is_null_; - CharT field_delimiter_; - std::pair subfield_delimiter_; - CharT quote_char_; - CharT quote_escape_char_; - CharT comment_starter_; - quote_style_type quote_style_; - std::pair mapping_; - unsigned long max_lines_; - size_t header_lines_; - string_type line_delimiter_; - bool infer_types_; - bool lossless_number_; - - std::vector column_names_; - std::vector column_types_; - std::vector column_defaults_; -public: - static const size_t default_indent = 4; - - static const basic_csv_options& default_options() - { - static basic_csv_options options{}; - return options; - } - -// Constructors - - basic_csv_options() : - floating_point_format_(chars_format()), - precision_(0), - assume_header_(false), - ignore_empty_values_(false), - ignore_empty_lines_(true), - trim_leading_(false), - trim_trailing_(false), - trim_leading_inside_quotes_(false), - trim_trailing_inside_quotes_(false), - unquoted_empty_value_is_null_(false), - field_delimiter_(','), - subfield_delimiter_(std::make_pair(',',false)), - quote_char_('\"'), - quote_escape_char_('\"'), - comment_starter_('\0'), - quote_style_(quote_style_type::minimal), - mapping_({mapping_type::n_rows,false}), - max_lines_((std::numeric_limits::max)()), - header_lines_(0), - infer_types_(true), - lossless_number_(false) - { - line_delimiter_.push_back('\n'); - } - -// Properties - - chars_format floating_point_format() const override - { - return floating_point_format_; - } - - basic_csv_options& floating_point_format(chars_format value) - { - floating_point_format_ = value; - return *this; - } - - int precision() const override - { - return precision_; - } - - basic_csv_options& precision(int value) - { - precision_ = value; - return *this; - } - - size_t header_lines() const override - { - return (assume_header_ && header_lines_ <= 1) ? 1 : header_lines_; - } - - basic_csv_options& header_lines(size_t value) - { - header_lines_ = value; - return *this; - } - - bool assume_header() const override - { - return assume_header_; - } - - basic_csv_options& assume_header(bool value) - { - assume_header_ = value; - return *this; - } - - bool ignore_empty_values() const override - { - return ignore_empty_values_; - } - - basic_csv_options& ignore_empty_values(bool value) - { - ignore_empty_values_ = value; - return *this; - } - - bool ignore_empty_lines() const override - { - return ignore_empty_lines_; - } - - basic_csv_options& ignore_empty_lines(bool value) - { - ignore_empty_lines_ = value; - return *this; - } - - bool trim_leading() const override - { - return trim_leading_; - } - - basic_csv_options& trim_leading(bool value) - { - trim_leading_ = value; - return *this; - } - - bool trim_trailing() const override - { - return trim_trailing_; - } - - basic_csv_options& trim_trailing(bool value) - { - trim_trailing_ = value; - return *this; - } - - bool trim_leading_inside_quotes() const override - { - return trim_leading_inside_quotes_; - } - - basic_csv_options& trim_leading_inside_quotes(bool value) - { - trim_leading_inside_quotes_ = value; - return *this; - } - - bool trim_trailing_inside_quotes() const override - { - return trim_trailing_inside_quotes_; - } - - basic_csv_options& trim_trailing_inside_quotes(bool value) - { - trim_trailing_inside_quotes_ = value; - return *this; - } - - bool trim() const override - { - return trim_leading_ && trim_trailing_; - } - - basic_csv_options& trim(bool value) - { - trim_leading_ = value; - trim_trailing_ = value; - return *this; - } - - bool trim_inside_quotes() const override - { - return trim_leading_inside_quotes_ && trim_trailing_inside_quotes_; - } - - basic_csv_options& trim_inside_quotes(bool value) - { - trim_leading_inside_quotes_ = value; - trim_trailing_inside_quotes_ = value; - return *this; - } - - bool unquoted_empty_value_is_null() const override - { - return unquoted_empty_value_is_null_; - } - - basic_csv_options& unquoted_empty_value_is_null(bool value) - { - unquoted_empty_value_is_null_ = value; - return *this; - } - - std::vector column_names() const override - { - return column_names_; - } - -#if !defined(JSONCONS_NO_DEPRECATED) - basic_csv_options& column_names(const std::vector& value) - { - column_names_ = value; - return *this; - } - - basic_csv_options& column_defaults(const std::vector& value) - { - column_defaults_ = value; - return *this; - } - - basic_csv_options& column_types(const std::vector& value) - { - if (value.size() > 0) - { - column_types_.reserve(value.size()); - for (size_t i = 0; i < value.size(); ++i) - { - if (value[i] == jsoncons::csv::detail::string_literal()()) - { - column_types_.emplace_back(csv_column_type::string_t,0); - } - else if (value[i] == jsoncons::csv::detail::integer_literal()()) - { - column_types_.emplace_back(csv_column_type::integer_t,0); - } - else if (value[i] == jsoncons::csv::detail::float_literal()()) - { - column_types_.emplace_back(csv_column_type::float_t,0); - } - else if (value[i] == jsoncons::csv::detail::boolean_literal()()) - { - column_types_.emplace_back(csv_column_type::boolean_t,0); - } - } - } - return *this; - } -#endif - basic_csv_options& column_names(const string_type& names) - { - column_names_ = parse_column_names(names); - return *this; - } - - std::vector column_types() const override - { - return column_types_; - } - - basic_csv_options& column_types(const string_type& types) - { - column_types_ = parse_column_types(types); - return *this; - } - - std::vector column_defaults() const override - { - return column_defaults_; - } - - basic_csv_options& column_defaults(const string_type& defaults) - { - column_defaults_ = parse_column_names(defaults); - return *this; - } - - CharT field_delimiter() const override - { - return field_delimiter_; - } - - const std::pair& subfield_delimiter() const override - { - return subfield_delimiter_; - } - - basic_csv_options& field_delimiter(CharT value) - { - field_delimiter_ = value; - return *this; - } - - basic_csv_options& subfield_delimiter(CharT value) - { - subfield_delimiter_ = std::make_pair(value,true); - return *this; - } - - string_type line_delimiter() const override - { - return line_delimiter_; - } - - basic_csv_options& line_delimiter(string_type value) - { - line_delimiter_ = value; - return *this; - } - - CharT quote_char() const override - { - return quote_char_; - } - - basic_csv_options& quote_char(CharT value) - { - quote_char_ = value; - return *this; - } - - bool infer_types() const override - { - return infer_types_; - } - - basic_csv_options& infer_types(bool value) - { - infer_types_ = value; - return *this; - } - - bool lossless_number() const override - { - return lossless_number_; - } - - basic_csv_options& lossless_number(bool value) - { - lossless_number_ = value; - return *this; - } - - CharT quote_escape_char() const override - { - return quote_escape_char_; - } - - basic_csv_options& quote_escape_char(CharT value) - { - quote_escape_char_ = value; - return *this; - } - - CharT comment_starter() const override - { - return comment_starter_; - } - - basic_csv_options& comment_starter(CharT value) - { - comment_starter_ = value; - return *this; - } - - quote_style_type quote_style() const override - { - return quote_style_; - } - - mapping_type mapping() const override - { - return mapping_.second ? (mapping_.first) : (assume_header() || column_names_.size() > 0 ? mapping_type::n_objects : mapping_type::n_rows); - } - - basic_csv_options& quote_style(quote_style_type value) - { - quote_style_ = value; - return *this; - } - - basic_csv_options& mapping(mapping_type value) - { - mapping_ = {value,true}; - return *this; - } - - unsigned long max_lines() const override - { - return max_lines_; - } - - basic_csv_options& max_lines(unsigned long value) - { - max_lines_ = value; - return *this; - } - - static std::vector parse_column_names(const string_type& names) - { - std::vector column_names; - - column_state state = column_state::sequence; - string_type buffer; - - auto p = names.begin(); - while (p != names.end()) - { - switch (state) - { - case column_state::sequence: - { - switch (*p) - { - case ' ': case '\t':case '\r': case '\n': - ++p; - break; - default: - buffer.clear(); - state = column_state::label; - break; - } - break; - } - case column_state::label: - { - switch (*p) - { - case ',': - column_names.push_back(buffer); - buffer.clear(); - ++p; - state = column_state::sequence; - break; - default: - buffer.push_back(*p); - ++p; - break; - } - break; - } - } - } - if (state == column_state::label) - { - column_names.push_back(buffer); - buffer.clear(); - } - return column_names; - } - - static std::vector parse_column_types(const string_type& types) - { - const std::unordered_map,std::equal_to> type_dictionary = - { - - {detail::string_literal(),csv_column_type::string_t}, - {detail::integer_literal(),csv_column_type::integer_t}, - {detail::float_literal(),csv_column_type::float_t}, - {detail::boolean_literal(),csv_column_type::boolean_t} - }; - - std::vector column_types; - - column_state state = column_state::sequence; - int depth = 0; - string_type buffer; - - auto p = types.begin(); - while (p != types.end()) - { - switch (state) - { - case column_state::sequence: - { - switch (*p) - { - case ' ': case '\t':case '\r': case '\n': - ++p; - break; - case '[': - ++depth; - ++p; - break; - case ']': - JSONCONS_ASSERT(depth > 0); - --depth; - ++p; - break; - case '*': - { - JSONCONS_ASSERT(column_types.size() != 0); - size_t offset = 0; - size_t level = column_types.size() > 0 ? column_types.back().level: 0; - if (level > 0) - { - for (auto it = column_types.rbegin(); - it != column_types.rend() && level == it->level; - ++it) - { - ++offset; - } - } - else - { - offset = 1; - } - column_types.emplace_back(csv_column_type::repeat_t,depth,offset); - ++p; - break; - } - default: - buffer.clear(); - state = column_state::label; - break; - } - break; - } - case column_state::label: - { - switch (*p) - { - case '*': - { - auto it = type_dictionary.find(buffer); - if (it != type_dictionary.end()) - { - column_types.emplace_back(it->second,depth); - buffer.clear(); - } - else - { - JSONCONS_ASSERT(false); - } - state = column_state::sequence; - break; - } - case ',': - { - auto it = type_dictionary.find(buffer); - if (it != type_dictionary.end()) - { - column_types.emplace_back(it->second,depth); - buffer.clear(); - } - else - { - JSONCONS_ASSERT(false); - } - ++p; - state = column_state::sequence; - break; - } - case ']': - { - JSONCONS_ASSERT(depth > 0); - auto it = type_dictionary.find(buffer); - if (it != type_dictionary.end()) - { - column_types.emplace_back(it->second,depth); - buffer.clear(); - } - else - { - JSONCONS_ASSERT(false); - } - --depth; - ++p; - state = column_state::sequence; - break; - } - default: - { - buffer.push_back(*p); - ++p; - break; - } - } - break; - } - } - } - if (state == column_state::label) - { - auto it = type_dictionary.find(buffer); - if (it != type_dictionary.end()) - { - column_types.emplace_back(it->second,depth); - buffer.clear(); - } - else - { - JSONCONS_ASSERT(false); - } - } - return column_types; - } - -}; - -typedef basic_csv_options csv_options; -typedef basic_csv_options wcsv_options; - -#if !defined(JSONCONS_NO_DEPRECATED) -typedef basic_csv_options csv_parameters; -typedef basic_csv_options wcsv_parameters; -typedef basic_csv_options csv_serializing_options; -typedef basic_csv_options wcsv_serializing_options; -#endif - - -}} -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_parser.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_parser.hpp deleted file mode 100644 index 51d131d9c1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_parser.hpp +++ /dev/null @@ -1,1357 +0,0 @@ -// Copyright 2015 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_CSV_CSV_PARSER_HPP -#define JSONCONS_CSV_CSV_PARSER_HPP - -#include // std::allocator -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace jsoncons { namespace csv { - -enum class csv_mode_type -{ - initial, - header, - data, - subfields -}; - -enum class csv_state_type -{ - start, - comment, - expect_value, - between_fields, - quoted_string, - unquoted_string, - escaped_value, - minus, - zero, - integer, - fraction, - exp1, - exp2, - exp3, - done -}; - -template> -class basic_csv_parser : public ser_context -{ - typedef basic_string_view string_view_type; - typedef CharT char_type; - typedef Allocator allocator_type; - typedef typename std::allocator_traits:: template rebind_alloc char_allocator_type; - typedef std::basic_string,char_allocator_type> string_type; - typedef typename std::allocator_traits:: template rebind_alloc string_allocator_type; - typedef typename std::allocator_traits:: template rebind_alloc csv_mode_allocator_type; - typedef typename std::allocator_traits:: template rebind_alloc csv_type_info_allocator_type; - typedef typename std::allocator_traits:: template rebind_alloc> string_vector_allocator_type; - typedef basic_json json_type; - - static const int default_depth = 3; - - default_parse_error_handler default_err_handler_; - csv_state_type state_; - int top_; - std::vector stack_; - basic_json_content_handler& handler_; - parse_error_handler& err_handler_; - unsigned long column_; - unsigned long line_; - CharT prev_char_; - string_type value_buffer_; - int depth_; - const basic_csv_decode_options& options_; - std::vector column_names_; - std::vector,string_vector_allocator_type> column_values_; - std::vector column_types_; - std::vector column_defaults_; - size_t column_index_; - basic_json_fragment_filter filter_; - size_t level_; - size_t offset_; - jsoncons::detail::string_to_double to_double_; - std::vector> decoders_; - const CharT* begin_input_; - const CharT* input_end_; - const CharT* input_ptr_; - bool continue_; - -public: - basic_csv_parser(basic_json_content_handler& handler) - : basic_csv_parser(handler, basic_csv_options::default_options(), default_err_handler_) - { - } - - basic_csv_parser(basic_json_content_handler& handler, - const basic_csv_decode_options& options) - : basic_csv_parser(handler, options, default_err_handler_) - { - } - - basic_csv_parser(basic_json_content_handler& handler, - parse_error_handler& err_handler) - : basic_csv_parser(handler, basic_csv_options::default_options(), err_handler) - { - } - - basic_csv_parser(basic_json_content_handler& handler, - const basic_csv_decode_options& options, - parse_error_handler& err_handler) - : top_(-1), - stack_(default_depth), - handler_(handler), - err_handler_(err_handler), - options_(options), - filter_(handler), - level_(0), - offset_(0), - begin_input_(nullptr), - input_end_(nullptr), - input_ptr_(nullptr), - continue_(true) - { - depth_ = default_depth; - state_ = csv_state_type::start; - top_ = -1; - line_ = 1; - column_ = 0; - column_index_ = 0; - } - - ~basic_csv_parser() - { - } - - bool done() const - { - return state_ == csv_state_type::done; - } - - bool stopped() const - { - return !continue_; - } - - bool source_exhausted() const - { - return input_ptr_ == input_end_; - } - - const std::vector>& column_labels() const - { - return column_names_; - } - - void before_field() - { - switch (stack_[top_]) - { - case csv_mode_type::header: - if (options_.assume_header() && line_ == 1) - { - column_names_.push_back(value_buffer_); - } - break; - case csv_mode_type::data: - case csv_mode_type::subfields: - switch (options_.mapping()) - { - case mapping_type::n_objects: - if (!(options_.ignore_empty_values() && value_buffer_.size() == 0)) - { - if (column_index_ < column_names_.size() + offset_) - { - continue_ = handler_.name(column_names_[column_index_ - offset_], *this); - } - } - break; - default: - break; - } - break; - default: - break; - } - } - - void before_multi_valued_field() - { - push_mode(csv_mode_type::subfields); - switch (options_.mapping()) - { - case mapping_type::n_rows: - case mapping_type::n_objects: - continue_ = handler_.begin_array(semantic_tag::none, *this); - break; - case mapping_type::m_columns: - decoders_[column_index_].begin_array(semantic_tag::none, *this); - break; - default: - break; - } - } - - void after_field() - { - if (stack_[top_] == csv_mode_type::subfields) - { - pop_mode(csv_mode_type::subfields); - switch (options_.mapping()) - { - case mapping_type::n_rows: - case mapping_type::n_objects: - continue_ = handler_.end_array(*this); - break; - case mapping_type::m_columns: - decoders_[column_index_].end_array(*this); - break; - default: - break; - } - } - ++column_index_; - } - - void before_record() - { - offset_ = 0; - if (stack_[top_] == csv_mode_type::data) - { - switch (options_.mapping()) - { - case mapping_type::n_rows: - continue_ = handler_.begin_array(semantic_tag::none, *this); - break; - case mapping_type::n_objects: - continue_ = handler_.begin_object(semantic_tag::none, *this); - break; - case mapping_type::m_columns: - break; - default: - break; - } - } - } - - void after_record() - { - if (column_types_.size() > 0) - { - if (level_ > 0) - { - continue_ = handler_.end_array(*this); - level_ = 0; - } - } - if (stack_[top_] == csv_mode_type::header) - { - if (line_ >= options_.header_lines()) - { - flip(csv_mode_type::header, csv_mode_type::data); - } - column_values_.resize(column_names_.size()); - switch (options_.mapping()) - { - case mapping_type::n_rows: - if (column_names_.size() > 0) - { - continue_ = handler_.begin_array(semantic_tag::none, *this); - for (const auto& name : column_names_) - { - continue_ = handler_.string_value(name, semantic_tag::none, *this); - } - continue_ = handler_.end_array(*this); - } - break; - case mapping_type::m_columns: - for (size_t i = 0; i < column_names_.size(); ++i) - { - decoders_.push_back(json_decoder()); - decoders_.back().begin_array(semantic_tag::none, *this); - } - break; - default: - break; - } - } - else if (stack_[top_] == csv_mode_type::data || stack_[top_] == csv_mode_type::subfields) - { - switch (options_.mapping()) - { - case mapping_type::n_rows: - continue_ = handler_.end_array(*this); - break; - case mapping_type::n_objects: - continue_ = handler_.end_object(*this); - break; - default: - break; - } - } - column_index_ = 0; - } - - void reset() - { - push_mode(csv_mode_type::initial); - - for (auto name : options_.column_names()) - { - column_names_.emplace_back(name.data(),name.size()); - } - for (auto name : options_.column_types()) - { - column_types_.push_back(name); - } - for (auto name : options_.column_defaults()) - { - column_defaults_.emplace_back(name.data(), name.size()); - } - if (options_.header_lines() > 0) - { - push_mode(csv_mode_type::header); - } - else - { - push_mode(csv_mode_type::data); - } - if (options_.mapping() != mapping_type::m_columns) - { - continue_ = handler_.begin_array(semantic_tag::none, *this); - } - state_ = csv_state_type::expect_value; - column_index_ = 0; - prev_char_ = 0; - column_ = 1; - level_ = 0; - } - - void parse_some() - { - std::error_code ec; - parse_some(ec); - if (ec) - { - throw ser_error(ec,line_,column_); - } - } - - void parse_some(std::error_code& ec) - { - const CharT* local_input_end = input_end_; - - if (input_ptr_ == local_input_end && continue_) - { - switch (state_) - { - case csv_state_type::unquoted_string: - if (options_.trim_leading() || options_.trim_trailing()) - { - trim_string_buffer(options_.trim_leading(),options_.trim_trailing()); - } - if (!options_.ignore_empty_lines() || (column_index_ > 0 || value_buffer_.length() > 0)) - { - if (column_index_ == 0) - { - before_record(); - } - if (stack_[top_] != csv_mode_type::subfields) - { - before_field(); - } - end_unquoted_string_value(); - after_field(); - } - break; - case csv_state_type::escaped_value: - if (options_.quote_escape_char() == options_.quote_char()) - { - if (column_index_ == 0) - { - before_record(); - } - if (stack_[top_] != csv_mode_type::subfields) - { - before_field(); - } - end_quoted_string_value(ec); - if (ec) return; - after_field(); - } - break; - default: - break; - } - if (column_index_ > 0) - { - after_record(); - } - switch (stack_[top_]) - { - case csv_mode_type::header: - pop_mode(csv_mode_type::header); - break; - case csv_mode_type::data: - pop_mode(csv_mode_type::data); - break; - default: - break; - } - if (options_.mapping() == mapping_type::m_columns) - { - basic_json_fragment_filter fragment_filter(handler_); - continue_ = handler_.begin_object(semantic_tag::none, *this); - for (size_t i = 0; i < column_names_.size(); ++i) - { - continue_ = handler_.name(column_names_[i],*this); - decoders_[i].end_array(*this); - decoders_[i].flush(); - decoders_[i].get_result().dump(fragment_filter); - } - continue_ = handler_.end_object(*this); - } - else - { - continue_ = handler_.end_array(*this); - } - if (!pop_mode(csv_mode_type::initial)) - { - err_handler_.fatal_error(csv_errc::unexpected_eof, *this); - ec = csv_errc::unexpected_eof; - continue_ = false; - return; - } - handler_.flush(); - continue_ = false; - } - - for (; (input_ptr_ < local_input_end) && continue_; ++input_ptr_) - { - CharT curr_char = *input_ptr_; -all_csv_states: - switch (state_) - { - case csv_state_type::comment: - if (curr_char == '\n') - { - state_ = csv_state_type::expect_value; - } - else if (prev_char_ == '\r') - { - state_ = csv_state_type::expect_value; - goto all_csv_states; - } - break; - case csv_state_type::expect_value: - if (column_ == 1 && curr_char == options_.comment_starter()) - { - state_ = csv_state_type::comment; - } - else - { - state_ = csv_state_type::unquoted_string; - goto all_csv_states; - } - break; - case csv_state_type::escaped_value: - { - if (curr_char == options_.quote_char()) - { - value_buffer_.push_back(static_cast(curr_char)); - state_ = csv_state_type::quoted_string; - } - else if (options_.quote_escape_char() == options_.quote_char()) - { - state_ = csv_state_type::between_fields; - goto all_csv_states; - } - } - break; - case csv_state_type::quoted_string: - { - if (curr_char == options_.quote_escape_char()) - { - state_ = csv_state_type::escaped_value; - } - else if (curr_char == options_.quote_char()) - { - state_ = csv_state_type::between_fields; - } - else - { - value_buffer_.push_back(static_cast(curr_char)); - } - } - break; - case csv_state_type::between_fields: - if (prev_char_ == '\r' && curr_char == '\n') - { - } - else if (curr_char == '\r' || curr_char == '\n') - { - if (options_.trim_leading() || options_.trim_trailing()) - { - trim_string_buffer(options_.trim_leading(),options_.trim_trailing()); - } - if (!options_.ignore_empty_lines() || (column_index_ > 0 || value_buffer_.length() > 0)) - { - if (column_index_ == 0) - { - before_record(); - } - if (stack_[top_] != csv_mode_type::subfields) - { - before_field(); - } - end_quoted_string_value(ec); - if (ec) return; - after_field(); - after_record(); - } - state_ = csv_state_type::expect_value; - } - else if (curr_char == options_.field_delimiter() || (options_.subfield_delimiter().second && curr_char == options_.subfield_delimiter().first)) - { - if (column_index_ == 0 && stack_[top_] != csv_mode_type::subfields) - { - before_record(); - } - if (options_.trim_leading() || options_.trim_trailing()) - { - trim_string_buffer(options_.trim_leading(),options_.trim_trailing()); - } - if (stack_[top_] != csv_mode_type::subfields) - { - before_field(); - if (options_.subfield_delimiter().second && curr_char == options_.subfield_delimiter().first) - { - before_multi_valued_field(); - } - } - end_quoted_string_value(ec); - if (ec) return; - if (curr_char == options_.field_delimiter()) - { - after_field(); - } - state_ = csv_state_type::unquoted_string; - } - break; - case csv_state_type::unquoted_string: - { - if (prev_char_ == '\r' && curr_char == '\n') - { - } - else if (curr_char == '\r' || curr_char == '\n') - { - if (options_.trim_leading() || options_.trim_trailing()) - { - trim_string_buffer(options_.trim_leading(),options_.trim_trailing()); - } - if (!options_.ignore_empty_lines() || (column_index_ > 0 || value_buffer_.length() > 0)) - { - if (column_index_ == 0) - { - before_record(); - } - if (stack_[top_] != csv_mode_type::subfields) - { - before_field(); - } - end_unquoted_string_value(); - after_field(); - after_record(); - } - state_ = csv_state_type::expect_value; - } - else if (curr_char == options_.field_delimiter() || (options_.subfield_delimiter().second && curr_char == options_.subfield_delimiter().first)) - { - if (column_index_ == 0 && stack_[top_] != csv_mode_type::subfields) - { - before_record(); - } - if (options_.trim_leading() || options_.trim_trailing()) - { - trim_string_buffer(options_.trim_leading(),options_.trim_trailing()); - } - if (stack_[top_] != csv_mode_type::subfields) - { - before_field(); - if (options_.subfield_delimiter().second && curr_char == options_.subfield_delimiter().first) - { - before_multi_valued_field(); - } - } - end_unquoted_string_value(); - if (curr_char == options_.field_delimiter()) - { - after_field(); - } - state_ = csv_state_type::unquoted_string; - } - else if (curr_char == options_.quote_char()) - { - value_buffer_.clear(); - state_ = csv_state_type::quoted_string; - } - else - { - value_buffer_.push_back(static_cast(curr_char)); - } - } - break; - default: - err_handler_.fatal_error(csv_errc::invalid_state, *this); - ec = csv_errc::invalid_state; - continue_ = false; - return; - } - if (line_ > options_.max_lines()) - { - state_ = csv_state_type::done; - continue_ = false; - } - switch (curr_char) - { - case '\r': - ++line_; - column_ = 1; - break; - case '\n': - if (prev_char_ != '\r') - { - ++line_; - } - column_ = 1; - break; - default: - ++column_; - break; - } - prev_char_ = curr_char; - } - } - - void finish_parse() - { - std::error_code ec; - finish_parse(ec); - if (ec) - { - throw ser_error(ec,line_,column_); - } - } - - void finish_parse(std::error_code& ec) - { - while (continue_) - { - parse_some(ec); - } - } - - csv_state_type state() const - { - return state_; - } - - void update(const string_view_type sv) - { - update(sv.data(),sv.length()); - } - - void update(const CharT* data, size_t length) - { - begin_input_ = data; - input_end_ = data + length; - input_ptr_ = begin_input_; - } - - size_t line() const override - { - return line_; - } - - size_t column() const override - { - return column_; - } -private: - - void trim_string_buffer(bool trim_leading, bool trim_trailing) - { - size_t start = 0; - size_t length = value_buffer_.length(); - if (trim_leading) - { - bool done = false; - while (!done && start < value_buffer_.length()) - { - if ((value_buffer_[start] < 256) && std::isspace(value_buffer_[start])) - { - ++start; - } - else - { - done = true; - } - } - } - if (trim_trailing) - { - bool done = false; - while (!done && length > 0) - { - if ((value_buffer_[length-1] < 256) && std::isspace(value_buffer_[length-1])) - { - --length; - } - else - { - done = true; - } - } - } - if (start != 0 || length != value_buffer_.size()) - { - value_buffer_ = value_buffer_.substr(start,length-start); - } - } - - void end_unquoted_string_value() - { - switch (stack_[top_]) - { - case csv_mode_type::data: - case csv_mode_type::subfields: - switch (options_.mapping()) - { - case mapping_type::n_rows: - if (options_.unquoted_empty_value_is_null() && value_buffer_.length() == 0) - { - continue_ = handler_.null_value(semantic_tag::none, *this); - } - else - { - end_value(value_buffer_,column_index_,options_.infer_types(),handler_); - } - break; - case mapping_type::n_objects: - if (!(options_.ignore_empty_values() && value_buffer_.size() == 0)) - { - if (column_index_ < column_names_.size() + offset_) - { - if (options_.unquoted_empty_value_is_null() && value_buffer_.length() == 0) - { - continue_ = handler_.null_value(semantic_tag::none, *this); - } - else - { - end_value(value_buffer_,column_index_,options_.infer_types(),handler_); - } - } - else if (level_ > 0) - { - if (options_.unquoted_empty_value_is_null() && value_buffer_.length() == 0) - { - continue_ = handler_.null_value(semantic_tag::none, *this); - } - else - { - end_value(value_buffer_,column_index_,options_.infer_types(),handler_); - } - } - } - break; - case mapping_type::m_columns: - if (column_index_ < decoders_.size()) - { - if (!(options_.ignore_empty_values() && value_buffer_.size() == 0)) - { - end_value(value_buffer_,column_index_,options_.infer_types(),decoders_[column_index_]); - } - } - break; - } - break; - default: - break; - } - state_ = csv_state_type::expect_value; - value_buffer_.clear(); - } - - void end_quoted_string_value(std::error_code& ec) - { - if (options_.trim_leading_inside_quotes() | options_.trim_trailing_inside_quotes()) - { - trim_string_buffer(options_.trim_leading_inside_quotes(),options_.trim_trailing_inside_quotes()); - } - switch (stack_[top_]) - { - case csv_mode_type::header: - break; - case csv_mode_type::data: - case csv_mode_type::subfields: - switch (options_.mapping()) - { - case mapping_type::n_rows: - end_value(value_buffer_,column_index_,false,handler_); - break; - case mapping_type::n_objects: - if (!(options_.ignore_empty_values() && value_buffer_.size() == 0)) - { - if (column_index_ < column_names_.size() + offset_) - { - if (options_.unquoted_empty_value_is_null() && value_buffer_.length() == 0) - { - continue_ = handler_.null_value(semantic_tag::none, *this); - } - else - { - end_value(value_buffer_,column_index_,false,handler_); - } - } - else if (level_ > 0) - { - if (options_.unquoted_empty_value_is_null() && value_buffer_.length() == 0) - { - continue_ = handler_.null_value(semantic_tag::none, *this); - } - else - { - end_value(value_buffer_,column_index_,false,handler_); - } - } - } - break; - case mapping_type::m_columns: - if (column_index_ < decoders_.size()) - { - end_value(value_buffer_,column_index_,options_.infer_types(),decoders_[column_index_]); - } - break; - } - break; - default: - err_handler_.fatal_error(csv_errc::invalid_csv_text, *this); - ec = csv_errc::invalid_csv_text; - continue_ = false; - return; - } - state_ = csv_state_type::expect_value; - value_buffer_.clear(); - } - - void end_value(const string_view_type& value, - size_t column_index, - bool infer_types, - basic_json_content_handler& handler) - { - if (column_index < column_types_.size() + offset_) - { - if (column_types_[column_index - offset_].col_type == csv_column_type::repeat_t) - { - offset_ = offset_ + column_types_[column_index - offset_].rep_count; - if (column_index - offset_ + 1 < column_types_.size()) - { - if (column_index == offset_ || level_ > column_types_[column_index-offset_].level) - { - handler.end_array(*this); - } - level_ = column_index == offset_ ? 0 : column_types_[column_index - offset_].level; - } - } - if (level_ < column_types_[column_index - offset_].level) - { - handler.begin_array(semantic_tag::none, *this); - level_ = column_types_[column_index - offset_].level; - } - else if (level_ > column_types_[column_index - offset_].level) - { - handler.end_array(*this); - level_ = column_types_[column_index - offset_].level; - } - switch (column_types_[column_index - offset_].col_type) - { - case csv_column_type::integer_t: - { - std::istringstream iss{ std::string(value) }; - int64_t val; - iss >> val; - if (!iss.fail()) - { - handler.int64_value(val, semantic_tag::none, *this); - } - else - { - if (column_index - offset_ < column_defaults_.size() && column_defaults_[column_index - offset_].length() > 0) - { - basic_json_parser parser(err_handler_); - parser.update(column_defaults_[column_index - offset_].data(),column_defaults_[column_index - offset_].length()); - parser.parse_some(filter_); - parser.finish_parse(filter_); - } - else - { - handler.null_value(semantic_tag::none, *this); - } - } - } - break; - case csv_column_type::float_t: - { - if (options_.lossless_number()) - { - handler.string_value(value,semantic_tag::bigdec, *this); - } - else - { - std::istringstream iss{ std::string(value) }; - double val; - iss >> val; - if (!iss.fail()) - { - handler.double_value(val, semantic_tag::none, *this); - } - else - { - if (column_index - offset_ < column_defaults_.size() && column_defaults_[column_index - offset_].length() > 0) - { - basic_json_parser parser(err_handler_); - parser.update(column_defaults_[column_index - offset_].data(),column_defaults_[column_index - offset_].length()); - parser.parse_some(filter_); - parser.finish_parse(filter_); - } - else - { - handler.null_value(semantic_tag::none, *this); - } - } - } - } - break; - case csv_column_type::boolean_t: - { - if (value.length() == 1 && value[0] == '0') - { - handler.bool_value(false, semantic_tag::none, *this); - } - else if (value.length() == 1 && value[0] == '1') - { - handler.bool_value(true, semantic_tag::none, *this); - } - else if (value.length() == 5 && ((value[0] == 'f' || value[0] == 'F') && (value[1] == 'a' || value[1] == 'A') && (value[2] == 'l' || value[2] == 'L') && (value[3] == 's' || value[3] == 'S') && (value[4] == 'e' || value[4] == 'E'))) - { - handler.bool_value(false, semantic_tag::none, *this); - } - else if (value.length() == 4 && ((value[0] == 't' || value[0] == 'T') && (value[1] == 'r' || value[1] == 'R') && (value[2] == 'u' || value[2] == 'U') && (value[3] == 'e' || value[3] == 'E'))) - { - handler.bool_value(true, semantic_tag::none, *this); - } - else - { - if (column_index - offset_ < column_defaults_.size() && column_defaults_[column_index - offset_].length() > 0) - { - basic_json_parser parser(err_handler_); - parser.update(column_defaults_[column_index - offset_].data(),column_defaults_[column_index - offset_].length()); - parser.parse_some(filter_); - parser.finish_parse(filter_); - } - else - { - handler.null_value(semantic_tag::none, *this); - } - } - } - break; - default: - if (value.length() > 0) - { - handler.string_value(value, semantic_tag::none, *this); - } - else - { - if (column_index < column_defaults_.size() + offset_ && column_defaults_[column_index - offset_].length() > 0) - { - basic_json_parser parser(err_handler_); - parser.update(column_defaults_[column_index - offset_].data(),column_defaults_[column_index - offset_].length()); - parser.parse_some(filter_); - parser.finish_parse(filter_); - } - else - { - handler.string_value(string_view_type(), semantic_tag::none, *this); - } - } - break; - } - } - else - { - if (infer_types) - { - end_value_with_numeric_check(value, handler); - } - else - { - handler.string_value(value, semantic_tag::none, *this); - } - } - } - - enum class numeric_check_state - { - initial, - null, - boolean_true, - boolean_false, - minus, - zero, - integer, - fraction1, - fraction, - exp1, - exp, - done - }; - - void end_value_with_numeric_check(const string_view_type& value, basic_json_content_handler& handler) - { - numeric_check_state state = numeric_check_state::initial; - bool is_negative = false; - int precision = 0; - uint8_t decimal_places = 0; - - auto last = value.end(); - - std::string buffer; - for (auto p = value.begin(); state != numeric_check_state::done && p != last; ++p) - { - switch (state) - { - case numeric_check_state::initial: - { - switch (*p) - { - case 'n':case 'N': - if ((last-p) == 4 && (p[1] == 'u' || p[1] == 'U') && (p[2] == 'l' || p[2] == 'L') && (p[3] == 'l' || p[3] == 'L')) - { - state = numeric_check_state::null; - } - else - { - state = numeric_check_state::done; - } - break; - case 't':case 'T': - if ((last-p) == 4 && (p[1] == 'r' || p[1] == 'R') && (p[2] == 'u' || p[2] == 'U') && (p[3] == 'e' || p[3] == 'U')) - { - state = numeric_check_state::boolean_true; - } - else - { - state = numeric_check_state::done; - } - break; - case 'f':case 'F': - if ((last-p) == 5 && (p[1] == 'a' || p[1] == 'A') && (p[2] == 'l' || p[2] == 'L') && (p[3] == 's' || p[3] == 'S') && (p[4] == 'e' || p[4] == 'E')) - { - state = numeric_check_state::boolean_false; - } - else - { - state = numeric_check_state::done; - } - break; - case '-': - is_negative = true; - buffer.push_back(*p); - state = numeric_check_state::minus; - break; - case '0': - ++precision; - buffer.push_back(*p); - state = numeric_check_state::zero; - break; - case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8':case '9': - ++precision; - buffer.push_back(*p); - state = numeric_check_state::integer; - break; - default: - state = numeric_check_state::done; - break; - } - break; - } - case numeric_check_state::zero: - { - switch (*p) - { - case '.': - buffer.push_back(to_double_.get_decimal_point()); - state = numeric_check_state::fraction1; - break; - case 'e':case 'E': - buffer.push_back(*p); - state = numeric_check_state::exp1; - break; - default: - state = numeric_check_state::done; - break; - } - break; - } - case numeric_check_state::integer: - { - switch (*p) - { - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8':case '9': - ++precision; - buffer.push_back(*p); - break; - case '.': - buffer.push_back(to_double_.get_decimal_point()); - state = numeric_check_state::fraction1; - break; - case 'e':case 'E': - buffer.push_back(*p); - state = numeric_check_state::exp1; - break; - default: - state = numeric_check_state::done; - break; - } - break; - } - case numeric_check_state::minus: - { - switch (*p) - { - case '0': - ++precision; - buffer.push_back(*p); - state = numeric_check_state::zero; - break; - case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8':case '9': - ++precision; - buffer.push_back(*p); - state = numeric_check_state::integer; - break; - case 'e':case 'E': - buffer.push_back(*p); - state = numeric_check_state::exp1; - break; - default: - state = numeric_check_state::done; - break; - } - break; - } - case numeric_check_state::fraction1: - { - switch (*p) - { - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8':case '9': - ++precision; - ++decimal_places; - buffer.push_back(*p); - state = numeric_check_state::fraction; - break; - default: - state = numeric_check_state::done; - break; - } - break; - } - case numeric_check_state::fraction: - { - switch (*p) - { - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8':case '9': - ++precision; - ++decimal_places; - buffer.push_back(*p); - break; - case 'e':case 'E': - buffer.push_back(*p); - state = numeric_check_state::exp1; - break; - default: - state = numeric_check_state::done; - break; - } - break; - } - case numeric_check_state::exp1: - { - switch (*p) - { - case '-': - buffer.push_back(*p); - state = numeric_check_state::exp; - break; - case '+': - state = numeric_check_state::exp; - break; - case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8':case '9': - buffer.push_back(*p); - state = numeric_check_state::integer; - break; - default: - state = numeric_check_state::done; - break; - } - break; - } - case numeric_check_state::exp: - { - switch (*p) - { - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8':case '9': - buffer.push_back(*p); - break; - default: - state = numeric_check_state::done; - break; - } - break; - } - default: - break; - } - } - - switch (state) - { - case numeric_check_state::null: - handler.null_value(semantic_tag::none, *this); - break; - case numeric_check_state::boolean_true: - handler.bool_value(true, semantic_tag::none, *this); - break; - case numeric_check_state::boolean_false: - handler.bool_value(false, semantic_tag::none, *this); - break; - case numeric_check_state::zero: - case numeric_check_state::integer: - { - if (is_negative) - { - auto result = jsoncons::detail::to_integer(value.data(), value.length()); - if (result.ec == jsoncons::detail::to_integer_errc()) - { - handler.int64_value(result.value, semantic_tag::none, *this); - } - else // Must be overflow - { - handler.string_value(value, semantic_tag::bigint, *this); - } - } - else - { - auto result = jsoncons::detail::to_integer(value.data(), value.length()); - if (result.ec == jsoncons::detail::to_integer_errc()) - { - handler.uint64_value(result.value, semantic_tag::none, *this); - } - else if (result.ec == jsoncons::detail::to_integer_errc::overflow) - { - handler.string_value(value, semantic_tag::bigint, *this); - } - else - { - JSONCONS_THROW(json_runtime_error(make_error_code(result.ec).message())); - } - } - break; - } - case numeric_check_state::fraction: - case numeric_check_state::exp: - { - if (options_.lossless_number()) - { - handler.string_value(value,semantic_tag::bigdec, *this); - } - else - { - double d = to_double_(buffer.c_str(), buffer.length()); - handler.double_value(d, semantic_tag::none, *this); - } - break; - } - default: - handler.string_value(value, semantic_tag::none, *this); - } - } - - void push_mode(csv_mode_type mode) - { - ++top_; - if (top_ >= depth_) - { - depth_ *= 2; - stack_.resize(depth_); - } - stack_[top_] = mode; - } - - int peek() - { - return stack_[top_]; - } - - bool peek(csv_mode_type mode) - { - return stack_[top_] == mode; - } - - bool flip(csv_mode_type mode1, csv_mode_type mode2) - { - if (top_ < 0 || stack_[top_] != mode1) - { - return false; - } - stack_[top_] = mode2; - return true; - } - - bool pop_mode(csv_mode_type mode) - { - if (top_ < 0 || stack_[top_] != mode) - { - return false; - } - --top_; - return true; - } -}; - -typedef basic_csv_parser csv_parser; -typedef basic_csv_parser wcsv_parser; - -}} - -#endif - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_reader.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_reader.hpp deleted file mode 100644 index 46d94a50b1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_reader.hpp +++ /dev/null @@ -1,328 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_CSV_CSV_READER_HPP -#define JSONCONS_CSV_CSV_READER_HPP - -#include -#include -#include -#include // std::allocator -#include // std::move -#include // std::basic_istream -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace jsoncons { namespace csv { - -template,class Allocator=std::allocator> -class basic_csv_reader -{ - struct stack_item - { - stack_item() - : array_begun_(false) - { - } - - bool array_begun_; - }; - typedef CharT char_type; - typedef Allocator allocator_type; - typedef typename std::allocator_traits:: template rebind_alloc char_allocator_type; - - basic_csv_reader(const basic_csv_reader&) = delete; - basic_csv_reader& operator = (const basic_csv_reader&) = delete; - - default_parse_error_handler default_err_handler_; - - basic_csv_parser parser_; - Src source_; - std::vector buffer_; - size_t buffer_length_; - bool eof_; - bool begin_; -public: - // Structural characters - static const size_t default_max_buffer_length = 16384; - //! Parse an input stream of CSV text into a json object - /*! - \param is The input stream to read from - */ - - template - basic_csv_reader(Source&& source, - basic_json_content_handler& handler) - - : basic_csv_reader(std::forward(source), - handler, - basic_csv_options::default_options(), - default_err_handler_) - { - } - - template - basic_csv_reader(Source&& source, - basic_json_content_handler& handler, - const basic_csv_options& options) - - : basic_csv_reader(std::forward(source), - handler, - options, - default_err_handler_) - { - } - - template - basic_csv_reader(Source&& source, - basic_json_content_handler& handler, - parse_error_handler& err_handler) - : basic_csv_reader(std::forward(source), - handler, - basic_csv_options::default_options(), - err_handler) - { - } - - template - basic_csv_reader(Source&& source, - basic_json_content_handler& handler, - const basic_csv_decode_options& options, - parse_error_handler& err_handler, - typename std::enable_if,Source>::value>::type* = 0) - : - parser_(handler, options, err_handler), - source_(std::forward(source)), - buffer_length_(default_max_buffer_length), - eof_(false), - begin_(true) - { - buffer_.reserve(buffer_length_); - } - - template - basic_csv_reader(Source&& source, - basic_json_content_handler& handler, - const basic_csv_decode_options& options, - parse_error_handler& err_handler, - typename std::enable_if,Source>::value>::type* = 0) - : - parser_(handler, options, err_handler), - buffer_length_(0), - eof_(false), - begin_(false) - { - basic_string_view sv(std::forward(source)); - auto result = unicons::skip_bom(sv.begin(), sv.end()); - if (result.ec != unicons::encoding_errc()) - { - throw ser_error(result.ec,parser_.line(),parser_.column()); - } - size_t offset = result.it - sv.begin(); - parser_.update(sv.data()+offset,sv.size()-offset); - } - - ~basic_csv_reader() - { - } - - void read() - { - std::error_code ec; - read(ec); - if (ec) - { - throw ser_error(ec,parser_.line(),parser_.column()); - } - } - - void read(std::error_code& ec) - { - try - { - read_internal(ec); - } - catch (const ser_error& e) - { - ec = e.code(); - } - } - - bool eof() const - { - return eof_; - } - - size_t buffer_length() const - { - return buffer_length_; - } - - void buffer_length(size_t length) - { - buffer_length_ = length; - buffer_.reserve(buffer_length_); - } - -#if !defined(JSONCONS_NO_DEPRECATED) - - size_t buffer_capacity() const - { - return buffer_length_; - } - - void buffer_capacity(size_t length) - { - buffer_length_ = length; - buffer_.reserve(buffer_length_); - } -#endif -private: - - void read_internal(std::error_code& ec) - { - if (source_.is_error()) - { - ec = csv_errc::source_error; - return; - } - parser_.reset(); - if (ec) - { - return; - } - while (!parser_.stopped()) - { - if (parser_.source_exhausted()) - { - if (!source_.eof()) - { - read_buffer(ec); - if (ec) - { - return; - } - /* buffer_.clear(); - buffer_.resize(buffer_length_); - size_t count = source_.read(buffer_.data(), buffer_length_); - buffer_.resize(count); - if (buffer_.size() == 0) - { - eof_ = true; - } - parser_.update(buffer_.data(),buffer_.size());*/ - } - else - { - parser_.update(buffer_.data(),0); - eof_ = true; - } - } - parser_.parse_some(ec); - if (ec) return; - } - } - void read_buffer(std::error_code& ec) - { - buffer_.clear(); - buffer_.resize(buffer_length_); - size_t count = source_.read(buffer_.data(), buffer_length_); - buffer_.resize(static_cast(count)); - if (buffer_.size() == 0) - { - eof_ = true; - } - else if (begin_) - { - auto result = unicons::skip_bom(buffer_.begin(), buffer_.end()); - if (result.ec != unicons::encoding_errc()) - { - ec = result.ec; - return; - } - size_t offset = result.it - buffer_.begin(); - parser_.update(buffer_.data()+offset,buffer_.size()-offset); - begin_ = false; - } - else - { - parser_.update(buffer_.data(),buffer_.size()); - } - } - -}; - -template -typename std::enable_if::value,T>::type -decode_csv(const std::basic_string& s, const basic_csv_options& options = basic_csv_options::default_options()) -{ - typedef CharT char_type; - - json_decoder decoder; - - basic_csv_reader> reader(s,decoder,options); - reader.read(); - return decoder.get_result(); -} - -template -typename std::enable_if::value,T>::type -decode_csv(const std::basic_string& s, const basic_csv_options& options = basic_csv_options::default_options()) -{ - typedef CharT char_type; - - json_decoder> decoder; - - basic_csv_reader> reader(s,decoder,options); - reader.read(); - return decoder.get_result().template as(); -} - -template -typename std::enable_if::value,T>::type -decode_csv(std::basic_istream& is, const basic_csv_options& options = basic_csv_options::default_options()) -{ - typedef CharT char_type; - - json_decoder decoder; - - basic_csv_reader> reader(is,decoder,options); - reader.read(); - return decoder.get_result(); -} - -template -typename std::enable_if::value,T>::type -decode_csv(std::basic_istream& is, const basic_csv_options& options = basic_csv_options::default_options()) -{ - typedef CharT char_type; - - json_decoder> decoder; - - basic_csv_reader> reader(is,decoder,options); - reader.read(); - return decoder.get_result().template as(); -} - -typedef basic_csv_reader csv_reader; -typedef basic_csv_reader wcsv_reader; - -#if !defined(JSONCONS_NO_DEPRECATED) -typedef basic_csv_reader> csv_string_reader; -typedef basic_csv_reader> wcsv_string_reader; -#endif - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_serializer.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_serializer.hpp deleted file mode 100644 index ec73510bf7..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/csv/csv_serializer.hpp +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_CSV_CSV_SERIALIZER_HPP -#define JSONCONS_CSV_CSV_SERIALIZER_HPP - -#include - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpatch/jsonpatch.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpatch/jsonpatch.hpp deleted file mode 100644 index 600dd9cffb..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpatch/jsonpatch.hpp +++ /dev/null @@ -1,492 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSONPOINTER_JSONPATCH_HPP -#define JSONCONS_JSONPOINTER_JSONPATCH_HPP - -#include -#include -#include -#include -#include // std::min -#include // std::move -#include -#include -#include - -namespace jsoncons { namespace jsonpatch { - -namespace detail { - - JSONCONS_STRING_LITERAL(test,'t','e','s','t') - JSONCONS_STRING_LITERAL(add,'a','d','d') - JSONCONS_STRING_LITERAL(remove,'r','e','m','o','v','e') - JSONCONS_STRING_LITERAL(replace,'r','e','p','l','a','c','e') - JSONCONS_STRING_LITERAL(move,'m','o','v','e') - JSONCONS_STRING_LITERAL(copy,'c','o','p','y') - JSONCONS_STRING_LITERAL(op,'o','p') - JSONCONS_STRING_LITERAL(path,'p','a','t','h') - JSONCONS_STRING_LITERAL(from,'f','r','o','m') - JSONCONS_STRING_LITERAL(value,'v','a','l','u','e') - - enum class op_type {add,remove,replace}; - enum class state_type {begin,abort,commit}; - - template - struct operation_unwinder - { - typedef typename Json::string_type string_type; - typedef typename Json::string_view_type string_view_type; - - struct entry - { - op_type op; - string_type path; - Json value; - }; - - Json& target; - state_type state; - std::vector stack; - - operation_unwinder(Json& j) - : target(j), state(state_type::begin) - { - } - - ~operation_unwinder() - { - std::error_code ec; - //std::cout << "state: " << std::boolalpha << (state == state_type::commit) << ", stack size: " << stack.size() << std::endl; - if (state != state_type::commit) - { - for (auto it = stack.rbegin(); it != stack.rend(); ++it) - { - if (it->op == op_type::add) - { - jsonpointer::insert_or_assign(target,it->path,it->value,ec); - if (ec) - { - //std::cout << "add: " << it->path << std::endl; - break; - } - } - else if (it->op == op_type::remove) - { - jsonpointer::remove(target,it->path,ec); - if (ec) - { - //std::cout << "remove: " << it->path << std::endl; - break; - } - } - else if (it->op == op_type::replace) - { - jsonpointer::replace(target,it->path,it->value,ec); - if (ec) - { - //std::cout << "replace: " << it->path << std::endl; - break; - } - } - } - } - } - }; - - template - Json from_diff(const Json& source, const Json& target, const typename Json::string_type& path) - { - typedef typename Json::char_type char_type; - - Json result = typename Json::array(); - - if (source == target) - { - return result; - } - - if (source.is_array() && target.is_array()) - { - size_t common = (std::min)(source.size(),target.size()); - for (size_t i = 0; i < common; ++i) - { - std::basic_ostringstream ss; - ss << path << '/' << i; - auto temp_diff = from_diff(source[i],target[i],ss.str()); - result.insert(result.array_range().end(),temp_diff.array_range().begin(),temp_diff.array_range().end()); - } - // Element in source, not in target - remove - for (size_t i = target.size(); i < source.size(); ++i) - { - std::basic_ostringstream ss; - ss << path << '/' << i; - Json val = typename Json::object(); - val.insert_or_assign(op_literal(), remove_literal()); - val.insert_or_assign(path_literal(), ss.str()); - result.push_back(std::move(val)); - } - // Element in target, not in source - add, - // Fix contributed by Alexander rog13 - for (size_t i = source.size(); i < target.size(); ++i) - { - const auto& a = target[i]; - std::basic_ostringstream ss; - ss << path << '/' << i; - Json val = typename Json::object(); - val.insert_or_assign(op_literal(), add_literal()); - val.insert_or_assign(path_literal(), ss.str()); - val.insert_or_assign(value_literal(), a); - result.push_back(std::move(val)); - } - } - else if (source.is_object() && target.is_object()) - { - for (const auto& a : source.object_range()) - { - std::basic_ostringstream ss; - ss << path << '/'; - jsonpointer::escape(a.key(),ss); - auto it = target.find(a.key()); - if (it != target.object_range().end()) - { - auto temp_diff = from_diff(a.value(),it->value(),ss.str()); - result.insert(result.array_range().end(),temp_diff.array_range().begin(),temp_diff.array_range().end()); - } - else - { - Json val = typename Json::object(); - val.insert_or_assign(op_literal(), remove_literal()); - val.insert_or_assign(path_literal(), ss.str()); - result.push_back(std::move(val)); - } - } - for (const auto& a : target.object_range()) - { - auto it = source.find(a.key()); - if (it == source.object_range().end()) - { - std::basic_ostringstream ss; - ss << path << '/'; - jsonpointer::escape(a.key(),ss); - Json val = typename Json::object(); - val.insert_or_assign(op_literal(), add_literal()); - val.insert_or_assign(path_literal(), ss.str()); - val.insert_or_assign(value_literal(), a.value()); - result.push_back(std::move(val)); - } - } - } - else - { - Json val = typename Json::object(); - val.insert_or_assign(op_literal(), replace_literal()); - val.insert_or_assign(path_literal(), path); - val.insert_or_assign(value_literal(), target); - result.push_back(std::move(val)); - } - - return result; - } -} - -template -void apply_patch(Json& target, const Json& patch, std::error_code& patch_ec) -{ - typedef typename Json::char_type char_type; - typedef typename Json::string_type string_type; - typedef typename Json::string_view_type string_view_type; - - jsoncons::jsonpatch::detail::operation_unwinder unwinder(target); - - // Validate - - string_type bad_path; - for (const auto& operation : patch.array_range()) - { - unwinder.state =jsoncons::jsonpatch::detail::state_type::begin; - - if (operation.count(detail::op_literal()) != 1 || operation.count(detail::path_literal()) != 1) - { - patch_ec = jsonpatch_errc::invalid_patch; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - else - { - string_view_type op = operation.at(detail::op_literal()).as_string_view(); - string_view_type path = operation.at(detail::path_literal()).as_string_view(); - - if (op ==jsoncons::jsonpatch::detail::test_literal()) - { - std::error_code ec; - Json val = jsonpointer::get(target,path,ec); - if (ec) - { - patch_ec = jsonpatch_errc::test_failed; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - else if (operation.count(detail::value_literal()) != 1) - { - patch_ec = jsonpatch_errc::invalid_patch; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - else if (val != operation.at(detail::value_literal())) - { - patch_ec = jsonpatch_errc::test_failed; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - } - else if (op ==jsoncons::jsonpatch::detail::add_literal()) - { - if (operation.count(detail::value_literal()) != 1) - { - patch_ec = jsonpatch_errc::invalid_patch; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - else - { - std::error_code insert_ec; - Json val = operation.at(detail::value_literal()); - auto npath = jsonpointer::normalized_path(target,path); - jsonpointer::insert(target,npath,val,insert_ec); // try insert without replace - if (insert_ec) // try a replace - { - std::error_code select_ec; - Json orig_val = jsonpointer::get(target,npath,select_ec); - if (select_ec) // shouldn't happen - { - patch_ec = jsonpatch_errc::add_failed; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - else - { - std::error_code replace_ec; - jsonpointer::replace(target,npath,val,replace_ec); - if (replace_ec) - { - patch_ec = jsonpatch_errc::add_failed; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - else - { - unwinder.stack.push_back({detail::op_type::replace,npath,orig_val}); - } - } - } - else // insert without replace succeeded - { - unwinder.stack.push_back({detail::op_type::remove,npath,Json::null()}); - } - } - } - else if (op ==jsoncons::jsonpatch::detail::remove_literal()) - { - std::error_code ec; - Json val = jsonpointer::get(target,path,ec); - if (ec) - { - patch_ec = jsonpatch_errc::remove_failed; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - else - { - jsonpointer::remove(target,path,ec); - if (ec) - { - patch_ec = jsonpatch_errc::remove_failed; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - else - { - unwinder.stack.push_back({detail::op_type::add,string_type(path),val}); - } - } - } - else if (op ==jsoncons::jsonpatch::detail::replace_literal()) - { - std::error_code ec; - Json val = jsonpointer::get(target,path,ec); - if (ec) - { - patch_ec = jsonpatch_errc::replace_failed; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - else if (operation.count(detail::value_literal()) != 1) - { - patch_ec = jsonpatch_errc::invalid_patch; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - else - { - jsonpointer::replace(target,path,operation.at(detail::value_literal()),ec); - if (ec) - { - patch_ec = jsonpatch_errc::replace_failed; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - else - { - unwinder.stack.push_back({detail::op_type::replace,string_type(path),val}); - } - } - } - else if (op ==jsoncons::jsonpatch::detail::move_literal()) - { - if (operation.count(detail::from_literal()) != 1) - { - patch_ec = jsonpatch_errc::invalid_patch; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - else - { - string_view_type from = operation.at(detail::from_literal()).as_string_view(); - std::error_code ec; - Json val = jsonpointer::get(target,from,ec); - if (ec) - { - patch_ec = jsonpatch_errc::move_failed; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - else - { - jsonpointer::remove(target,from,ec); - if (ec) - { - patch_ec = jsonpatch_errc::move_failed; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - else - { - unwinder.stack.push_back({detail::op_type::add,string_type(from),val}); - // add - std::error_code insert_ec; - auto npath = jsonpointer::normalized_path(target,path); - jsonpointer::insert(target,npath,val,insert_ec); // try insert without replace - if (insert_ec) // try a replace - { - std::error_code select_ec; - Json orig_val = jsonpointer::get(target,npath,select_ec); - if (select_ec) // shouldn't happen - { - patch_ec = jsonpatch_errc::copy_failed; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - else - { - std::error_code replace_ec; - jsonpointer::replace(target, npath, val, replace_ec); - if (replace_ec) - { - patch_ec = jsonpatch_errc::copy_failed; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - - } - else - { - unwinder.stack.push_back({jsoncons::jsonpatch::detail::op_type::replace,npath,orig_val }); - } - - } - } - else - { - unwinder.stack.push_back({detail::op_type::remove,npath,Json::null()}); - } - } - } - } - } - else if (op ==jsoncons::jsonpatch::detail::copy_literal()) - { - if (operation.count(detail::from_literal()) != 1) - { - patch_ec = jsonpatch_errc::invalid_patch; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - else - { - std::error_code ec; - string_view_type from = operation.at(detail::from_literal()).as_string_view(); - Json val = jsonpointer::get(target,from,ec); - if (ec) - { - patch_ec = jsonpatch_errc::copy_failed; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - else - { - // add - auto npath = jsonpointer::normalized_path(target,path); - std::error_code insert_ec; - jsonpointer::insert(target,npath,val,insert_ec); // try insert without replace - if (insert_ec) // Failed, try a replace - { - std::error_code select_ec; - Json orig_val = jsonpointer::get(target,npath, select_ec); - if (select_ec) // shouldn't happen - { - patch_ec = jsonpatch_errc::copy_failed; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - else - { - std::error_code replace_ec; - jsonpointer::replace(target, npath, val,replace_ec); - if (replace_ec) - { - patch_ec = jsonpatch_errc::copy_failed; - unwinder.state =jsoncons::jsonpatch::detail::state_type::abort; - } - else - { - unwinder.stack.push_back({jsoncons::jsonpatch::detail::op_type::replace,npath,orig_val }); - } - } - } - else - { - unwinder.stack.push_back({detail::op_type::remove,npath,Json::null()}); - } - } - } - } - if (unwinder.state !=jsoncons::jsonpatch::detail::state_type::begin) - { - bad_path = string_type(path); - } - } - if (unwinder.state !=jsoncons::jsonpatch::detail::state_type::begin) - { - break; - } - } - if (unwinder.state ==jsoncons::jsonpatch::detail::state_type::begin) - { - unwinder.state =jsoncons::jsonpatch::detail::state_type::commit; - } -} - -template -Json from_diff(const Json& source, const Json& target) -{ - typename Json::string_type path; - return jsoncons::jsonpatch::detail::from_diff(source, target, path); -} - -template -void apply_patch(Json& target, const Json& patch) -{ - std::error_code ec; - apply_patch(target, patch, ec); - if (ec) - { - JSONCONS_THROW(jsonpatch_error(ec)); - } -} - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpatch/jsonpatch_error.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpatch/jsonpatch_error.hpp deleted file mode 100644 index 731b25f816..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpatch/jsonpatch_error.hpp +++ /dev/null @@ -1,114 +0,0 @@ -/// Copyright 2017 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSONPATCH_JSONPATCH_ERROR_HPP -#define JSONCONS_JSONPATCH_JSONPATCH_ERROR_HPP - -#include -#include - -namespace jsoncons { namespace jsonpatch { - -class jsonpatch_error : public std::system_error, public virtual json_exception -{ -public: - jsonpatch_error(const std::error_code& ec) - : std::system_error(ec) - { - } - - jsonpatch_error(const std::error_code& ec, const std::string& what_arg) - : std::system_error(ec, what_arg) - { - } - - jsonpatch_error(const std::error_code& ec, const char* what_arg) - : std::system_error(ec, what_arg) - { - } - - jsonpatch_error(const jsonpatch_error& other) = default; - - jsonpatch_error(jsonpatch_error&& other) = default; - - const char* what() const noexcept override - { - return std::system_error::what(); - } - - jsonpatch_error& operator=(const jsonpatch_error& e) = default; - jsonpatch_error& operator=(jsonpatch_error&& e) = default; -private: -}; - -enum class jsonpatch_errc -{ - ok = 0, - invalid_patch = 1, - test_failed, - add_failed, - remove_failed, - replace_failed, - move_failed, - copy_failed - -}; - -class jsonpatch_error_category_impl - : public std::error_category -{ -public: - const char* name() const noexcept override - { - return "jsoncons/jsonpatch"; - } - std::string message(int ev) const override - { - switch (static_cast(ev)) - { - case jsonpatch_errc::invalid_patch: - return "Invalid JSON Patch document"; - case jsonpatch_errc::test_failed: - return "JSON Patch test operation failed"; - case jsonpatch_errc::add_failed: - return "JSON Patch add operation failed"; - case jsonpatch_errc::remove_failed: - return "JSON Patch remove operation failed"; - case jsonpatch_errc::replace_failed: - return "JSON Patch replace operation failed"; - case jsonpatch_errc::move_failed: - return "JSON Patch move operation failed"; - case jsonpatch_errc::copy_failed: - return "JSON Patch copy operation failed"; - default: - return "Unknown JSON Patch error"; - } - } -}; - -inline -const std::error_category& jsonpatch_error_category() -{ - static jsonpatch_error_category_impl instance; - return instance; -} - -inline -std::error_code make_error_code(jsonpatch_errc result) -{ - return std::error_code(static_cast(result),jsonpatch_error_category()); -} - -}} - -namespace std { - template<> - struct is_error_code_enum : public true_type - { - }; -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpath/json_query.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpath/json_query.hpp deleted file mode 100644 index 0c97f3371b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpath/json_query.hpp +++ /dev/null @@ -1,1808 +0,0 @@ -// Copyright 2013 Daniel Parkerpath_single_quoted -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSONPATH_JSON_QUERY_HPP -#define JSONCONS_JSONPATH_JSON_QUERY_HPP - -#include // std::array -#include -#include -#include -#include // std::is_const -#include // std::numeric_limits -#include // std::move -#include -#include // std::set -#include // std::make_move_iterator -#include -#include -#include -#include - -namespace jsoncons { namespace jsonpath { - -struct array_slice -{ - size_t start_; - bool is_start_positive; - size_t end_; - bool is_end_positive; - bool is_end_defined; - size_t step_; - bool is_step_positive; - - array_slice() - : start_(0), is_start_positive(true), - end_(0), is_end_positive(true), is_end_defined(false), - step_(1), is_step_positive(true) - { - } - - array_slice(size_t start, bool is_start_positive, - size_t end, bool is_end_positive, bool is_end_defined, - size_t step, bool is_step_positive) - : start_(start), is_start_positive(is_start_positive), - end_(end), is_end_positive(is_end_positive), is_end_defined(is_end_defined), - step_(step), is_step_positive(is_step_positive) - { - } - - size_t get_start(size_t size) const - { - return is_start_positive ? start_ : size - start_; - } - - size_t get_end(size_t size) const - { - if (is_end_defined) - { - return is_end_positive ? end_ : size - end_; - } - else - { - return size; - } - } - - size_t step() const - { - return step_; - } - - array_slice(const array_slice&) = default; - - array_slice& operator=(const array_slice&) = default; -}; - -// work around for std::make_unique not being available until C++14 -template -std::unique_ptr make_unique_ptr(Args&&... args) -{ - return std::unique_ptr(new T(std::forward(args)...)); -} - -enum class result_type {value,path}; - -template -Json json_query(const Json& root, const typename Json::string_view_type& path, result_type result_t = result_type::value) -{ - if (result_t == result_type::value) - { - jsoncons::jsonpath::detail::jsonpath_evaluator> evaluator; - evaluator.evaluate(root, path); - return evaluator.get_values(); - } - else - { - jsoncons::jsonpath::detail::jsonpath_evaluator> evaluator; - evaluator.evaluate(root, path); - return evaluator.get_normalized_paths(); - } -} - -template -void json_replace(Json& root, const typename Json::string_view_type& path, T&& new_value) -{ - jsoncons::jsonpath::detail::jsonpath_evaluator> evaluator; - evaluator.evaluate(root, path); - evaluator.replace(std::forward(new_value)); -} - -namespace detail { - -template -bool try_string_to_index(const CharT *s, size_t length, size_t* value, bool* positive) -{ - static const size_t max_value = (std::numeric_limits::max)(); - static const size_t max_value_div_10 = max_value / 10; - - size_t start = 0; - size_t n = 0; - if (length > 0) - { - if (s[start] == '-') - { - *positive = false; - ++start; - } - else - { - *positive = true; - } - } - if (length > start) - { - for (size_t i = start; i < length; ++i) - { - CharT c = s[i]; - switch (c) - { - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8':case '9': - { - size_t x = c - '0'; - if (n > max_value_div_10) - { - return false; - } - n = n * 10; - if (n > max_value - x) - { - return false; - } - - n += x; - break; - } - default: - return false; - break; - } - } - *value = n; - return true; - } - else - { - return false; - } -} - -enum class path_state -{ - start, - dot_or_left_bracket, - name_or_left_bracket, - name, - unquoted_name, - unquoted_name2, - single_quoted_name, - double_quoted_name, - bracketed_unquoted_name, - bracketed_single_quoted_name, - bracketed_double_quoted_name, - bracketed_name_or_path, - bracketed_wildcard_or_path, - wildcard_or_rpath_or_slice_or_filter, - slice_end_or_end_step, - slice_end, - slice_step, - slice_step2, - comma_or_right_bracket, - path_or_function_name, - function, - arg_or_right_paren, - path_argument, - unquoted_arg, - single_quoted_arg, - double_quoted_arg, - more_args_or_right_paren, - dot, - path, - path2, - path_single_quoted, - path_double_quoted -}; - -struct state_item -{ - path_state state; - bool is_recursive_descent; - bool is_union; - - state_item() - : state(path_state::start), is_recursive_descent(false), is_union(false) - { - } - - explicit state_item(path_state state) - : state(state), is_recursive_descent(false), is_union(false) - { - } - - state_item(path_state state, const state_item& parent) - : state(state), - is_recursive_descent(parent.is_recursive_descent), - is_union(parent.is_union) - { - } - - state_item(const state_item&) = default; - state_item& operator=(const state_item&) = default; -}; - -JSONCONS_STRING_LITERAL(length, 'l', 'e', 'n', 'g', 't', 'h') - -template -class jsonpath_evaluator : public ser_context -{ - typedef typename Json::char_type char_type; - typedef typename Json::char_traits_type char_traits_type; - typedef std::basic_string string_type; - typedef typename Json::string_view_type string_view_type; - typedef JsonReference reference; - using pointer = typename std::conditional::type>::value,typename Json::const_pointer,typename Json::pointer>::type; - typedef typename Json::const_pointer const_pointer; - - struct node_type - { - string_type path; - pointer val_ptr; - - node_type() = default; - node_type(const string_type& p, const pointer& valp) - : path(p),val_ptr(valp) - { - } - node_type(string_type&& p, pointer&& valp) - : path(std::move(p)),val_ptr(valp) - { - } - node_type(const node_type&) = default; - node_type(node_type&& other) - : path(std::move(other.path)), val_ptr(other.val_ptr) - { - - } - node_type& operator=(const node_type&) = default; - node_type& operator=(node_type&& other) - { - path.swap(other.path); - val_ptr = other.val_ptr; - } - - }; - typedef std::vector node_set; - - struct node_less - { - bool operator()(const node_type& a, const node_type& b) const - { - return *(a.val_ptr) < *(b.val_ptr); - } - }; - - class selector_base - { - public: - virtual ~selector_base() - { - } - virtual void select(jsonpath_evaluator& evaluator, - const string_type& path, reference val, node_set& nodes) = 0; - - virtual bool is_filter() const - { - return false; - } - }; - - class path_selector final : public selector_base - { - private: - std::basic_string path_; - public: - path_selector(const std::basic_string& path) - : path_(path) - { - } - - void select(jsonpath_evaluator&, - const string_type& path, reference val, - node_set& nodes) override - { - std::error_code ec; - jsonpath_evaluator e; - e.evaluate(val, path_, ec); - if (!ec) - { - for (auto ptr : e.get_pointers()) - { - nodes.emplace_back(PathCons()(path,path_),ptr); - } - } - } - }; - - class expr_selector final : public selector_base - { - private: - jsonpath_filter_expr result_; - public: - expr_selector(const jsonpath_filter_expr& result) - : result_(result) - { - } - - void select(jsonpath_evaluator& evaluator, - const string_type& path, reference val, - node_set& nodes) override - { - auto index = result_.eval(val); - if (index.template is()) - { - size_t start = index.template as(); - if (val.is_array() && start < val.size()) - { - nodes.emplace_back(PathCons()(path,start),std::addressof(val[start])); - } - } - else if (index.is_string()) - { - name_selector selector(index.as_string_view()); - selector.select(evaluator, path, val, nodes); - } - } - }; - - class filter_selector final : public selector_base - { - private: - jsonpath_filter_expr result_; - public: - filter_selector(const jsonpath_filter_expr& result) - : result_(result) - { - } - - bool is_filter() const override - { - return true; - } - - void select(jsonpath_evaluator&, - const string_type& path, reference val, - node_set& nodes) override - { - //std::cout << "filter_selector select "; - if (val.is_array()) - { - //std::cout << "from array \n"; - for (size_t i = 0; i < val.size(); ++i) - { - if (result_.exists(val[i])) - { - nodes.emplace_back(PathCons()(path,i),std::addressof(val[i])); - } - } - } - else if (val.is_object()) - { - //std::cout << "from object \n"; - if (result_.exists(val)) - { - nodes.emplace_back(path, std::addressof(val)); - } - } - } - }; - - class name_selector final : public selector_base - { - private: - string_type name_; - public: - name_selector(const string_view_type& name) - : name_(name) - { - } - - void select(jsonpath_evaluator& evaluator, - const string_type& path, reference val, - node_set& nodes) override - { - bool is_start_positive = true; - - if (val.is_object() && val.contains(name_)) - { - nodes.emplace_back(PathCons()(path,name_),std::addressof(val.at(name_))); - } - else if (val.is_array()) - { - size_t pos = 0; - if (try_string_to_index(name_.data(), name_.size(), &pos, &is_start_positive)) - { - size_t index = is_start_positive ? pos : val.size() - pos; - if (index < val.size()) - { - nodes.emplace_back(PathCons()(path,index),std::addressof(val[index])); - } - } - else if (name_ == length_literal() && val.size() > 0) - { - pointer ptr = evaluator.create_temp(val.size()); - nodes.emplace_back(PathCons()(path, name_), ptr); - } - } - else if (val.is_string()) - { - size_t pos = 0; - string_view_type sv = val.as_string_view(); - if (try_string_to_index(name_.data(), name_.size(), &pos, &is_start_positive)) - { - size_t index = is_start_positive ? pos : sv.size() - pos; - auto sequence = unicons::sequence_at(sv.data(), sv.data() + sv.size(), index); - if (sequence.length() > 0) - { - pointer ptr = evaluator.create_temp(sequence.begin(),sequence.length()); - nodes.emplace_back(PathCons()(path, index), ptr); - } - } - else if (name_ == length_literal() && sv.size() > 0) - { - size_t count = unicons::u32_length(sv.begin(),sv.end()); - pointer ptr = evaluator.create_temp(count); - nodes.emplace_back(PathCons()(path, name_), ptr); - } - } - } - }; - - class array_slice_selector final : public selector_base - { - private: - array_slice slice_; - public: - array_slice_selector(const array_slice& slice) - : slice_(slice) - { - } - - void select(jsonpath_evaluator&, - const string_type& path, reference val, - node_set& nodes) override - { - if (slice_.is_step_positive) - { - end_array_slice1(path, val, nodes); - } - else - { - end_array_slice2(path, val, nodes); - } - } - - void end_array_slice1(const string_type& path, reference val, node_set& nodes) - { - if (val.is_array()) - { - size_t start = slice_.get_start(val.size()); - size_t end = slice_.get_end(val.size()); - for (size_t j = start; j < end; j += slice_.step()) - { - if (j < val.size()) - { - nodes.emplace_back(PathCons()(path,j),std::addressof(val[j])); - } - } - } - } - - void end_array_slice2(const string_type& path, reference val, node_set& nodes) - { - if (val.is_array()) - { - size_t start = slice_.get_start(val.size()); - size_t end = slice_.get_end(val.size()); - - size_t j = end + slice_.step() - 1; - while (j > (start+slice_.step()-1)) - { - j -= slice_.step(); - if (j < val.size()) - { - nodes.emplace_back(PathCons()(path,j),std::addressof(val[j])); - } - } - } - } - }; - - function_table functions_; - - default_parse_error_handler default_err_handler_; - node_set nodes_; - std::vector stack_; - size_t line_; - size_t column_; - const char_type* begin_input_; - const char_type* end_input_; - const char_type* p_; - std::vector> selectors_; - std::vector> temp_json_values_; - - typedef std::vector argument_type; - std::vector function_stack_; - std::vector state_stack_; - -public: - jsonpath_evaluator() - : line_(1), column_(1), - begin_input_(nullptr), end_input_(nullptr), - p_(nullptr) - { - } - - jsonpath_evaluator(size_t line, size_t column) - : line_(line), column_(column), - begin_input_(nullptr), end_input_(nullptr), - p_(nullptr) - { - } - - size_t line() const - { - return line_; - } - - size_t column() const - { - return column_; - } - - Json get_values() const - { - Json result = typename Json::array(); - - if (stack_.size() > 0) - { - result.reserve(stack_.back().size()); - for (const auto& p : stack_.back()) - { - result.push_back(*(p.val_ptr)); - } - } - return result; - } - - std::vector get_pointers() const - { - std::vector result; - - if (stack_.size() > 0) - { - result.reserve(stack_.back().size()); - for (const auto& p : stack_.back()) - { - result.push_back(p.val_ptr); - } - } - return result; - } - - void call_function(const string_type& function_name, std::error_code& ec) - { - auto f = functions_.get(function_name, ec); - if (ec) - { - return; - } - auto result = f(function_stack_, ec); - if (ec) - { - return; - } - - string_type s = {'$'}; - node_set v; - pointer ptr = create_temp(std::move(result)); - v.emplace_back(s,ptr); - stack_.push_back(v); - } - - template - pointer create_temp(Args&& ... args) - { - auto temp = make_unique_ptr(std::forward(args)...); - pointer ptr = temp.get(); - temp_json_values_.emplace_back(std::move(temp)); - return ptr; - } - - Json get_normalized_paths() const - { - Json result = typename Json::array(); - if (stack_.size() > 0) - { - result.reserve(stack_.back().size()); - for (const auto& p : stack_.back()) - { - result.push_back(p.path); - } - } - return result; - } - - template - void replace(T&& new_value) - { - if (stack_.size() > 0) - { - for (size_t i = 0; i < stack_.back().size(); ++i) - { - *(stack_.back()[i].val_ptr) = new_value; - } - } - } - - void evaluate(reference root, const string_view_type& path) - { - std::error_code ec; - evaluate(root, path.data(), path.length(), ec); - if (ec) - { - throw jsonpath_error(ec, line_, column_); - } - } - - void evaluate(reference root, const string_view_type& path, std::error_code& ec) - { - try - { - evaluate(root, path.data(), path.length(), ec); - } - catch (...) - { - ec = jsonpath_errc::unidentified_error; - } - } - - void evaluate(reference root, - const char_type* path, - size_t length, - std::error_code& ec) - { - state_stack_.emplace_back(path_state::start); - - string_type function_name; - string_type buffer; - - begin_input_ = path; - end_input_ = path + length; - p_ = begin_input_; - - string_type s = {'$'}; - node_set v; - v.emplace_back(std::move(s),std::addressof(root)); - stack_.push_back(v); - - array_slice slice; - - while (p_ < end_input_) - { - switch (state_stack_.back().state) - { - case path_state::start: - { - switch (*p_) - { - case ' ':case '\t':case '\r':case '\n': - advance_past_space_character(); - break; - case '$': - { - state_stack_.emplace_back(path_state::dot_or_left_bracket, state_stack_.back()); - ++p_; - ++column_; - break; - } - default: - { - switch (*p_) - { - case '.': - case '[': - ec = jsonpath_errc::expected_root; - return; - default: // might be function, validate name later - state_stack_.emplace_back(path_state::dot_or_left_bracket, state_stack_.back()); - state_stack_.emplace_back(path_state::path_or_function_name, state_stack_.back()); - break; - } - break; - } - - return; - }; - break; - } - case path_state::path_or_function_name: - switch (*p_) - { - case ' ':case '\t':case '\r':case '\n': - { - selectors_.push_back(make_unique_ptr(buffer)); - apply_selectors(); - buffer.clear(); - state_stack_.pop_back(); - advance_past_space_character(); - break; - } - case '(': - state_stack_.back().state = path_state::arg_or_right_paren; - function_name = std::move(buffer); - buffer.clear(); - ++p_; - ++column_; - break; - case '[': - { - if (buffer.size() > 0) - { - selectors_.push_back(make_unique_ptr(buffer)); - apply_selectors(); - buffer.clear(); - } - slice.start_ = 0; - - state_stack_.back().state = path_state::wildcard_or_rpath_or_slice_or_filter; - ++p_; - ++column_; - break; - } - case '.': - { - if (buffer.size() > 0) - { - selectors_.push_back(make_unique_ptr(buffer)); - apply_selectors(); - buffer.clear(); - } - state_stack_.back().state = path_state::dot; - ++p_; - ++column_; - break; - } - case '*': - { - end_all(); - transfer_nodes(); - state_stack_.back().state = path_state::dot; - ++p_; - ++column_; - break; - } - case '\'': - { - buffer.clear(); - state_stack_.back().state = path_state::single_quoted_name; - ++p_; - ++column_; - break; - } - case '\"': - { - buffer.clear(); - state_stack_.back().state = path_state::double_quoted_name; - ++p_; - ++column_; - break; - } - default: - buffer.push_back(*p_); - ++p_; - ++column_; - break; - } - break; - case path_state::arg_or_right_paren: - switch (*p_) - { - case ' ':case '\t':case '\r':case '\n': - advance_past_space_character(); - break; - case '$': - buffer.clear(); - buffer.push_back(*p_); - state_stack_.emplace_back(path_state::path_argument, state_stack_.back()); - ++p_; - ++column_; - break; - case '\'': - buffer.clear(); - buffer.push_back('\"'); - state_stack_.back().state = path_state::more_args_or_right_paren; - state_stack_.emplace_back(path_state::single_quoted_arg, state_stack_.back()); - ++p_; - ++column_; - break; - case '\"': - buffer.clear(); - buffer.push_back('\"'); - state_stack_.back().state = path_state::more_args_or_right_paren; - state_stack_.emplace_back(path_state::double_quoted_arg, state_stack_.back()); - ++p_; - ++column_; - break; - case ')': - { - jsonpath_evaluator evaluator; - evaluator.evaluate(root, buffer, ec); - if (ec) - { - return; - } - function_stack_.push_back(evaluator.get_pointers()); - - call_function(function_name, ec); - if (ec) - { - return; - } - state_stack_.pop_back(); - ++p_; - ++column_; - break; - } - default: - buffer.clear(); - state_stack_.back().state = path_state::more_args_or_right_paren; - state_stack_.emplace_back(path_state::unquoted_arg, state_stack_.back()); - ++p_; - ++column_; - break; - } - break; - case path_state::path_argument: - switch (*p_) - { - case ',': - { - jsonpath_evaluator evaluator; - evaluator.evaluate(root, buffer, ec); - if (ec) - { - return; - } - function_stack_.push_back(evaluator.get_pointers()); - state_stack_.pop_back(); - ++p_; - ++column_; - break; - } - case ')': - { - state_stack_.pop_back(); - break; - } - default: - buffer.push_back(*p_); // path_argument - ++p_; - ++column_; - break; - } - break; - case path_state::unquoted_arg: - switch (*p_) - { - case ',': - try - { - auto val = Json::parse(buffer); - auto temp = create_temp(val); - function_stack_.push_back(std::vector{temp}); - } - catch (const ser_error&) - { - ec = jsonpath_errc::argument_parse_error; - return; - } - buffer.clear(); - //state_ = path_state::arg_or_right_paren; - state_stack_.pop_back(); - break; - case ')': - { - try - { - auto val = Json::parse(buffer); - auto temp = create_temp(val); - function_stack_.push_back(std::vector{temp}); - } - catch (const ser_error&) - { - ec = jsonpath_errc::argument_parse_error; - return; - } - call_function(function_name, ec); - if (ec) - { - return; - } - state_stack_.pop_back(); - break; - } - default: - buffer.push_back(*p_); - break; - } - ++p_; - ++column_; - break; - case path_state::single_quoted_arg: - switch (*p_) - { - case '\'': - buffer.push_back('\"'); - state_stack_.pop_back(); - break; - case '\"': - buffer.push_back('\\'); - buffer.push_back('\"'); - state_stack_.pop_back(); - break; - default: - buffer.push_back(*p_); - break; - } - ++p_; - ++column_; - break; - case path_state::double_quoted_arg: - switch (*p_) - { - case '\"': - buffer.push_back('\"'); - state_stack_.pop_back(); - break; - default: - buffer.push_back(*p_); - break; - } - ++p_; - ++column_; - break; - case path_state::more_args_or_right_paren: - switch (*p_) - { - case ' ':case '\t':case '\r':case '\n': - advance_past_space_character(); - break; - case ',': - try - { - auto val = Json::parse(buffer); - auto temp = create_temp(val); - function_stack_.push_back(std::vector{temp}); - } - catch (const ser_error&) - { - ec = jsonpath_errc::argument_parse_error; - return; - } - buffer.clear(); - //state_ = path_state::arg_or_right_paren; - state_stack_.pop_back(); - ++p_; - ++column_; - break; - case ')': - { - try - { - auto val = Json::parse(buffer); - auto temp = create_temp(val); - function_stack_.push_back(std::vector{temp}); - } - catch (const ser_error&) - { - ec = jsonpath_errc::argument_parse_error; - return; - } - call_function(function_name, ec); - if (ec) - { - return; - } - state_stack_.pop_back(); - ++p_; - ++column_; - break; - } - default: - ec = jsonpath_errc::invalid_filter_unsupported_operator; - return; - } - break; - case path_state::dot: - switch (*p_) - { - case '.': - state_stack_.back().is_recursive_descent = true; - ++p_; - ++column_; - state_stack_.back().state = path_state::name_or_left_bracket; - break; - default: - state_stack_.back().state = path_state::name; - break; - } - break; - case path_state::name_or_left_bracket: - switch (*p_) - { - case ' ':case '\t':case '\r':case '\n': - advance_past_space_character(); - break; - case '[': // [ can follow .. - state_stack_.back().state = path_state::wildcard_or_rpath_or_slice_or_filter; - ++p_; - ++column_; - break; - default: - buffer.clear(); - state_stack_.back().state = path_state::name; - break; - } - break; - case path_state::name: - switch (*p_) - { - case ' ':case '\t':case '\r':case '\n': - advance_past_space_character(); - break; - case '*': - end_all(); - transfer_nodes(); - state_stack_.pop_back(); - ++p_; - ++column_; - break; - case '\'': - state_stack_.back().state = path_state::single_quoted_name; - ++p_; - ++column_; - break; - case '\"': - state_stack_.back().state = path_state::double_quoted_name; - ++p_; - ++column_; - break; - case '[': - case '.': - ec = jsonpath_errc::expected_name; - return; - default: - buffer.clear(); - state_stack_.back().state = path_state::unquoted_name; - break; - } - break; - case path_state::dot_or_left_bracket: - switch (*p_) - { - case ' ':case '\t':case '\r':case '\n': - advance_past_space_character(); - break; - case '.': - state_stack_.emplace_back(path_state::dot, state_stack_.back()); - ++p_; - ++column_; - break; - case '[': - state_stack_.emplace_back(path_state::wildcard_or_rpath_or_slice_or_filter, state_stack_.back()); - ++p_; - ++column_; - break; - default: - ec = jsonpath_errc::expected_separator; - return; - }; - break; - case path_state::unquoted_name: - switch (*p_) - { - case '[': - case '.': - case ' ':case '\t': - case '\r': - case '\n': - state_stack_.back().state = path_state::unquoted_name2; - break; - default: - buffer.push_back(*p_); - ++p_; - ++column_; - break; - }; - break; - case path_state::unquoted_name2: - switch (*p_) - { - case ' ':case '\t':case '\r':case '\n': - advance_past_space_character(); - break; - case '[': - selectors_.push_back(make_unique_ptr(buffer)); - apply_selectors(); - buffer.clear(); - slice.start_ = 0; - state_stack_.pop_back(); - break; - case '.': - selectors_.push_back(make_unique_ptr(buffer)); - apply_selectors(); - buffer.clear(); - state_stack_.pop_back(); - break; - default: - ec = jsonpath_errc::expected_name; - return; - }; - break; - case path_state::single_quoted_name: - switch (*p_) - { - case '\'': - selectors_.push_back(make_unique_ptr(buffer)); - apply_selectors(); - buffer.clear(); - state_stack_.pop_back(); - break; - case '\\': - if (p_+1 < end_input_) - { - ++p_; - ++column_; - buffer.push_back(*p_); - } - else - { - ec = jsonpath_errc::unexpected_end_of_input; - return; - } - break; - default: - buffer.push_back(*p_); - break; - }; - ++p_; - ++column_; - break; - case path_state::double_quoted_name: - switch (*p_) - { - case '\"': - selectors_.push_back(make_unique_ptr(buffer)); - apply_selectors(); - buffer.clear(); - state_stack_.pop_back(); - break; - case '\\': - if (p_+1 < end_input_) - { - ++p_; - ++column_; - buffer.push_back(*p_); - } - else - { - ec = jsonpath_errc::unexpected_end_of_input; - return; - } - break; - default: - buffer.push_back(*p_); - break; - }; - ++p_; - ++column_; - break; - case path_state::comma_or_right_bracket: - switch (*p_) - { - case ' ':case '\t':case '\r':case '\n': - advance_past_space_character(); - break; - case ',': - state_stack_.back().is_union = true; - state_stack_.back().state = path_state::wildcard_or_rpath_or_slice_or_filter; - ++p_; - ++column_; - break; - case ']': - apply_selectors(); - state_stack_.pop_back(); - ++p_; - ++column_; - break; - default: - ec = jsonpath_errc::expected_right_bracket; - return; - } - break; - case path_state::wildcard_or_rpath_or_slice_or_filter: - switch (*p_) - { - case ' ':case '\t':case '\r':case '\n': - advance_past_space_character(); - break; - case '(': - { - jsonpath_filter_parser parser(line_,column_); - auto result = parser.parse(root, p_,end_input_,&p_); - line_ = parser.line(); - column_ = parser.column(); - selectors_.push_back(make_unique_ptr(result)); - state_stack_.back().state = path_state::comma_or_right_bracket; - break; - } - case '?': - { - jsonpath_filter_parser parser(line_,column_); - auto result = parser.parse(root,p_,end_input_,&p_); - line_ = parser.line(); - column_ = parser.column(); - selectors_.push_back(make_unique_ptr(result)); - state_stack_.back().state = path_state::comma_or_right_bracket; - break; - } - case ':': - slice = array_slice(); - buffer.clear(); - state_stack_.back().state = path_state::comma_or_right_bracket; - state_stack_.emplace_back(path_state::slice_end_or_end_step, state_stack_.back()); - ++p_; - ++column_; - break; - case '*': - state_stack_.back().state = path_state::comma_or_right_bracket; - state_stack_.emplace_back(path_state::bracketed_wildcard_or_path, state_stack_.back()); - ++p_; - ++column_; - break; - case '\'': - state_stack_.back().state = path_state::comma_or_right_bracket; - state_stack_.emplace_back(path_state::bracketed_single_quoted_name, state_stack_.back()); - ++p_; - ++column_; - break; - case '\"': - state_stack_.back().state = path_state::comma_or_right_bracket; - state_stack_.emplace_back(path_state::bracketed_double_quoted_name, state_stack_.back()); - ++p_; - ++column_; - break; - default: - slice = array_slice(); - buffer.clear(); - buffer.push_back(*p_); - state_stack_.back().state = path_state::comma_or_right_bracket; - state_stack_.emplace_back(path_state::bracketed_unquoted_name, state_stack_.back()); - ++p_; - ++column_; - break; - } - break; - case path_state::bracketed_unquoted_name: - switch (*p_) - { - case ':': - if (!try_string_to_index(buffer.data(), buffer.size(), &slice.start_, &slice.is_start_positive)) - { - ec = jsonpath_errc::expected_slice_start; - return; - } - state_stack_.back().state = path_state::slice_end_or_end_step; - ++p_; - ++column_; - break; - case '.': - case '[': - case ',': - case ']': - state_stack_.back().state = path_state::bracketed_name_or_path; - break; - default: - buffer.push_back(*p_); - ++p_; - ++column_; - break; - } - break; - case path_state::bracketed_name_or_path: - switch (*p_) - { - case ' ':case '\t':case '\r':case '\n': - advance_past_space_character(); - break; - case '.': - buffer.push_back(*p_); - state_stack_.back().state = path_state::path; - ++p_; - ++column_; - break; - case '[': - buffer.push_back(*p_); - state_stack_.back().state = path_state::path2; - ++p_; - ++column_; - break; - case ',': - case ']': - if (!buffer.empty()) - { - selectors_.push_back(make_unique_ptr(buffer)); - buffer.clear(); - } - state_stack_.pop_back(); - break; - default: - ec = jsonpath_errc::expected_right_bracket; - return; - } - break; - case path_state::bracketed_wildcard_or_path: - switch (*p_) - { - case ' ':case '\t':case '\r':case '\n': - advance_past_space_character(); - break; - case '.': - buffer.push_back('*'); - buffer.push_back(*p_); - state_stack_.back().state = path_state::path; - ++p_; - ++column_; - break; - case '[': - buffer.push_back('*'); - buffer.push_back(*p_); - state_stack_.back().state = path_state::path2; - ++p_; - ++column_; - break; - case ',': - case ']': - end_all(); - state_stack_.pop_back(); - break; - default: - ec = jsonpath_errc::expected_right_bracket; - return; - } - break; - case path_state::path: - switch (*p_) - { - case '\'': - buffer.push_back(*p_); - state_stack_.emplace_back(path_state::path_single_quoted, state_stack_.back()); - ++p_; - ++column_; - break; - case '\"': - buffer.push_back(*p_); - state_stack_.emplace_back(path_state::path_double_quoted, state_stack_.back()); - ++p_; - ++column_; - break; - case ',': - case ']': - if (!buffer.empty()) - { - selectors_.push_back(make_unique_ptr(buffer)); - buffer.clear(); - } - state_stack_.pop_back(); - break; - default: - buffer.push_back(*p_); - ++p_; - ++column_; - break; - } - break; - case path_state::path_double_quoted: - switch (*p_) - { - case '\"': - buffer.push_back(*p_); - state_stack_.pop_back(); - break; - case '\\': - if (p_+1 < end_input_) - { - ++p_; - ++column_; - buffer.push_back(*p_); - } - else - { - ec = jsonpath_errc::unexpected_end_of_input; - return; - } - break; - default: - buffer.push_back(*p_); - break; - } - ++p_; - ++column_; - break; - case path_state::path_single_quoted: - switch (*p_) - { - case '\'': - buffer.push_back(*p_); - state_stack_.pop_back(); - break; - case '\\': - if (p_+1 < end_input_) - { - ++p_; - ++column_; - buffer.push_back(*p_); - } - else - { - ec = jsonpath_errc::unexpected_end_of_input; - return; - } - break; - default: - buffer.push_back(*p_); - break; - } - ++p_; - ++column_; - break; - case path_state::path2: - switch (*p_) - { - case ']': - buffer.push_back(*p_); - state_stack_.back().state = path_state::path; - break; - default: - buffer.push_back(*p_); - break; - } - ++p_; - ++column_; - break; - case path_state::slice_end_or_end_step: - switch (*p_) - { - case '-': - slice.is_end_positive = false; - state_stack_.back().state = path_state::slice_end; - ++p_; - ++column_; - break; - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8':case '9': - slice.is_end_defined = true; - slice.end_ = static_cast(*p_-'0'); - state_stack_.back().state = path_state::slice_end; - ++p_; - ++column_; - break; - case ':': - slice.step_ = 0; - state_stack_.back().state = path_state::slice_step; - ++p_; - ++column_; - break; - case ',': - case ']': - selectors_.push_back(make_unique_ptr(slice)); - state_stack_.pop_back(); - break; - default: - ec = jsonpath_errc::expected_minus_or_digit_or_colon_or_comma_or_right_bracket; - return; - } - break; - case path_state::slice_end: - switch (*p_) - { - case ':': - slice.step_ = 0; - state_stack_.back().state = path_state::slice_step; - ++p_; - ++column_; - break; - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8':case '9': - slice.is_end_defined = true; - slice.end_ = slice.end_*10 + static_cast(*p_-'0'); - ++p_; - ++column_; - break; - case ',': - case ']': - if (!slice.is_end_defined) - { - ec = jsonpath_errc::expected_slice_end; - return; - } - selectors_.push_back(make_unique_ptr(slice)); - state_stack_.pop_back(); - break; - default: - ec = jsonpath_errc::expected_digit_or_colon_or_comma_or_right_bracket; - return; - } - break; - case path_state::slice_step: - switch (*p_) - { - case '-': - slice.is_step_positive = false; - slice.step_ = 0; - state_stack_.back().state = path_state::slice_step2; - ++p_; - ++column_; - break; - default: - slice.step_ = 0; - state_stack_.back().state = path_state::slice_step2; - break; - } - break; - case path_state::slice_step2: - switch (*p_) - { - case '0':case '1':case '2':case '3':case '4':case '5':case '6':case '7':case '8':case '9': - slice.step_ = slice.step_*10 + static_cast(*p_-'0'); - ++p_; - ++column_; - break; - case ',': - case ']': - if (slice.step_ == 0) - { - ec = jsonpath_errc::expected_slice_step; - return; - } - selectors_.push_back(make_unique_ptr(slice)); - state_stack_.pop_back(); - break; - default: - ec = jsonpath_errc::expected_minus_or_digit_or_comma_or_right_bracket; - return; - } - break; - case path_state::bracketed_single_quoted_name: - switch (*p_) - { - case '\'': - state_stack_.back().state = path_state::bracketed_name_or_path; - break; - case '\\': - if (p_+1 < end_input_) - { - ++p_; - ++column_; - buffer.push_back(*p_); - } - else - { - ec = jsonpath_errc::unexpected_end_of_input; - return; - } - break; - default: - buffer.push_back(*p_); - break; - }; - ++p_; - ++column_; - break; - case path_state::bracketed_double_quoted_name: - switch (*p_) - { - case '\"': - state_stack_.back().state = path_state::bracketed_name_or_path; - break; - case '\\': - if (p_+1 < end_input_) - { - ++p_; - ++column_; - buffer.push_back(*p_); - } - else - { - ec = jsonpath_errc::unexpected_end_of_input; - return; - } - break; - default: - buffer.push_back(*p_); - break; - }; - ++p_; - ++column_; - break; - default: - ++p_; - ++column_; - break; - } - } - - switch (state_stack_.back().state) - { - case path_state::unquoted_name: - case path_state::unquoted_name2: - { - selectors_.push_back(make_unique_ptr(buffer)); - apply_selectors(); - buffer.clear(); - state_stack_.pop_back(); // unquoted_name - break; - } - default: - break; - } - - if (state_stack_.size() > 2) - { - ec = jsonpath_errc::unexpected_end_of_input; - return; - } - - JSONCONS_ASSERT(state_stack_.size() == 2); - state_stack_.pop_back(); - - JSONCONS_ASSERT(state_stack_.back().state == path_state::start); - state_stack_.pop_back(); - } - - void end_all() - { - for (const auto& node : stack_.back()) - { - const auto& path = node.path; - pointer p = node.val_ptr; - - if (p->is_array()) - { - for (auto it = p->array_range().begin(); it != p->array_range().end(); ++it) - { - nodes_.emplace_back(PathCons()(path,it - p->array_range().begin()),std::addressof(*it)); - } - } - else if (p->is_object()) - { - for (auto it = p->object_range().begin(); it != p->object_range().end(); ++it) - { - nodes_.emplace_back(PathCons()(path,it->key()),std::addressof(it->value())); - } - } - - } - } - - void apply_selectors() - { - //std::cout << "apply_selectors count: " << selectors_.size() << "\n"; - if (selectors_.size() > 0) - { - for (auto& node : stack_.back()) - { - //std::cout << "apply selector to:\n" << pretty_print(*(node.val_ptr)) << "\n"; - for (auto& selector : selectors_) - { - apply_selector(node.path, *(node.val_ptr), *selector, true); - } - } - selectors_.clear(); - } - transfer_nodes(); - } - - void apply_selector(const string_type& path, reference val, selector_base& selector, bool process) - { - if (process) - { - selector.select(*this, path, val, nodes_); - } - //std::cout << "*it: " << val << "\n"; - //std::cout << "apply_selectors 1 done\n"; - if (state_stack_.back().is_recursive_descent) - { - //std::cout << "is_recursive_descent\n"; - if (val.is_object()) - { - //std::cout << "is_object\n"; - for (auto& nvp : val.object_range()) - { - if (nvp.value().is_array() || nvp.value().is_object()) - { - apply_selector(PathCons()(path,nvp.key()), nvp.value(), selector, true); - } - } - } - else if (val.is_array()) - { - //std::cout << "is_array\n"; - auto first = val.array_range().begin(); - auto last = val.array_range().end(); - for (auto it = first; it != last; ++it) - { - if (it->is_array()) - { - apply_selector(PathCons()(path,it - first), *it,selector, true); - //std::cout << "*it: " << *it << "\n"; - } - else if (it->is_object()) - { - apply_selector(PathCons()(path,it - first), *it, selector, !selector.is_filter()); - } - } - } - } - } - - void transfer_nodes() - { - if (state_stack_.back().is_union) - { - std::set temp(nodes_.begin(), nodes_.end()); - stack_.push_back(std::vector(temp.begin(),temp.end())); - } - else - { - stack_.push_back(std::move(nodes_)); - } - nodes_.clear(); - state_stack_.back().is_recursive_descent = false; - state_stack_.back().is_union = false; - } - - void advance_past_space_character() - { - switch (*p_) - { - case ' ':case '\t': - ++p_; - ++column_; - break; - case '\r': - if (p_+1 < end_input_ && *(p_+1) == '\n') - ++p_; - ++line_; - column_ = 1; - ++p_; - break; - case '\n': - ++line_; - column_ = 1; - ++p_; - break; - default: - break; - } - } -}; - -} - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpath/jsonpath_error.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpath/jsonpath_error.hpp deleted file mode 100644 index 54aa4f6ae8..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpath/jsonpath_error.hpp +++ /dev/null @@ -1,206 +0,0 @@ -/// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSONPATH_JSONPATH_ERROR_HPP -#define JSONCONS_JSONPATH_JSONPATH_ERROR_HPP - -#include -#include - -namespace jsoncons { namespace jsonpath { - -class jsonpath_error : public std::system_error, public virtual json_exception -{ - std::string buffer_; - size_t line_number_; - size_t column_number_; -public: - jsonpath_error(std::error_code ec) - : std::system_error(ec), line_number_(0), column_number_(0) - { - } - jsonpath_error(std::error_code ec, size_t position) - : std::system_error(ec), line_number_(0), column_number_(position) - { - } - jsonpath_error(std::error_code ec, size_t line, size_t column) - : std::system_error(ec), line_number_(line), column_number_(column) - { - } - jsonpath_error(const jsonpath_error& other) = default; - - jsonpath_error(jsonpath_error&& other) = default; - - const char* what() const noexcept override - { - try - { - std::ostringstream os; - os << std::system_error::what(); - if (line_number_ != 0 && column_number_ != 0) - { - os << " at line " << line_number_ << " and column " << column_number_; - } - else if (column_number_ != 0) - { - os << " at position " << column_number_; - } - const_cast(buffer_) = os.str(); - return buffer_.c_str(); - } - catch (...) - { - return std::system_error::what(); - } - } - - size_t line() const noexcept - { - return line_number_; - } - - size_t column() const noexcept - { - return column_number_; - } -#if !defined(JSONCONS_NO_DEPRECATED) - size_t line_number() const noexcept - { - return line(); - } - - size_t column_number() const noexcept - { - return column(); - } -#endif -}; - -enum class jsonpath_errc -{ - ok = 0, - expected_root, - expected_current_node, - expected_right_bracket, - expected_name, - expected_separator, - invalid_filter, - invalid_filter_expected_slash, - invalid_filter_unbalanced_paren, - invalid_filter_unsupported_operator, - invalid_filter_expected_right_brace, - invalid_filter_expected_primary, - expected_slice_start, - expected_slice_end, - expected_slice_step, - expected_left_bracket_token, - expected_minus_or_digit_or_colon_or_comma_or_right_bracket, - expected_digit_or_colon_or_comma_or_right_bracket, - expected_minus_or_digit_or_comma_or_right_bracket, - expected_digit_or_comma_or_right_bracket, - unexpected_operator, - invalid_function_name, - invalid_argument, - function_name_not_found, - parse_error_in_filter, - argument_parse_error, - unidentified_error, - unexpected_end_of_input -}; - -class jsonpath_error_category_impl - : public std::error_category -{ -public: - const char* name() const noexcept override - { - return "jsoncons/jsonpath"; - } - std::string message(int ev) const override - { - switch (static_cast(ev)) - { - case jsonpath_errc::expected_root: - return "Expected $"; - case jsonpath_errc::expected_current_node: - return "Expected @"; - case jsonpath_errc::expected_right_bracket: - return "Expected ]"; - case jsonpath_errc::expected_name: - return "Expected a name following a dot"; - case jsonpath_errc::expected_slice_start: - return "Expected slice start"; - case jsonpath_errc::expected_slice_end: - return "Expected slice end"; - case jsonpath_errc::expected_slice_step: - return "Expected slice step"; - case jsonpath_errc::expected_separator: - return "Expected dot or left bracket separator"; - case jsonpath_errc::invalid_filter: - return "Invalid path filter"; - case jsonpath_errc::invalid_filter_expected_slash: - return "Invalid path filter, expected '/'"; - case jsonpath_errc::invalid_filter_unbalanced_paren: - return "Invalid path filter, unbalanced parenthesis"; - case jsonpath_errc::invalid_filter_unsupported_operator: - return "Unsupported operator"; - case jsonpath_errc::invalid_filter_expected_right_brace: - return "Invalid path filter, expected right brace }"; - case jsonpath_errc::invalid_filter_expected_primary: - return "Invalid path filter, expected primary expression."; - case jsonpath_errc::expected_left_bracket_token: - return "Expected ?,',\",0-9,*"; - case jsonpath_errc::expected_minus_or_digit_or_colon_or_comma_or_right_bracket: - return "Expected - or 0-9 or : or , or ]"; - case jsonpath_errc::expected_minus_or_digit_or_comma_or_right_bracket: - return "Expected - or 0-9 or , or ]"; - case jsonpath_errc::expected_digit_or_comma_or_right_bracket: - return "Expected - or 0-9 or , or ]"; - case jsonpath_errc::expected_digit_or_colon_or_comma_or_right_bracket: - return "Expected 0-9 or : or , or ]"; - case jsonpath_errc::invalid_function_name: - return "Invalid function name"; - case jsonpath_errc::invalid_argument: - return "Invalid argument type"; - case jsonpath_errc::function_name_not_found: - return "Function name not found"; - case jsonpath_errc::parse_error_in_filter: - return "Could not parse JSON expression in a JSONPath filter"; - case jsonpath_errc::argument_parse_error: - return "Could not parse JSON expression passed to JSONPath function"; - case jsonpath_errc::unidentified_error: - return "Unidentified error"; - case jsonpath_errc::unexpected_end_of_input: - return "Unexpected end of jsonpath input"; - default: - return "Unknown jsonpath parser error"; - } - } -}; - -inline -const std::error_category& jsonpath_error_category() -{ - static jsonpath_error_category_impl instance; - return instance; -} - -inline -std::error_code make_error_code(jsonpath_errc result) -{ - return std::error_code(static_cast(result),jsonpath_error_category()); -} - -}} - -namespace std { - template<> - struct is_error_code_enum : public true_type - { - }; -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpath/jsonpath_filter.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpath/jsonpath_filter.hpp deleted file mode 100644 index 444ac17f77..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpath/jsonpath_filter.hpp +++ /dev/null @@ -1,1953 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSONPATH_FILTER_HPP -#define JSONCONS_JSONPATH_FILTER_HPP - -#include -#include // std::map -#include -#include -#include -#include -#include -#include // std::numeric_limits -#include -#include - -namespace jsoncons { namespace jsonpath { namespace detail { - -JSONCONS_STRING_LITERAL(eqtilde,'=','~') -JSONCONS_STRING_LITERAL(star,'*') -JSONCONS_STRING_LITERAL(forwardslash,'/') -JSONCONS_STRING_LITERAL(plus,'+') -JSONCONS_STRING_LITERAL(minus,'-') -JSONCONS_STRING_LITERAL(lt,'<') -JSONCONS_STRING_LITERAL(lte,'<','=') -JSONCONS_STRING_LITERAL(gt,'>') -JSONCONS_STRING_LITERAL(gte,'>','=') -JSONCONS_STRING_LITERAL(eq,'=','=') -JSONCONS_STRING_LITERAL(ne,'!', '=') -JSONCONS_STRING_LITERAL(ampamp,'&','&') -JSONCONS_STRING_LITERAL(pipepipe,'|','|') - -template -struct PathConstructor -{ - typedef typename Json::char_type char_type; - typedef typename Json::string_view_type string_view_type; - typedef typename Json::string_type string_type; - - string_type operator()(const string_type& path, size_t index) const - { - char_type buf[255]; - char_type* p = buf; - do - { - *p++ = static_cast(48 + index % 10); - } while (index /= 10); - - string_type s; - s.append(path); - s.push_back('['); - while (--p >= buf) - { - s.push_back(*p); - } - s.push_back(']'); - return s; - } - - string_type operator()(const string_type& path, const string_view_type& sv) const - { - string_type s; - s.append(path); - s.push_back('['); - s.push_back('\''); - s.append(sv.data(),sv.length()); - s.push_back('\''); - s.push_back(']'); - return s; - } -}; - -template -struct VoidPathConstructor -{ - typedef typename Json::char_type char_type; - typedef typename Json::string_view_type string_view_type; - typedef typename Json::string_type string_type; - - string_type operator()(const string_type&, size_t) const - { - return string_type{}; - } - - string_type operator()(const string_type&, string_view_type) const - { - return string_type{}; - } -}; - -template -class jsonpath_evaluator; - -enum class filter_path_mode -{ - path, - root_path, - current_path -}; - -enum class filter_state -{ - start, - expect_right_round_bracket, - expect_oper_or_right_round_bracket, - expect_path_or_value_or_unary_op, - expect_regex, - regex, - single_quoted_text, - double_quoted_text, - unquoted_text, - path, - value, - oper, - expect_arg, - path_argument, - unquoted_argument, - single_quoted_argument, - double_quoted_argument, - expect_more_args_or_right_round_bracket, - done -}; - -enum class token_type -{ - operand, - unary_operator, - binary_operator, - lparen, - rparen -}; - -template -class term -{ -public: - typedef typename Json::string_type string_type; - typedef typename Json::char_type char_type; - - virtual ~term() {} - - virtual void initialize(const Json&) = 0; - - virtual bool accept_single_node() const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual Json get_single_node() const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual bool exclaim() const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual bool eq_term(const term&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual bool eq(const Json&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual bool ne_term(const term&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual bool ne(const Json&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual bool regex_term(const term&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual bool regex2(const string_type&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual bool ampamp_term(const term&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual bool ampamp(const Json&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual bool pipepipe_term(const term&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual bool pipepipe(const Json&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual bool lt_term(const term&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual bool lt(const Json&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual bool gt_term(const term&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual bool gt(const Json&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - - virtual Json minus_term(const term&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual Json minus(const Json&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - - virtual Json left_minus(const Json&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - - virtual Json unary_minus() const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual Json plus_term(const term&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual Json plus(const Json&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual Json mult_term(const term&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual Json mult(const Json&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - - virtual Json div_term(const term&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - virtual Json div(const Json&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } - - virtual Json left_div(const Json&) const - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator); - } -}; - -template -struct operator_properties -{ - typedef std::function&, const term&)> operator_type; - - size_t precedence_level; - bool is_right_associative; - operator_type op; -}; - -template -class token -{ -public: - typedef std::function&)> unary_operator_type; - typedef std::function&, const term&)> operator_type; -private: - token_type type_; - size_t precedence_level_; - bool is_right_associative_; - std::shared_ptr> operand_ptr_; - unary_operator_type unary_operator_; - operator_type operator_; -public: - - token(token_type type) - : type_(type),precedence_level_(0),is_right_associative_(false) - { - } - token(token_type type, std::shared_ptr> term_ptr) - : type_(type),precedence_level_(0),is_right_associative_(false),operand_ptr_(term_ptr) - { - } - token(size_t precedence_level, - bool is_right_associative, - unary_operator_type unary_operator) - : type_(token_type::unary_operator), - precedence_level_(precedence_level), - is_right_associative_(is_right_associative), - unary_operator_(unary_operator) - { - } - token(const operator_properties& properties) - : type_(token_type::binary_operator), - precedence_level_(properties.precedence_level), - is_right_associative_(properties.is_right_associative), - operator_(properties.op) - { - } - - token(const token& t) = default; - - token_type type() const - { - return type_; - } - - Json operator()(const term& a) - { - return unary_operator_(a); - } - - Json operator()(const term& a, const term& b) - { - return operator_(a,b); - } - - token& operator=(const token& val) = default; - - bool is_operator() const - { - return is_unary_operator() || is_binary_operator(); - } - - bool is_unary_operator() const - { - return type_ == token_type::unary_operator; - } - - bool is_binary_operator() const - { - return type_ == token_type::binary_operator; - } - - bool is_operand() const - { - return type_ == token_type::operand; - } - - bool is_lparen() const - { - return type_ == token_type::lparen; - } - - bool is_rparen() const - { - return type_ == token_type::rparen; - } - - size_t precedence_level() const - { - return precedence_level_; - } - - bool is_right_associative() const - { - return is_right_associative_; - } - - const term& operand() - { - JSONCONS_ASSERT(type_ == token_type::operand && operand_ptr_ != nullptr); - return *operand_ptr_; - } - - void initialize(const Json& current_node) - { - if (operand_ptr_.get() != nullptr) - { - operand_ptr_->initialize(current_node); - } - } -}; - -template -bool ampamp(const Json& lhs, const Json& rhs) -{ - return lhs.as_bool() && rhs.as_bool(); -} - -template -bool pipepipe(const Json& lhs, const Json& rhs) -{ - return lhs.as_bool() || rhs.as_bool(); -} - -template -bool lt(const Json& lhs, const Json& rhs) -{ - bool result = false; - if (lhs.template is() && rhs.template is()) - { - result = lhs.template as() < rhs.template as(); - } - else if (lhs.template is() && rhs.template is()) - { - result = lhs.template as() < rhs.template as(); - } - else if (lhs.is_number() && rhs.is_number()) - { - result = lhs.as_double() < rhs.as_double(); - } - else if (lhs.is_string() && rhs.is_string()) - { - result = lhs.as_string_view() < rhs.as_string_view(); - } - return result; -} - -template -bool gt(const Json& lhs, const Json& rhs) -{ - return lt(rhs,lhs); -} - -template -Json plus(const Json& lhs, const Json& rhs) -{ - Json result = Json(jsoncons::null_type()); - if (lhs.is_int64() && rhs.is_int64()) - { - result = Json(((lhs.template as() + rhs.template as()))); - } - else if (lhs.is_uint64() && rhs.is_uint64()) - { - result = Json((lhs.template as() + rhs.template as())); - } - else if ((lhs.is_number() && rhs.is_number())) - { - result = Json((lhs.as_double() + rhs.as_double())); - } - return result; -} - -template -Json mult(const Json& lhs, const Json& rhs) -{ - Json result = Json(jsoncons::null_type()); - if (lhs.is_int64() && rhs.is_int64()) - { - result = Json(((lhs.template as() * rhs.template as()))); - } - else if (lhs.is_uint64() && rhs.is_uint64()) - { - result = Json((lhs.template as() * rhs.template as())); - } - else if ((lhs.is_number() && rhs.is_number())) - { - result = Json((lhs.as_double() * rhs.as_double())); - } - return result; -} - -template -Json div(const Json& lhs, const Json& rhs) -{ - Json result = Json(jsoncons::null_type()); - if (lhs.is_int64() && rhs.is_int64()) - { - result = Json((double)(lhs.template as() / (double)rhs.template as())); - } - else if (lhs.is_uint64() && rhs.is_uint64()) - { - result = Json((double)(lhs.template as() / (double)rhs.template as())); - } - else if ((lhs.is_number() && rhs.is_number())) - { - result = Json((lhs.as_double() / rhs.as_double())); - } - return result; -} - -template -Json unary_minus(const Json& lhs) -{ - Json result = Json::null(); - if (lhs.is_int64()) - { - result = -lhs.template as(); - } - else if (lhs.is_double()) - { - result = -lhs.as_double(); - } - return result; -} - -template -Json minus(const Json& lhs, const Json& rhs) -{ - Json result = Json::null(); - if (lhs.is_int64() && rhs.is_int64()) - { - result = ((lhs.template as() - rhs.template as())); - } - else if (lhs.is_uint64() && rhs.is_uint64() && lt(rhs,lhs)) - { - result = (lhs.template as() - rhs.template as()); - } - else if ((lhs.is_number() && rhs.is_number())) - { - result = (lhs.as_double() - rhs.as_double()); - } - return result; -} - -template -class value_term final : public term -{ - Json value_; -public: - value_term(const Json& val) - : value_(val) - { - } - value_term(Json&& val) - : value_(std::move(val)) - { - } - - void initialize(const Json&) override - { - } - - bool accept_single_node() const override - { - return value_.as_bool(); - } - - Json get_single_node() const override - { - return value_; - } - - bool exclaim() const override - { - return !value_.as_bool(); - } - - bool eq_term(const term& rhs) const override - { - return rhs.eq(value_); - } - - bool eq(const Json& rhs) const override - { - return value_ == rhs; - } - - bool ne_term(const term& rhs) const override - { - return rhs.ne(value_); - } - bool ne(const Json& rhs) const override - { - return value_ != rhs; - } - bool regex_term(const term& rhs) const override - { - return rhs.regex2(value_.as_string()); - } - bool ampamp_term(const term& rhs) const override - { - return rhs.ampamp(value_); - } - bool ampamp(const Json& rhs) const override - { - return jsoncons::jsonpath::detail::ampamp(value_,rhs); - } - bool pipepipe_term(const term& rhs) const override - { - return rhs.pipepipe(value_); - } - bool pipepipe(const Json& rhs) const override - { - return jsoncons::jsonpath::detail::pipepipe(value_,rhs); - } - - bool lt_term(const term& rhs) const override - { - return rhs.gt(value_); - } - bool lt(const Json& rhs) const override - { - return jsoncons::jsonpath::detail::lt(value_,rhs); - } - - bool gt_term(const term& rhs) const override - { - return rhs.lt(value_); - } - bool gt(const Json& rhs) const override - { - return jsoncons::jsonpath::detail::gt(value_,rhs); - } - - Json minus_term(const term& rhs) const override - { - return rhs.left_minus(value_); - } - Json minus(const Json& rhs) const override - { - return jsoncons::jsonpath::detail::minus(value_,rhs); - } - - Json left_minus(const Json& lhs) const override - { - return jsoncons::jsonpath::detail::minus(lhs,value_); - } - - Json unary_minus() const override - { - return jsoncons::jsonpath::detail::unary_minus(value_); - } - - Json plus_term(const term& rhs) const override - { - return rhs.plus(value_); - } - - Json plus(const Json& rhs) const override - { - return jsoncons::jsonpath::detail::plus(value_,rhs); - } - Json mult_term(const term& rhs) const override - { - return rhs.mult(value_); - } - - Json mult(const Json& rhs) const override - { - return jsoncons::jsonpath::detail::mult(value_,rhs); - } - - Json div_term(const term& rhs) const override - { - return rhs.left_div(value_); - } - Json div(const Json& rhs) const override - { - return jsoncons::jsonpath::detail::div(value_,rhs); - } - - Json left_div(const Json& lhs) const override - { - return jsoncons::jsonpath::detail::div(lhs,value_); - } -}; - -template -class regex_term final : public term -{ - typedef typename Json::char_type char_type; - typedef typename Json::string_type string_type; - const std::basic_regex pattern_; -public: - regex_term(const string_type& pattern, std::regex::flag_type flags) - : pattern_(pattern,flags) - { - } - - void initialize(const Json&) override - { - } - - bool regex2(const string_type& subject) const override - { - return std::regex_match(subject, pattern_); - } -}; - -template -class path_term final : public term -{ - typedef typename Json::string_type string_type; - - string_type path_; - size_t line_; - size_t column_; - Json nodes_; -public: - path_term(const string_type& path, size_t line, size_t column) - : path_(path), line_(line), column_(column) - { - } - - void initialize(const Json& current_node) override - { - jsonpath_evaluator> evaluator(line_,column_); - evaluator.evaluate(current_node, path_); - nodes_ = evaluator.get_values(); - } - - bool accept_single_node() const override - { - return nodes_.size() != 0; - } - - Json get_single_node() const override - { - return nodes_.size() == 1 ? nodes_[0] : nodes_; - } - - bool exclaim() const override - { - return nodes_.size() == 0; - } - - bool eq_term(const term& rhs) const override - { - bool result = false; - if (nodes_.size() > 0) - { - result = true; - for (size_t i = 0; result && i < nodes_.size(); ++i) - { - result = rhs.eq(nodes_[i]); - } - } - return result; - } - - bool eq(const Json& rhs) const override - { - bool result = false; - if (nodes_.size() > 0) - { - result = true; - for (size_t i = 0; result && i < nodes_.size(); ++i) - { - result = nodes_[i] == rhs; - } - } - return result; - } - - bool ne_term(const term& rhs) const override - { - bool result = false; - if (nodes_.size() > 0) - { - result = true; - for (size_t i = 0; result && i < nodes_.size(); ++i) - { - result = rhs.ne(nodes_[i]); - } - } - return result; - - } - bool ne(const Json& rhs) const override - { - bool result = false; - if (nodes_.size() > 0) - { - result = true; - for (size_t i = 0; result && i < nodes_.size(); ++i) - { - result = nodes_[i] != rhs; - } - } - return result; - } - bool regex_term(const term& rhs) const override - { - bool result = false; - if (nodes_.size() > 0) - { - result = true; - for (size_t i = 0; result && i < nodes_.size(); ++i) - { - result = rhs.regex2(nodes_[i].as_string()); - } - } - return result; - } - bool ampamp_term(const term& rhs) const override - { - bool result = false; - if (nodes_.size() > 0) - { - result = true; - for (size_t i = 0; result && i < nodes_.size(); ++i) - { - result = rhs.ampamp(nodes_[i]); - } - } - return result; - } - bool ampamp(const Json& rhs) const override - { - bool result = false; - if (nodes_.size() > 0) - { - result = true; - for (size_t i = 0; result && i < nodes_.size(); ++i) - { - result = jsoncons::jsonpath::detail::ampamp(nodes_[i],rhs); - } - } - return result; - } - bool pipepipe_term(const term& rhs) const override - { - bool result = false; - if (nodes_.size() > 0) - { - result = true; - for (size_t i = 0; result && i < nodes_.size(); ++i) - { - result = rhs.pipepipe(nodes_[i]); - } - } - return result; - } - bool pipepipe(const Json& rhs) const override - { - bool result = false; - if (nodes_.size() > 0) - { - result = true; - for (size_t i = 0; result && i < nodes_.size(); ++i) - { - result = jsoncons::jsonpath::detail::pipepipe(nodes_[i],rhs); - } - } - return result; - } - - bool lt(const Json& rhs) const override - { - bool result = false; - if (nodes_.size() > 0) - { - result = true; - for (size_t i = 0; result && i < nodes_.size(); ++i) - { - result = jsoncons::jsonpath::detail::lt(nodes_[i],rhs); - } - } - return result; - } - - bool lt_term(const term& rhs) const override - { - bool result = false; - if (nodes_.size() > 0) - { - result = true; - for (size_t i = 0; result && i < nodes_.size(); ++i) - { - result = rhs.gt(nodes_[i]); - } - } - return result; - } - - bool gt(const Json& rhs) const override - { - bool result = false; - if (nodes_.size() > 0) - { - result = true; - for (size_t i = 0; result && i < nodes_.size(); ++i) - { - result = jsoncons::jsonpath::detail::gt(nodes_[i],rhs); - } - } - return result; - } - - bool gt_term(const term& rhs) const override - { - bool result = false; - if (nodes_.size() > 0) - { - result = true; - for (size_t i = 0; result && i < nodes_.size(); ++i) - { - result = rhs.lt(nodes_[i]); - } - } - return result; - } - - Json minus_term(const term& rhs) const override - { - static auto a_null = Json(jsoncons::null_type()); - return nodes_.size() == 1 ? rhs.left_minus(nodes_[0]) : a_null; - } - Json minus(const Json& rhs) const override - { - return nodes_.size() == 1 ? jsoncons::jsonpath::detail::minus(nodes_[0],rhs) : Json(jsoncons::null_type()); - } - - Json left_minus(const Json& lhs) const override - { - static auto a_null = Json(jsoncons::null_type()); - return nodes_.size() == 1 ? jsoncons::jsonpath::detail::minus(lhs,nodes_[0]) : a_null; - } - - Json unary_minus() const override - { - return nodes_.size() == 1 ? jsoncons::jsonpath::detail::unary_minus(nodes_[0]) : Json::null(); - } - - Json plus_term(const term& rhs) const override - { - static auto a_null = Json(jsoncons::null_type()); - return nodes_.size() == 1 ? rhs.plus(nodes_[0]) : a_null; - } - Json plus(const Json& rhs) const override - { - static auto a_null = Json(jsoncons::null_type()); - return nodes_.size() == 1 ? jsoncons::jsonpath::detail::plus(nodes_[0],rhs) : a_null; - } - - Json mult_term(const term& rhs) const override - { - static auto a_null = Json(jsoncons::null_type()); - return nodes_.size() == 1 ? rhs.mult(nodes_[0]) : a_null; - } - Json mult(const Json& rhs) const override - { - static auto a_null = Json(jsoncons::null_type()); - return nodes_.size() == 1 ? jsoncons::jsonpath::detail::mult(nodes_[0],rhs) : a_null; - } - - Json div_term(const term& rhs) const override - { - static auto a_null = Json(jsoncons::null_type()); - return nodes_.size() == 1 ? rhs.left_div(nodes_[0]) : a_null; - } - Json div(const Json& rhs) const override - { - static auto a_null = Json(jsoncons::null_type()); - return nodes_.size() == 1 ? jsoncons::jsonpath::detail::div(nodes_[0],rhs) : a_null; - } - - Json left_div(const Json& lhs) const override - { - static auto a_null = Json(jsoncons::null_type()); - return nodes_.size() == 1 ? jsoncons::jsonpath::detail::div(lhs, nodes_[0]) : a_null; - } -}; - -template -token evaluate(const Json& context, std::vector>& tokens) -{ - for (auto it= tokens.begin(); it != tokens.end(); ++it) - { - it->initialize(context); - } - std::vector> stack; - for (auto t : tokens) - { - if (t.is_operand()) - { - stack.push_back(t); - } - else if (t.is_unary_operator()) - { - auto rhs = stack.back(); - stack.pop_back(); - Json val = t(rhs.operand()); - stack.push_back(token(token_type::operand,std::make_shared>(std::move(val)))); - } - else if (t.is_binary_operator()) - { - auto rhs = stack.back(); - stack.pop_back(); - auto lhs = stack.back(); - stack.pop_back(); - Json val = t(lhs.operand(), rhs.operand()); - stack.push_back(token(token_type::operand,std::make_shared>(std::move(val)))); - } - } - if (stack.size() != 1) - { - JSONCONS_THROW(json_runtime_error("Invalid state")); - } - - return stack.back(); -} - -template -class jsonpath_filter_expr -{ -public: - std::vector> tokens_; -public: - jsonpath_filter_expr() - { - } - - jsonpath_filter_expr(const std::vector>& tokens) - : tokens_(tokens) - { - } - - Json eval(const Json& current_node) - { - auto t = evaluate(current_node, tokens_); - return t.operand().get_single_node(); - } - - bool exists(const Json& current_node) - { - auto t = evaluate(current_node,tokens_); - return t.operand().accept_single_node(); - } -}; - -template -class jsonpath_filter_parser -{ - typedef typename Json::string_type string_type; - typedef typename Json::string_view_type string_view_type; - typedef typename Json::char_type char_type; - - std::vector> output_stack_; - std::vector> operator_stack_; - std::vector state_stack_; - std::vector path_mode_stack_; - - size_t line_; - size_t column_; - - static const operator_properties op_properties_[]; - - class binary_operator_table - { - typedef std::map> binary_operator_map; - - const binary_operator_map operators = - { - {eqtilde_literal(),{2,false,[](const term& a, const term& b) {return Json(a.regex_term(b)); }}}, - {star_literal(),{3,false,[](const term& a, const term& b) {return a.mult_term(b); }}}, - {forwardslash_literal(),{3,false,[](const term& a, const term& b) {return a.div_term(b); }}}, - {plus_literal(),{4,false,[](const term& a, const term& b) {return a.plus_term(b); }}}, - {minus_literal(),{4,false,[](const term& a, const term& b) {return a.minus_term(b); }}}, - {lt_literal(),{5,false,[](const term& a, const term& b) {return Json(a.lt_term(b)); }}}, - {lte_literal(),{5,false,[](const term& a, const term& b) {return Json(a.lt_term(b) || a.eq_term(b)); }}}, - {gt_literal(),{5,false,[](const term& a, const term& b) {return Json(a.gt_term(b)); }}}, - {gte_literal(),{5,false,[](const term& a, const term& b) {return Json(a.gt_term(b) || a.eq_term(b)); }}}, - {eq_literal(),{6,false,[](const term& a, const term& b) {return Json(a.eq_term(b)); }}}, - {ne_literal(),{6,false,[](const term& a, const term& b) {return Json(a.ne_term(b)); }}}, - {ampamp_literal(),{7,false,[](const term& a, const term& b) {return Json(a.ampamp_term(b)); }}}, - {pipepipe_literal(),{8,false,[](const term& a, const term& b) {return Json(a.pipepipe_term(b)); }}} - }; - - public: - typename binary_operator_map::const_iterator find(const string_type& key) const - { - return operators.find(key); - } - typename binary_operator_map::const_iterator end() const - { - return operators.end(); - } - }; - - binary_operator_table binary_operators_; - -public: - jsonpath_filter_parser() - : line_(1), column_(1) - { - } - jsonpath_filter_parser(size_t line, size_t column) - : line_(line), column_(column) - { - } - - size_t line() const - { - return line_; - } - - size_t column() const - { - return column_; - } - - void push_state(filter_state state) - { - state_stack_.push_back(state); - } - - filter_state pop_state() - { - JSONCONS_ASSERT(!state_stack_.empty()) - filter_state state = state_stack_.back(); - state_stack_.pop_back(); - return state; - } - - void push_token(token token) - { - switch (token.type()) - { - case token_type::operand: - output_stack_.push_back(token); - break; - case token_type::lparen: - operator_stack_.push_back(token); - break; - case token_type::rparen: - { - auto it = operator_stack_.rbegin(); - while (it != operator_stack_.rend() && !it->is_lparen()) - { - output_stack_.push_back(*it); - ++it; - } - if (it == operator_stack_.rend()) - { - JSONCONS_THROW(json_runtime_error("Unbalanced parenthesis")); - } - operator_stack_.erase(it.base(),operator_stack_.end()); - operator_stack_.pop_back(); - break; - } - case token_type::unary_operator: - case token_type::binary_operator: - { - if (operator_stack_.empty() || operator_stack_.back().is_lparen()) - { - operator_stack_.push_back(token); - } - else if (token.precedence_level() < operator_stack_.back().precedence_level() - || (token.precedence_level() == operator_stack_.back().precedence_level() && token.is_right_associative())) - { - operator_stack_.push_back(token); - } - else - { - auto it = operator_stack_.rbegin(); - while (it != operator_stack_.rend() && it->is_operator() - && (token.precedence_level() > it->precedence_level() - || (token.precedence_level() == it->precedence_level() && token.is_right_associative()))) - { - output_stack_.push_back(*it); - ++it; - } - - operator_stack_.erase(it.base(),operator_stack_.end()); - operator_stack_.push_back(token); - } - break; - } - default: - break; - } - } - - jsonpath_filter_expr parse(const Json& root, const char_type* p, const char_type* end_expr, const char_type** end_ptr) - { - output_stack_.clear(); - operator_stack_.clear(); - state_stack_.clear(); - - string_type buffer; - size_t buffer_line = 1; - size_t buffer_column = 1; - - int depth = 0; - filter_state state = filter_state::start; - while (p < end_expr && state != filter_state::done) - { - switch (state) - { - case filter_state::start: - switch (*p) - { - case '\r': - if (p+1 < end_expr && *(p+1) == '\n') - ++p; - ++line_; - column_ = 1; - ++p; - break; - case '\n': - ++line_; - column_ = 1; - ++p; - break; - case '(': - state = filter_state::expect_path_or_value_or_unary_op; - ++depth; - push_token(token(token_type::lparen)); - break; - case ')': - state = filter_state::expect_path_or_value_or_unary_op; - push_token(token(token_type::rparen)); - if (--depth == 0) - { - state = filter_state::done; - } - break; - } - ++p; - ++column_; - break; - - case filter_state::expect_arg: - { - switch (*p) - { - case ' ':case '\t': - break; - case '\r': - if (p+1 < end_expr && *(p+1) == '\n') - ++p; - ++line_; - column_ = 1; - ++p; - break; - case '\n': - ++line_; - column_ = 1; - ++p; - break; - case '$': - buffer.push_back(*p); - path_mode_stack_.back() = filter_path_mode::root_path; - state = filter_state::path_argument; - break; - case '@': - buffer.push_back('$'); - path_mode_stack_.back() = filter_path_mode::current_path; - state = filter_state::path_argument; - break; - // Maybe error from here down - case '\'': - buffer.push_back('\"'); - state = filter_state::single_quoted_argument; - break; - case '\"': - buffer.push_back('\"'); - state = filter_state::double_quoted_argument; - break; - default: - buffer.push_back(*p); - state = filter_state::unquoted_argument; - break; - } - ++p; - ++column_; - break; - } - - case filter_state::path_argument: - { - switch (*p) - { - case '\r': - if (p+1 < end_expr && *(p+1) == '\n') - ++p; - ++line_; - column_ = 1; - ++p; - break; - case '\n': - ++line_; - column_ = 1; - ++p; - break; - case ' ':case '\t': - break; - case ',': - buffer.push_back(*p); - state = filter_state::expect_arg; - break; - case ')': - { - buffer.push_back(*p); - state = filter_state::path; - break; - } - default: - buffer.push_back(*p); - break; - } - ++p; - ++column_; - break; - } - case filter_state::single_quoted_argument: - { - switch (*p) - { - case '\'': - buffer.push_back('\"'); - state = filter_state::expect_more_args_or_right_round_bracket; - break; - default: - buffer.push_back(*p); - break; - } - ++p; - ++column_; - break; - } - case filter_state::double_quoted_argument: - { - switch (*p) - { - case '\"': - buffer.push_back('\"'); - state = filter_state::expect_more_args_or_right_round_bracket; - break; - default: - buffer.push_back(*p); - break; - } - ++p; - ++column_; - break; - } - case filter_state::unquoted_argument: - { - switch (*p) - { - case ',': - buffer.push_back(*p); - state = filter_state::expect_arg; - break; - case ')': - { - buffer.push_back(*p); - state = filter_state::path; - break; - } - default: - buffer.push_back(*p); - break; - } - ++p; - ++column_; - break; - } - case filter_state::expect_more_args_or_right_round_bracket: - { - switch (*p) - { - case ' ': - case '\t': - break; - case ',': - buffer.push_back(*p); - state = filter_state::expect_arg; - break; - case ')': - { - buffer.push_back(*p); - state = filter_state::path; - break; - } - default: - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator, line_, column_); - } - ++p; - ++column_; - break; - } - - case filter_state::oper: - switch (*p) - { - case '~': - { - buffer.push_back(*p); - ++p; - ++column_; - auto it = binary_operators_.find(buffer); - if (it == binary_operators_.end()) - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator, line_, column_); - } - buffer.clear(); - buffer_line = buffer_column = 1; - push_token(token(it->second)); - state = filter_state::expect_regex; - break; - } - case '=': - case '&': - case '|': - { - buffer.push_back(*p); - ++p; - ++column_; - auto it = binary_operators_.find(buffer); - if (it == binary_operators_.end()) - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator, line_, column_); - } - buffer.clear(); - buffer_line = buffer_column = 1; - push_token(token(it->second)); - state = filter_state::expect_path_or_value_or_unary_op; - break; - } - default: - { - auto it = binary_operators_.find(buffer); - if (it == binary_operators_.end()) - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unsupported_operator, line_, column_); - } - buffer.clear(); - buffer_line = buffer_column = 1; - push_token(token(it->second)); - state = filter_state::expect_path_or_value_or_unary_op; - break; - } - } - break; - case filter_state::unquoted_text: - { - switch (*p) - { - case ' ':case '\t': - if (buffer.length() > 0) - { - try - { - auto val = Json::parse(buffer); - push_token(token(token_type::operand,std::make_shared>(std::move(val)))); - } - catch (const ser_error&) - { - throw jsonpath_error(jsonpath_errc::parse_error_in_filter,line_,column_); - } - buffer.clear(); - buffer_line = buffer_column = 1; - } - ++p; - ++column_; - break; - case '(': - { - buffer.push_back(*p); - path_mode_stack_.push_back(filter_path_mode::path); - state = filter_state::expect_arg; - ++p; - ++column_; - break; - } - case '<': - case '>': - case '!': - case '=': - case '&': - case '|': - case '+': - case '-': - case '*': - case '/': - { - if (buffer.length() > 0) - { - try - { - auto val = Json::parse(buffer); - push_token(token(token_type::operand,std::make_shared>(std::move(val)))); - } - catch (const ser_error&) - { - throw jsonpath_error(jsonpath_errc::parse_error_in_filter,line_,column_); - } - buffer.clear(); - buffer_line = buffer_column = 1; - } - buffer.push_back(*p); - state = filter_state::oper; - ++p; - ++column_; - } - break; - case ')': - if (buffer.length() > 0) - { - try - { - auto val = Json::parse(buffer); - push_token(token(token_type::operand,std::make_shared>(std::move(val)))); - } - catch (const ser_error&) - { - throw jsonpath_error(jsonpath_errc::parse_error_in_filter,line_,column_); - } - buffer.clear(); - buffer_line = buffer_column = 1; - } - push_token(token(token_type::rparen)); - if (--depth == 0) - { - state = filter_state::done; - } - else - { - state = filter_state::expect_path_or_value_or_unary_op; - } - ++p; - ++column_; - break; - default: - buffer.push_back(*p); - ++p; - ++column_; - break; - } - } - break; - case filter_state::single_quoted_text: - { - switch (*p) - { - case '\\': - buffer.push_back(*p); - if (p+1 < end_expr) - { - ++p; - ++column_; - buffer.push_back(*p); - } - break; - case '\'': - buffer.push_back('\"'); - //if (buffer.length() > 0) - { - try - { - auto val = Json::parse(buffer); - push_token(token(token_type::operand,std::make_shared>(std::move(val)))); - } - catch (const ser_error&) - { - throw jsonpath_error(jsonpath_errc::parse_error_in_filter,line_,column_); - } - buffer.clear(); - buffer_line = buffer_column = 1; - } - state = filter_state::expect_path_or_value_or_unary_op; - break; - - default: - buffer.push_back(*p); - break; - } - } - ++p; - ++column_; - break; - case filter_state::double_quoted_text: - { - switch (*p) - { - case '\\': - buffer.push_back(*p); - if (p+1 < end_expr) - { - ++p; - ++column_; - buffer.push_back(*p); - } - break; - case '\"': - buffer.push_back(*p); - try - { - auto val = Json::parse(buffer); - push_token(token(token_type::operand,std::make_shared>(std::move(val)))); - } - catch (const ser_error&) - { - throw jsonpath_error(jsonpath_errc::parse_error_in_filter,line_,column_); - } - buffer.clear(); - buffer_line = buffer_column = 1; - state = filter_state::expect_path_or_value_or_unary_op; - break; - - default: - buffer.push_back(*p); - break; - } - } - ++p; - ++column_; - break; - case filter_state::expect_path_or_value_or_unary_op: - switch (*p) - { - case '\r': - if (p+1 < end_expr && *(p+1) == '\n') - ++p; - ++line_; - column_ = 1; - ++p; - break; - case '\n': - ++line_; - column_ = 1; - ++p; - break; - case ' ':case '\t': - ++p; - ++column_; - break; - case '!': - { - std::function&)> f = [](const term& b) {return Json(b.exclaim());}; - push_token(token(1, true, f)); - ++p; - ++column_; - break; - } - case '-': - { - std::function&)> f = [](const term& b) {return b.unary_minus();}; - push_token(token(1, true, f)); - ++p; - ++column_; - break; - } - case '@': - buffer_line = line_; - buffer_column = column_; - buffer.push_back('$'); - state = filter_state::path; - ++p; - ++column_; - break; - case '\'': - buffer.push_back('\"'); - state = filter_state::single_quoted_text; - ++p; - ++column_; - break; - case '\"': - buffer.push_back(*p); - state = filter_state::double_quoted_text; - ++p; - ++column_; - break; - case '(': - ++depth; - push_token(token(token_type::lparen)); - ++p; - ++column_; - break; - case ')': - push_token(token(token_type::rparen)); - if (--depth == 0) - { - state = filter_state::done; - } - ++p; - ++column_; - break; - default: - // don't increment - state = filter_state::unquoted_text; - break; - }; - break; - case filter_state::expect_oper_or_right_round_bracket: - switch (*p) - { - case '\r': - if (p+1 < end_expr && *(p+1) == '\n') - ++p; - ++line_; - column_ = 1; - ++p; - break; - case '\n': - ++line_; - column_ = 1; - ++p; - break; - case ' ':case '\t': - ++p; - ++column_; - break; - case ')': - push_token(token(token_type::rparen)); - if (--depth == 0) - { - state = filter_state::done; - ++p; // fix - } - break; - case '<': - case '>': - case '!': - case '=': - case '&': - case '|': - case '+': - case '-': - case '*': - case '/': - { - buffer.push_back(*p); - state = filter_state::oper; - ++p; - ++column_; - } - break; - default: - throw jsonpath_error(jsonpath_errc::invalid_filter,line_,column_); - break; - }; - break; - case filter_state::expect_right_round_bracket: - switch (*p) - { - case '\r': - if (p+1 < end_expr && *(p+1) == '\n') - ++p; - ++line_; - column_ = 1; - ++p; - break; - case '\n': - ++line_; - column_ = 1; - ++p; - break; - case ' ':case '\t': - break; - case ')': - push_token(token(token_type::rparen)); - if (--depth == 0) - { - state = filter_state::done; - } - else - { - state = filter_state::expect_oper_or_right_round_bracket; - } - break; - default: - throw jsonpath_error(jsonpath_errc::invalid_filter,line_,column_); - break; - }; - ++p; - ++column_; - break; - case filter_state::path: - switch (*p) - { - case '<': - case '>': - case '!': - case '=': - case '&': - case '|': - case '+': - case '-': - case '*': - case '/': - { - if (!path_mode_stack_.empty()) - { - if (path_mode_stack_[0] == filter_path_mode::root_path) - { - jsonpath_evaluator> evaluator(buffer_line,buffer_column); - evaluator.evaluate(root, buffer); - auto result = evaluator.get_values(); - if (result.size() > 0) - { - push_token(token(token_type::operand,std::make_shared>(std::move(result[0])))); - } - } - else - { - push_token(token(token_type::operand,std::make_shared>(buffer, buffer_line, buffer_column))); - } - path_mode_stack_.pop_back(); - } - else - { - push_token(token(token_type::operand,std::make_shared>(buffer, buffer_line, buffer_column))); - } - buffer.clear(); - buffer_line = buffer_column = 1; - buffer.push_back(*p); - ++p; - ++column_; - state = filter_state::oper; - } - break; - case ')': - if (!path_mode_stack_.empty()) - { - if (path_mode_stack_[0] == filter_path_mode::root_path) - { - jsonpath_evaluator> evaluator(buffer_line,buffer_column); - evaluator.evaluate(root, buffer); - auto result = evaluator.get_values(); - if (result.size() > 0) - { - push_token(token(token_type::operand,std::make_shared>(std::move(result[0])))); - } - push_token(token(token_type::rparen)); - } - else - { - push_token(token(token_type::operand,std::make_shared>(buffer, buffer_line, buffer_column))); - } - path_mode_stack_.pop_back(); - } - else - { - push_token(token(token_type::operand,std::make_shared>(buffer, buffer_line, buffer_column))); - push_token(token(token_type::rparen)); - } - buffer.clear(); - buffer_line = buffer_column = 1; - if (--depth == 0) - { - state = filter_state::done; - } - else - { - state = filter_state::expect_path_or_value_or_unary_op; - } - ++p; - ++column_; - break; - default: - buffer.push_back(*p); - ++p; - ++column_; - break; - }; - break; - case filter_state::expect_regex: - switch (*p) - { - case '\r': - if (p+1 < end_expr && *(p+1) == '\n') - ++p; - ++line_; - column_ = 1; - ++p; - break; - case '\n': - ++line_; - column_ = 1; - ++p; - break; - case ' ':case '\t': - break; - case '/': - state = filter_state::regex; - break; - default: - throw jsonpath_error(jsonpath_errc::invalid_filter_expected_slash,line_,column_); - break; - }; - ++p; - ++column_; - break; - case filter_state::regex: - { - switch (*p) - { - case '/': - //if (buffer.length() > 0) - { - std::regex::flag_type flags = std::regex_constants::ECMAScript; - if (p+1 < end_expr && *(p+1) == 'i') - { - ++p; - ++column_; - flags |= std::regex_constants::icase; - } - push_token(token(token_type::operand,std::make_shared>(buffer,flags))); - buffer.clear(); - buffer_line = buffer_column = 1; - } - state = filter_state::expect_path_or_value_or_unary_op; - break; - - default: - buffer.push_back(*p); - break; - } - ++p; - ++column_; - break; - } - default: - ++p; - ++column_; - break; - } - } - if (depth != 0) - { - throw jsonpath_error(jsonpath_errc::invalid_filter_unbalanced_paren,line_,column_); - } - *end_ptr = p; - - return jsonpath_filter_expr(output_stack_); - } -}; - -template -const operator_properties jsonpath_filter_parser::op_properties_[] = -{ - {2,false,[](const term& a, const term& b) {return Json(a.regex_term(b));}}, - {3,false,[](const term& a, const term& b) {return a.mult_term(b);}}, - {3,false,[](const term& a, const term& b) {return a.div_term(b);}}, - {4,false,[](const term& a, const term& b) {return a.plus_term(b);}}, - {4,false,[](const term& a, const term& b) {return a.minus_term(b);}}, - {5,false,[](const term& a, const term& b) {return Json(a.lt_term(b));}}, - {5,false,[](const term& a, const term& b) {return Json(a.lt_term(b) || a.eq_term(b));}}, - {5,false,[](const term& a, const term& b) {return Json(a.gt_term(b));}}, - {5,false,[](const term& a, const term& b) {return Json(a.gt_term(b) || a.eq_term(b));}}, - {6,false,[](const term& a, const term& b) {return Json(a.eq_term(b)); }}, - {6,false,[](const term& a, const term& b) {return Json(a.ne_term(b)); }}, - {7,false,[](const term& a, const term& b) {return Json(a.ampamp_term(b));}}, - {8,false,[](const term& a, const term& b) {return Json(a.pipepipe_term(b));}} -}; - -}}} - -#endif \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpath/jsonpath_function.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpath/jsonpath_function.hpp deleted file mode 100644 index 793ea2aa58..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpath/jsonpath_function.hpp +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSONPATH_JSONPATH_FUNCTION_HPP -#define JSONCONS_JSONPATH_JSONPATH_FUNCTION_HPP - -#include // std::basic_string -#include // std::vector -#include // std::unordered_map -#include // std::numeric_limits -#include // std::move -#include - -namespace jsoncons { namespace jsonpath { - -JSONCONS_STRING_LITERAL(keys,'k','e','y','s') -JSONCONS_STRING_LITERAL(avg,'a','v','g') -JSONCONS_STRING_LITERAL(max,'m','a','x') -JSONCONS_STRING_LITERAL(min,'m','i','n') -JSONCONS_STRING_LITERAL(sum,'s','u','m') -JSONCONS_STRING_LITERAL(prod,'p','r','o','d') -JSONCONS_STRING_LITERAL(count,'c','o','u','n','t') -JSONCONS_STRING_LITERAL(tokenize,'t','o','k','e','n','i','z','e') - -template -class function_table -{ -public: - typedef typename Json::char_type char_type; - typedef typename Json::char_traits_type char_traits_type; - typedef std::basic_string string_type; - typedef typename Json::string_view_type string_view_type; - typedef JsonPointer pointer; - typedef std::vector argument_type; - typedef std::function&, std::error_code&)> function_type; - typedef std::unordered_map function_dictionary; -private: - const function_dictionary functions_ = - { - { - keys_literal(),[](const std::vector& args, std::error_code& ec) - { - Json j = typename Json::array(); - if (args.size() != 1) - { - ec = jsonpath_errc::invalid_argument; - return j; - } - if (args[0].size() != 1 && !args[0][0]->is_object()) - { - return j; - } - pointer arg = args[0][0]; - for (const auto& kv : arg->object_range()) - { - j.emplace_back(kv.key()); - } - - return j; - } - }, - { - max_literal(),[](const std::vector& args, std::error_code& ec) - { - if (args.size() != 1) - { - ec = jsonpath_errc::invalid_argument; - return Json(); - } - const auto& arg = args[0]; - double v = std::numeric_limits::lowest(); - for (auto& node : arg) - { - double x = node->template as(); - if (x > v) - { - v = x; - } - } - return Json(v); - } - }, - { - min_literal(),[](const std::vector& args, std::error_code& ec) - { - if (args.size() != 1) - { - ec = jsonpath_errc::invalid_argument; - return Json(); - } - const auto& arg = args[0]; - double v = (std::numeric_limits::max)(); - for (const auto& node : arg) - { - double x = node->template as(); - if (x < v) - { - v = x; - } - } - return Json(v); - } - }, - { - avg_literal(),[](const std::vector& args, std::error_code& ec) - { - if (args.size() != 1) - { - ec = jsonpath_errc::invalid_argument; - return Json(); - } - const auto& arg = args[0]; - double v = 0.0; - for (const auto& node : arg) - { - v += node->template as(); - } - return arg.size() > 0 ? Json(v/arg.size()) : Json(null_type()); - } - }, - { - sum_literal(),[](const std::vector& args, std::error_code& ec) - { - if (args.size() != 1) - { - ec = jsonpath_errc::invalid_argument; - return Json(); - } - const auto& arg = args[0]; - double v = 0.0; - for (const auto& node : arg) - { - v += node->template as(); - } - return Json(v); - } - }, - { - count_literal(),[](const std::vector& args, std::error_code& ec) - { - if (args.size() != 1) - { - ec = jsonpath_errc::invalid_argument; - return Json(); - } - const auto& arg = args[0]; - size_t count = 0; - while (count < arg.size()) - { - ++count; - } - return Json(count); - } - }, - { - prod_literal(),[](const std::vector& args, std::error_code& ec) - { - if (args.size() != 1) - { - ec = jsonpath_errc::invalid_argument; - return Json(); - } - const auto& arg = args[0]; - double v = 0.0; - for (const auto& node : arg) - { - double x = node->template as(); - v == 0.0 && x != 0.0 - ? (v = x) - : (v *= x); - - } - return Json(v); - } - } -#if !(defined(__GNUC__) && (__GNUC__ == 4 && __GNUC_MINOR__ < 9)) -// GCC 4.8 has broken regex support: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53631 - ,{ - tokenize_literal(),[](const std::vector& args, std::error_code& ec) - { - if (args.size() != 2) - { - ec = jsonpath_errc::invalid_argument; - return Json(); - } - string_type arg1 = args[0][0]->as_string(); - string_type arg2 = args[1][0]->as_string(); - - std::regex::flag_type flags = std::regex_constants::ECMAScript; - std::basic_regex pieces_regex(arg2, flags); - - std::regex_token_iterator rit ( arg1.begin(), arg1.end(), pieces_regex, -1); - std::regex_token_iterator rend; - - Json j = typename Json::array(); - while (rit!=rend) - { - j.push_back(rit->str()); - ++rit; - } - return j; - } - } -#endif - }; -public: - function_type get(const string_type& name, std::error_code& ec) const - { - auto it = functions_.find(name); - if (it == functions_.end()) - { - //std::cout << "Function name " << name << " not found\n"; - ec = jsonpath_errc::function_name_not_found; - return nullptr; - } - return it->second; - } -}; - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpointer/jsonpointer.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpointer/jsonpointer.hpp deleted file mode 100644 index ec1b00824d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpointer/jsonpointer.hpp +++ /dev/null @@ -1,959 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSONPOINTER_JSONPOINTER_HPP -#define JSONCONS_JSONPOINTER_JSONPOINTER_HPP - -#include -#include -#include -#include -#include -#include -#include // std::move -#include // system_error -#include // std::enable_if, std::true_type -#include -#include -#include - -namespace jsoncons { namespace jsonpointer { - -// find_by_reference - -template -struct is_accessible_by_reference : std::false_type {}; - -template -struct is_accessible_by_reference().at(typename J::string_view_type()))>::value - && std::is_reference().at(0))>::value>::type> -: std::true_type {}; - -namespace detail { - -enum class pointer_state -{ - start, - escaped, - delim -}; - -} // detail - -// address_iterator -template -class address_iterator -{ - typedef typename std::iterator_traits::value_type char_type; - typedef std::basic_string string_type; - typedef InputIt base_iterator; - - base_iterator path_ptr_; - base_iterator end_input_; - base_iterator p_; - base_iterator q_; - jsonpointer::detail::pointer_state state_; - size_t line_; - size_t column_; - std::basic_string buffer_; -public: - typedef string_type value_type; - typedef std::ptrdiff_t difference_type; - typedef value_type* pointer; - typedef const value_type& reference; - typedef std::input_iterator_tag iterator_category; - - address_iterator(base_iterator first, base_iterator last) - : address_iterator(first, last, first) - { - std::error_code ec; - increment(ec); - } - - address_iterator(base_iterator first, base_iterator last, base_iterator current) - : path_ptr_(first), end_input_(last), p_(current), q_(current), state_(jsonpointer::detail::pointer_state::start) - { - } - - address_iterator(const address_iterator&) = default; - - address_iterator(address_iterator&&) = default; - - address_iterator& operator=(const address_iterator&) = default; - - address_iterator& operator=(address_iterator&&) = default; - - address_iterator& operator++() - { - std::error_code ec; - increment(ec); - if (ec) - { - throw jsonpointer_error(ec); - } - return *this; - } - - address_iterator& increment(std::error_code& ec) - { - q_ = p_; - buffer_.clear(); - - bool done = false; - while (p_ != end_input_ && !done) - { - switch (state_) - { - case jsonpointer::detail::pointer_state::start: - switch (*p_) - { - case '/': - state_ = jsonpointer::detail::pointer_state::delim; - break; - default: - ec = jsonpointer_errc::expected_slash; - done = true; - break; - }; - break; - case jsonpointer::detail::pointer_state::delim: - switch (*p_) - { - case '/': - state_ = jsonpointer::detail::pointer_state::delim; - done = true; - break; - case '~': - state_ = jsonpointer::detail::pointer_state::escaped; - break; - default: - buffer_.push_back(*p_); - break; - }; - break; - case jsonpointer::detail::pointer_state::escaped: - switch (*p_) - { - case '0': - buffer_.push_back('~'); - state_ = jsonpointer::detail::pointer_state::delim; - break; - case '1': - buffer_.push_back('/'); - state_ = jsonpointer::detail::pointer_state::delim; - break; - default: - ec = jsonpointer_errc::expected_0_or_1; - done = true; - break; - }; - break; - default: - JSONCONS_UNREACHABLE(); - break; - } - ++p_; - ++column_; - } - return *this; - } - - address_iterator operator++(int) // postfix increment - { - address_iterator temp(*this); - ++(*this); - return temp; - } - - reference operator*() const - { - return buffer_; - } - - friend bool operator==(const address_iterator& it1, const address_iterator& it2) - { - return it1.q_ == it2.q_; - } - friend bool operator!=(const address_iterator& it1, const address_iterator& it2) - { - return !(it1 == it2); - } - -private: -}; - -template -std::basic_string escape_string(const std::basic_string& s) -{ - std::basic_string result; - for (auto c : s) - { - switch (c) - { - case '~': - result.push_back('~'); - result.push_back('0'); - break; - case '/': - result.push_back('~'); - result.push_back('1'); - break; - default: - result.push_back(c); - break; - } - } - return result; -} - -// address - -template -class basic_address -{ -public: - std::basic_string path_; -public: - // Member types - typedef CharT char_type; - typedef std::basic_string string_type; - typedef basic_string_view string_view_type; - typedef address_iterator const_iterator; - typedef const_iterator iterator; - - // Constructors - basic_address() - { - } - explicit basic_address(const string_type& s) - : path_(s) - { - } - explicit basic_address(string_type&& s) - : path_(std::move(s)) - { - } - explicit basic_address(const CharT* s) - : path_(s) - { - } - - basic_address(const basic_address&) = default; - - basic_address(basic_address&&) = default; - - // operator= - basic_address& operator=(const basic_address&) = default; - - basic_address& operator=(basic_address&&) = default; - - // Modifiers - - void clear() - { - path_.clear(); - } - - basic_address& operator/=(const string_type& s) - { - path_.push_back('/'); - path_.append(escape_string(s)); - - return *this; - } - - basic_address& operator+=(const basic_address& p) - { - path_.append(p.path_); - return *this; - } - - // Accessors - bool empty() const - { - return path_.empty(); - } - - const string_type& string() const - { - return path_; - } - - operator string_view_type() const - { - return path_; - } - - // Iterators - iterator begin() const - { - return iterator(path_.begin(),path_.end()); - } - iterator end() const - { - return iterator(path_.begin(), path_.end(), path_.end()); - } - - // Non-member functions - friend basic_address operator/(const basic_address& lhs, const string_type& rhs) - { - basic_address p(lhs); - p /= rhs; - return p; - } - - friend basic_address operator+( const basic_address& lhs, const basic_address& rhs ) - { - basic_address p(lhs); - p += rhs; - return p; - } - - friend bool operator==( const basic_address& lhs, const basic_address& rhs ) - { - return lhs.path_ == rhs.path_; - } - - friend bool operator!=( const basic_address& lhs, const basic_address& rhs ) - { - return lhs.path_ != rhs.path_; - } - - friend std::basic_ostream& - operator<<( std::basic_ostream& os, const basic_address& p ) - { - os << p.path_; - return os; - } -}; - -typedef basic_address address; - -namespace detail { - -template -class handle_type -{ -public: - using value_type = typename J::value_type; - using type = value_type; - - handle_type(const value_type& val) noexcept - : val_(val) - { - } - - handle_type(value_type&& val) noexcept - : val_(std::move(val)) - { - } - - handle_type(const handle_type& w) noexcept - : val_(w.val_) - { - } - - handle_type& operator=(const handle_type&) noexcept = default; - - type get() const noexcept - { - return val_; - } -private: - value_type val_; -}; - -template -class handle_type::value>::type> -{ -public: - using reference = JReference; - using type = reference; - using pointer = typename std::conditional::type>::value,typename J::const_pointer,typename J::pointer>::type; - - handle_type(reference ref) noexcept - : ptr_(std::addressof(ref)) - { - } - - handle_type(const handle_type&) noexcept = default; - - handle_type& operator=(const handle_type&) noexcept = default; - - type get() const noexcept - { - return *ptr_; - } -private: - pointer ptr_; -}; - -template -class jsonpointer_evaluator : public ser_context -{ - typedef typename handle_type::type type; - typedef typename J::string_type string_type; - typedef typename string_type::value_type char_type; - typedef typename J::string_view_type string_view_type; - using reference = JReference; - using pointer = typename std::conditional::type>::value,typename J::const_pointer,typename J::pointer>::type; - - size_t line_; - size_t column_; - string_type buffer_; - std::vector> current_; -public: - type get_result() - { - return current_.back().get(); - } - - void get(reference root, const string_view_type& path, std::error_code& ec) - { - evaluate(root, path, ec); - if (ec) - { - return; - } - if (path.empty()) - { - return; - } - resolve(current_, buffer_, ec); - } - - string_type normalized_path(reference root, const string_view_type& path) - { - std::error_code ec; - evaluate(root, path, ec); - if (ec) - { - return string_type(path); - } - if (current_.back().get().is_array() && buffer_.size() == 1 && buffer_[0] == '-') - { - string_type p = string_type(path.substr(0,path.length()-1)); - std::string s = std::to_string(current_.back().get().size()); - for (auto c : s) - { - p.push_back(c); - } - return p; - } - else - { - return string_type(path); - } - } - - void insert_or_assign(reference root, const string_view_type& path, const J& value, std::error_code& ec) - { - evaluate(root, path, ec); - if (ec) - { - return; - } - if (current_.back().get().is_array()) - { - if (buffer_.size() == 1 && buffer_[0] == '-') - { - current_.back().get().push_back(value); - } - else - { - if (!jsoncons::detail::is_integer(buffer_.data(), buffer_.length())) - { - ec = jsonpointer_errc::invalid_index; - return; - } - auto result = jsoncons::detail::to_integer(buffer_.data(), buffer_.length()); - if (result.ec != jsoncons::detail::to_integer_errc()) - { - ec = jsonpointer_errc::invalid_index; - return; - } - size_t index = result.value; - if (index > current_.back().get().size()) - { - ec = jsonpointer_errc::index_exceeds_array_size; - return; - } - if (index == current_.back().get().size()) - { - current_.back().get().push_back(value); - } - else - { - current_.back().get().insert(current_.back().get().array_range().begin()+index,value); - } - } - } - else if (current_.back().get().is_object()) - { - current_.back().get().insert_or_assign(buffer_,value); - } - else - { - ec = jsonpointer_errc::expected_object_or_array; - return; - } - } - - void insert(reference root, const string_view_type& path, const J& value, std::error_code& ec) - { - evaluate(root, path, ec); - if (ec) - { - return; - } - if (current_.back().get().is_array()) - { - if (buffer_.size() == 1 && buffer_[0] == '-') - { - current_.back().get().push_back(value); - } - else - { - if (!jsoncons::detail::is_integer(buffer_.data(), buffer_.length())) - { - ec = jsonpointer_errc::invalid_index; - return; - } - auto result = jsoncons::detail::to_integer(buffer_.data(), buffer_.length()); - if (result.ec != jsoncons::detail::to_integer_errc()) - { - ec = jsonpointer_errc::invalid_index; - return; - } - size_t index = result.value; - if (index > current_.back().get().size()) - { - ec = jsonpointer_errc::index_exceeds_array_size; - return; - } - if (index == current_.back().get().size()) - { - current_.back().get().push_back(value); - } - else - { - current_.back().get().insert(current_.back().get().array_range().begin()+index,value); - } - } - } - else if (current_.back().get().is_object()) - { - if (current_.back().get().contains(buffer_)) - { - ec = jsonpointer_errc::key_already_exists; - return; - } - else - { - current_.back().get().insert_or_assign(buffer_,value); - } - } - else - { - ec = jsonpointer_errc::expected_object_or_array; - return; - } - } - - void remove(reference root, const string_view_type& path, std::error_code& ec) - { - evaluate(root, path, ec); - if (ec) - { - return; - } - if (current_.back().get().is_array()) - { - if (buffer_.size() == 1 && buffer_[0] == '-') - { - ec = jsonpointer_errc::index_exceeds_array_size; - return; - } - else - { - if (!jsoncons::detail::is_integer(buffer_.data(), buffer_.length())) - { - ec = jsonpointer_errc::invalid_index; - return; - } - auto result = jsoncons::detail::to_integer(buffer_.data(), buffer_.length()); - if (result.ec != jsoncons::detail::to_integer_errc()) - { - ec = jsonpointer_errc::invalid_index; - return; - } - size_t index = result.value; - if (index >= current_.back().get().size()) - { - ec = jsonpointer_errc::index_exceeds_array_size; - return; - } - current_.back().get().erase(current_.back().get().array_range().begin()+index); - } - } - else if (current_.back().get().is_object()) - { - if (!current_.back().get().contains(buffer_)) - { - ec = jsonpointer_errc::name_not_found; - return; - } - else - { - current_.back().get().erase(buffer_); - } - } - else - { - ec = jsonpointer_errc::expected_object_or_array; - return; - } - } - - void replace(reference root, const string_view_type& path, const J& value, std::error_code& ec) - { - evaluate(root, path, ec); - if (ec) - { - return; - } - if (current_.back().get().is_array()) - { - if (buffer_.size() == 1 && buffer_[0] == '-') - { - ec = jsonpointer_errc::index_exceeds_array_size; - return; - } - else - { - if (!jsoncons::detail::is_integer(buffer_.data(), buffer_.length())) - { - ec = jsonpointer_errc::invalid_index; - return; - } - auto result = jsoncons::detail::to_integer(buffer_.data(), buffer_.length()); - if (result.ec != jsoncons::detail::to_integer_errc()) - { - ec = jsonpointer_errc::invalid_index; - return; - } - size_t index = result.value; - if (index >= current_.back().get().size()) - { - ec = jsonpointer_errc::index_exceeds_array_size; - return; - } - (current_.back().get())[index] = value; - } - } - else if (current_.back().get().is_object()) - { - if (!current_.back().get().contains(buffer_)) - { - ec = jsonpointer_errc::key_already_exists; - return; - } - else - { - current_.back().get().insert_or_assign(buffer_,value); - } - } - else - { - ec = jsonpointer_errc::expected_object_or_array; - return; - } - } - - void evaluate(reference root, const string_view_type& path, std::error_code& ec) - { - current_.push_back(root); - - address_iterator it(path.begin(), path.end()); - address_iterator end(path.begin(), path.end(), path.end()); - while (it != end) - { - buffer_ = *it; - it.increment(ec); - if (ec) - return; - if (it == end) - { - return; - } - resolve(current_, buffer_, ec); - if (ec) - return; - } - } - - static void resolve(std::vector>& current, - const string_view_type& buffer, - std::error_code& ec) - { - if (current.back().get().is_array()) - { - if (buffer.size() == 1 && buffer[0] == '-') - { - ec = jsonpointer_errc::index_exceeds_array_size; - return; - } - else - { - if (!jsoncons::detail::is_integer(buffer.data(), buffer.length())) - { - ec = jsonpointer_errc::invalid_index; - return; - } - auto result = jsoncons::detail::to_integer(buffer.data(), buffer.length()); - if (result.ec != jsoncons::detail::to_integer_errc()) - { - ec = jsonpointer_errc::invalid_index; - return; - } - size_t index = result.value; - if (index >= current.back().get().size()) - { - ec = jsonpointer_errc::index_exceeds_array_size; - return; - } - current.push_back(current.back().get().at(index)); - } - } - else if (current.back().get().is_object()) - { - if (!current.back().get().contains(buffer)) - { - ec = jsonpointer_errc::name_not_found; - return; - } - current.push_back(current.back().get().at(buffer)); - } - else - { - ec = jsonpointer_errc::expected_object_or_array; - return; - } - } - - // ser_context - - size_t line() const override - { - return line_; - } - - size_t column() const override - { - return column_; - } -}; - -} - -template -typename J::string_type normalized_path(const J& root, const typename J::string_view_type& path) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - return evaluator.normalized_path(root,path); -} - -template -typename std::enable_if::value,J&>::type -get(J& root, const typename J::string_view_type& path) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - std::error_code ec; - evaluator.get(root, path, ec); - if (ec) - { - JSONCONS_THROW(jsonpointer_error(ec)); - } - return evaluator.get_result(); -} - -template -typename std::enable_if::value,const J&>::type -get(const J& root, const typename J::string_view_type& path) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - std::error_code ec; - evaluator.get(root, path, ec); - if (ec) - { - JSONCONS_THROW(jsonpointer_error(ec)); - } - return evaluator.get_result(); -} - -template -typename std::enable_if::value,J>::type -get(const J& root, const typename J::string_view_type& path) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - std::error_code ec; - evaluator.get(root, path, ec); - if (ec) - { - JSONCONS_THROW(jsonpointer_error(ec)); - } - return evaluator.get_result(); -} - -template -typename std::enable_if::value,J&>::type -get(J& root, const typename J::string_view_type& path, std::error_code& ec) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - evaluator.get(root, path, ec); - return evaluator.get_result(); -} - -template -typename std::enable_if::value,const J&>::type -get(const J& root, const typename J::string_view_type& path, std::error_code& ec) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - evaluator.get(root, path, ec); - return evaluator.get_result(); -} - -template -typename std::enable_if::value,J>::type -get(const J& root, const typename J::string_view_type& path, std::error_code& ec) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - evaluator.get(root, path, ec); - return evaluator.get_result(); -} - -template -bool contains(const J& root, const typename J::string_view_type& path) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - std::error_code ec; - evaluator.get(root, path, ec); - return !ec ? true : false; -} - -template -void insert_or_assign(J& root, const typename J::string_view_type& path, const J& value) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - std::error_code ec; - evaluator.insert_or_assign(root, path, value, ec); - if (ec) - { - JSONCONS_THROW(jsonpointer_error(ec)); - } -} - -template -void insert_or_assign(J& root, const typename J::string_view_type& path, const J& value, std::error_code& ec) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - evaluator.insert_or_assign(root, path, value, ec); -} - -template -void insert(J& root, const typename J::string_view_type& path, const J& value) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - std::error_code ec; - evaluator.insert(root, path, value, ec); - if (ec) - { - JSONCONS_THROW(jsonpointer_error(ec)); - } -} - -template -void insert(J& root, const typename J::string_view_type& path, const J& value, std::error_code& ec) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - evaluator.insert(root, path, value, ec); -} - -template -void remove(J& root, const typename J::string_view_type& path) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - std::error_code ec; - evaluator.remove(root, path, ec); - if (ec) - { - JSONCONS_THROW(jsonpointer_error(ec)); - } -} - -template -void remove(J& root, const typename J::string_view_type& path, std::error_code& ec) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - evaluator.remove(root, path, ec); -} - -template -void replace(J& root, const typename J::string_view_type& path, const J& value) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - std::error_code ec; - evaluator.replace(root, path, value, ec); - if (ec) - { - JSONCONS_THROW(jsonpointer_error(ec)); - } -} - -template -void replace(J& root, const typename J::string_view_type& path, const J& value, std::error_code& ec) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - evaluator.replace(root, path, value, ec); -} - -template -void escape(const String& s, std::basic_ostringstream& os) -{ - for (auto c : s) - { - if (c == '~') - { - os.put('~'); - os.put('0'); - } - else if (c == '/') - { - os.put('~'); - os.put('1'); - } - else - { - os.put(c); - } - } -} - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpointer/jsonpointer.hpp.orig b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpointer/jsonpointer.hpp.orig deleted file mode 100644 index c18dd12aef..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpointer/jsonpointer.hpp.orig +++ /dev/null @@ -1,1060 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSONPOINTER_JSONPOINTER_HPP -#define JSONCONS_JSONPOINTER_JSONPOINTER_HPP - -#include -#include -#include -#include -#include -#include -#include // std::move -#include // std::enable_if, std::true_type -#include -#include -#include - -namespace jsoncons { namespace jsonpointer { - -class jsonpointer_error : public std::exception, public virtual json_exception -{ -public: - jsonpointer_error(const std::error_code& ec) - : error_code_(ec) - { - } - jsonpointer_error(const jsonpointer_error& other) = default; - - jsonpointer_error(jsonpointer_error&& other) = default; - - const char* what() const noexcept override - { - try - { - const_cast(buffer_) = error_code_.message(); - return buffer_.c_str(); - } - catch (...) - { - return ""; - } - } - - const std::error_code code() const - { - return error_code_; - } - - jsonpointer_error& operator=(const jsonpointer_error& e) = default; - jsonpointer_error& operator=(jsonpointer_error&& e) = default; -private: - std::string buffer_; - std::error_code error_code_; -}; - -// find_by_reference - -template -struct is_accessible_by_reference : std::false_type {}; - -template -struct is_accessible_by_reference().at(typename J::string_view_type()))>::value - && std::is_reference().at(0))>::value>::type> -: std::true_type {}; - -namespace detail { - -enum class pointer_state -{ - start, - escaped, - delim -}; - -} // detail - -// address_iterator -template -class address_iterator -{ - typedef typename std::iterator_traits::value_type char_type; - typedef std::basic_string string_type; - typedef InputIt base_iterator; - - base_iterator path_ptr_; - base_iterator end_input_; - base_iterator p_; - base_iterator q_; - jsonpointer::detail::pointer_state state_; - size_t line_; - size_t column_; - std::basic_string buffer_; -public: - typedef string_type value_type; - typedef std::ptrdiff_t difference_type; - typedef value_type* pointer; - typedef const value_type& reference; - typedef std::input_iterator_tag iterator_category; - - address_iterator(base_iterator first, base_iterator last) - : address_iterator(first, last, first) - { - std::error_code ec; - increment(ec); - } - - address_iterator(base_iterator first, base_iterator last, base_iterator current) - : path_ptr_(first), end_input_(last), p_(current), q_(current), state_(jsonpointer::detail::pointer_state::start) - { - } - - address_iterator(const address_iterator&) = default; - - address_iterator(address_iterator&&) = default; - - address_iterator& operator=(const address_iterator&) = default; - - address_iterator& operator=(address_iterator&&) = default; - - address_iterator& operator++() - { - std::error_code ec; - increment(ec); - if (ec) - { - throw jsonpointer_error(ec); - } - return *this; - } - - address_iterator& increment(std::error_code& ec) - { - q_ = p_; - buffer_.clear(); - - bool done = false; - while (p_ != end_input_ && !done) - { - switch (state_) - { - case jsonpointer::detail::pointer_state::start: - switch (*p_) - { - case '/': - state_ = jsonpointer::detail::pointer_state::delim; - break; - default: - ec = jsonpointer_errc::expected_slash; - done = true; - break; - }; - break; - case jsonpointer::detail::pointer_state::delim: - switch (*p_) - { - case '/': - state_ = jsonpointer::detail::pointer_state::delim; - done = true; - break; - case '~': - state_ = jsonpointer::detail::pointer_state::escaped; - break; - default: - buffer_.push_back(*p_); - break; - }; - break; - case jsonpointer::detail::pointer_state::escaped: - switch (*p_) - { - case '0': - buffer_.push_back('~'); - state_ = jsonpointer::detail::pointer_state::delim; - break; - case '1': - buffer_.push_back('/'); - state_ = jsonpointer::detail::pointer_state::delim; - break; - default: - ec = jsonpointer_errc::expected_0_or_1; - done = true; - break; - }; - break; - default: - JSONCONS_UNREACHABLE(); - break; - } - ++p_; - ++column_; - } - return *this; - } - - address_iterator operator++(int) // postfix increment - { - address_iterator temp(*this); - ++(*this); - return temp; - } - - reference operator*() const - { - return buffer_; - } - - friend bool operator==(const address_iterator& it1, const address_iterator& it2) - { - return it1.q_ == it2.q_; - } - friend bool operator!=(const address_iterator& it1, const address_iterator& it2) - { - return !(it1 == it2); - } - -private: -}; - -inline -std::basic_string escape_string(const std::basic_string& s) -{ - std::basic_string result; - for (auto c : s) - { - switch (c) - { - case '~': - result.push_back('~'); - result.push_back('0'); - break; - case '/': - result.push_back('~'); - result.push_back('1'); - break; - default: - result.push_back(c); - break; - } - } - return result; -} - -// address - -<<<<<<< HEAD -template -class basic_address -======= -class address ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0 -{ -public: - std::basic_string path_; -public: - // Member types - typedef char char_type; - typedef std::basic_string string_type; - typedef basic_string_view string_view_type; - typedef address_iterator const_iterator; - typedef const_iterator iterator; - - // Constructors -<<<<<<< HEAD - basic_address() - { - } - explicit basic_address(const string_type& s) - : path_(s) - { - } - explicit basic_address(string_type&& s) - : path_(std::move(s)) - { - } - explicit basic_address(const CharT* s) -======= - address() - { - } - explicit address(const string_type& s) - : path_(s) - { - } - explicit address(string_type&& s) - : path_(std::move(s)) - { - } - explicit address(const char_type* s) ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0 - : path_(s) - { - } - -<<<<<<< HEAD - basic_address(const basic_address&) = default; - - basic_address(basic_address&&) = default; - - // operator= - basic_address& operator=(const basic_address&) = default; - - basic_address& operator=(basic_address&&) = default; -======= - address(const address&) = default; - - address(address&&) = default; - - // operator= - address& operator=(const address&) = default; - - address& operator=(address&&) = default; ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0 - - // Modifiers - - void clear() - { - path_.clear(); - } - -<<<<<<< HEAD - basic_address& operator/=(const string_type& s) -======= - address& operator/=(const string_type& s) ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0 - { - path_.push_back('/'); - path_.append(escape_string(s)); - - return *this; - } - -<<<<<<< HEAD - basic_address& operator+=(const basic_address& p) -======= - address& operator+=(const address& p) ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0 - { - path_.append(p.path_); - return *this; - } - - // Accessors - bool empty() const - { - return path_.empty(); - } - - const string_type& string() const - { - return path_; - } - - operator string_view_type() const - { - return path_; - } - - // Iterators - iterator begin() const - { - return iterator(path_.begin(),path_.end()); - } - iterator end() const - { - return iterator(path_.begin(), path_.end(), path_.end()); - } - - // Non-member functions -<<<<<<< HEAD - friend basic_address operator/(const basic_address& lhs, const string_type& rhs) - { - basic_address p(lhs); -======= - friend address operator/(const address& lhs, const string_type& rhs) - { - address p(lhs); ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0 - p /= rhs; - return p; - } - -<<<<<<< HEAD - friend basic_address operator+( const basic_address& lhs, const basic_address& rhs ) - { - basic_address p(lhs); -======= - friend address operator+( const address& lhs, const address& rhs ) - { - address p(lhs); ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0 - p += rhs; - return p; - } - -<<<<<<< HEAD - friend bool operator==( const basic_address& lhs, const basic_address& rhs ) -======= - friend bool operator==( const address& lhs, const address& rhs ) ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0 - { - return lhs.path_ == rhs.path_; - } - -<<<<<<< HEAD - friend bool operator!=( const basic_address& lhs, const basic_address& rhs ) -======= - friend bool operator!=( const address& lhs, const address& rhs ) ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0 - { - return lhs.path_ != rhs.path_; - } - -<<<<<<< HEAD - friend std::basic_ostream& - operator<<( std::basic_ostream& os, const basic_address& p ) -======= - friend std::ostream& - operator<<( std::ostream& os, const address& p ) ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0 - { - os << p.path_; - return os; - } -}; - -<<<<<<< HEAD -typedef basic_address address; - -======= ->>>>>>> efc18942efb17086d3de2d1d853d56a11666ebb0 -namespace detail { - -template -class handle_type -{ -public: - using value_type = typename J::value_type; - using type = value_type; - - handle_type(const value_type& val) noexcept - : val_(val) - { - } - - handle_type(value_type&& val) noexcept - : val_(std::move(val)) - { - } - - handle_type(const handle_type& w) noexcept - : val_(w.val_) - { - } - - handle_type& operator=(const handle_type&) noexcept = default; - - type get() const noexcept - { - return val_; - } -private: - value_type val_; -}; - -template -class handle_type::value>::type> -{ -public: - using reference = JReference; - using type = reference; - using pointer = typename std::conditional::type>::value,typename J::const_pointer,typename J::pointer>::type; - - handle_type(reference ref) noexcept - : ptr_(std::addressof(ref)) - { - } - - handle_type(const handle_type&) noexcept = default; - - handle_type& operator=(const handle_type&) noexcept = default; - - type get() const noexcept - { - return *ptr_; - } -private: - pointer ptr_; -}; - -template -class jsonpointer_evaluator : private serializing_context -{ - typedef typename handle_type::type type; - typedef typename J::string_type string_type; - typedef typename string_type::value_type char_type; - typedef typename J::string_view_type string_view_type; - using reference = JReference; - using pointer = typename std::conditional::type>::value,typename J::const_pointer,typename J::pointer>::type; - - size_t line_; - size_t column_; - string_type buffer_; - std::vector> current_; -public: - type get_result() - { - return current_.back().get(); - } - - void get(reference root, const string_view_type& path, std::error_code& ec) - { - evaluate(root, path, ec); - if (ec) - { - return; - } - if (path.empty()) - { - return; - } - resolve(current_, buffer_, ec); - } - - string_type normalized_path(reference root, const string_view_type& path) - { - std::error_code ec; - evaluate(root, path, ec); - if (ec) - { - return string_type(path); - } - if (current_.back().get().is_array() && buffer_.size() == 1 && buffer_[0] == '-') - { - string_type p = string_type(path.substr(0,path.length()-1)); - std::string s = std::to_string(current_.back().get().size()); - for (auto c : s) - { - p.push_back(c); - } - return p; - } - else - { - return string_type(path); - } - } - - void insert_or_assign(reference root, const string_view_type& path, const J& value, std::error_code& ec) - { - evaluate(root, path, ec); - if (ec) - { - return; - } - if (current_.back().get().is_array()) - { - if (buffer_.size() == 1 && buffer_[0] == '-') - { - current_.back().get().push_back(value); - } - else - { - if (!jsoncons::detail::is_integer(buffer_.data(), buffer_.length())) - { - ec = jsonpointer_errc::invalid_index; - return; - } - auto result = jsoncons::detail::to_integer(buffer_.data(), buffer_.length()); - if (result.overflow) - { - ec = jsonpointer_errc::invalid_index; - return; - } - size_t index = result.value; - if (index > current_.back().get().size()) - { - ec = jsonpointer_errc::index_exceeds_array_size; - return; - } - if (index == current_.back().get().size()) - { - current_.back().get().push_back(value); - } - else - { - current_.back().get().insert(current_.back().get().array_range().begin()+index,value); - } - } - } - else if (current_.back().get().is_object()) - { - current_.back().get().insert_or_assign(buffer_,value); - } - else - { - ec = jsonpointer_errc::expected_object_or_array; - return; - } - } - - void insert(reference root, const string_view_type& path, const J& value, std::error_code& ec) - { - evaluate(root, path, ec); - if (ec) - { - return; - } - if (current_.back().get().is_array()) - { - if (buffer_.size() == 1 && buffer_[0] == '-') - { - current_.back().get().push_back(value); - } - else - { - if (!jsoncons::detail::is_integer(buffer_.data(), buffer_.length())) - { - ec = jsonpointer_errc::invalid_index; - return; - } - auto result = jsoncons::detail::to_integer(buffer_.data(), buffer_.length()); - if (result.overflow) - { - ec = jsonpointer_errc::invalid_index; - return; - } - size_t index = result.value; - if (index > current_.back().get().size()) - { - ec = jsonpointer_errc::index_exceeds_array_size; - return; - } - if (index == current_.back().get().size()) - { - current_.back().get().push_back(value); - } - else - { - current_.back().get().insert(current_.back().get().array_range().begin()+index,value); - } - } - } - else if (current_.back().get().is_object()) - { - if (current_.back().get().contains(buffer_)) - { - ec = jsonpointer_errc::key_already_exists; - return; - } - else - { - current_.back().get().insert_or_assign(buffer_,value); - } - } - else - { - ec = jsonpointer_errc::expected_object_or_array; - return; - } - } - - void remove(reference root, const string_view_type& path, std::error_code& ec) - { - evaluate(root, path, ec); - if (ec) - { - return; - } - if (current_.back().get().is_array()) - { - if (buffer_.size() == 1 && buffer_[0] == '-') - { - ec = jsonpointer_errc::index_exceeds_array_size; - return; - } - else - { - if (!jsoncons::detail::is_integer(buffer_.data(), buffer_.length())) - { - ec = jsonpointer_errc::invalid_index; - return; - } - auto result = jsoncons::detail::to_integer(buffer_.data(), buffer_.length()); - if (result.overflow) - { - ec = jsonpointer_errc::invalid_index; - return; - } - size_t index = result.value; - if (index >= current_.back().get().size()) - { - ec = jsonpointer_errc::index_exceeds_array_size; - return; - } - current_.back().get().erase(current_.back().get().array_range().begin()+index); - } - } - else if (current_.back().get().is_object()) - { - if (!current_.back().get().contains(buffer_)) - { - ec = jsonpointer_errc::name_not_found; - return; - } - else - { - current_.back().get().erase(buffer_); - } - } - else - { - ec = jsonpointer_errc::expected_object_or_array; - return; - } - } - - void replace(reference root, const string_view_type& path, const J& value, std::error_code& ec) - { - evaluate(root, path, ec); - if (ec) - { - return; - } - if (current_.back().get().is_array()) - { - if (buffer_.size() == 1 && buffer_[0] == '-') - { - ec = jsonpointer_errc::index_exceeds_array_size; - return; - } - else - { - if (!jsoncons::detail::is_integer(buffer_.data(), buffer_.length())) - { - ec = jsonpointer_errc::invalid_index; - return; - } - auto result = jsoncons::detail::to_integer(buffer_.data(), buffer_.length()); - if (result.overflow) - { - ec = jsonpointer_errc::invalid_index; - return; - } - size_t index = result.value; - if (index >= current_.back().get().size()) - { - ec = jsonpointer_errc::index_exceeds_array_size; - return; - } - (current_.back().get())[index] = value; - } - } - else if (current_.back().get().is_object()) - { - if (!current_.back().get().contains(buffer_)) - { - ec = jsonpointer_errc::key_already_exists; - return; - } - else - { - current_.back().get().insert_or_assign(buffer_,value); - } - } - else - { - ec = jsonpointer_errc::expected_object_or_array; - return; - } - } - - void evaluate(reference root, const string_view_type& path, std::error_code& ec) - { - current_.push_back(root); - - address_iterator it(path.begin(), path.end()); - address_iterator end(path.begin(), path.end(), path.end()); - while (it != end) - { - buffer_ = *it; - it.increment(ec); - if (ec) - return; - if (it == end) - { - return; - } - resolve(current_, buffer_, ec); - if (ec) - return; - } - } - - static void resolve(std::vector>& current, - const string_view_type& buffer, - std::error_code& ec) - { - if (current.back().get().is_array()) - { - if (buffer.size() == 1 && buffer[0] == '-') - { - ec = jsonpointer_errc::index_exceeds_array_size; - return; - } - else - { - if (!jsoncons::detail::is_integer(buffer.data(), buffer.length())) - { - ec = jsonpointer_errc::invalid_index; - return; - } - auto result = jsoncons::detail::to_integer(buffer.data(), buffer.length()); - if (result.overflow) - { - ec = jsonpointer_errc::invalid_index; - return; - } - size_t index = result.value; - if (index >= current.back().get().size()) - { - ec = jsonpointer_errc::index_exceeds_array_size; - return; - } - current.push_back(current.back().get().at(index)); - } - } - else if (current.back().get().is_object()) - { - if (!current.back().get().contains(buffer)) - { - ec = jsonpointer_errc::name_not_found; - return; - } - current.push_back(current.back().get().at(buffer)); - } - else - { - ec = jsonpointer_errc::expected_object_or_array; - return; - } - } - - // serializing_context - - size_t line_number() const override - { - return line_; - } - - size_t column_number() const override - { - return column_; - } -}; - -} - -template -typename J::string_type normalized_path(const J& root, const typename J::string_view_type& path) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - return evaluator.normalized_path(root,path); -} - -template -typename std::enable_if::value,J&>::type -get(J& root, const typename J::string_view_type& path) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - std::error_code ec; - evaluator.get(root, path, ec); - if (ec) - { - JSONCONS_THROW(jsonpointer_error(ec)); - } - return evaluator.get_result(); -} - -template -typename std::enable_if::value,const J&>::type -get(const J& root, const typename J::string_view_type& path) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - std::error_code ec; - evaluator.get(root, path, ec); - if (ec) - { - JSONCONS_THROW(jsonpointer_error(ec)); - } - return evaluator.get_result(); -} - -template -typename std::enable_if::value,J>::type -get(const J& root, const typename J::string_view_type& path) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - std::error_code ec; - evaluator.get(root, path, ec); - if (ec) - { - JSONCONS_THROW(jsonpointer_error(ec)); - } - return evaluator.get_result(); -} - -template -typename std::enable_if::value,J&>::type -get(J& root, const typename J::string_view_type& path, std::error_code& ec) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - evaluator.get(root, path, ec); - return evaluator.get_result(); -} - -template -typename std::enable_if::value,const J&>::type -get(const J& root, const typename J::string_view_type& path, std::error_code& ec) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - evaluator.get(root, path, ec); - return evaluator.get_result(); -} - -template -typename std::enable_if::value,J>::type -get(const J& root, const typename J::string_view_type& path, std::error_code& ec) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - evaluator.get(root, path, ec); - return evaluator.get_result(); -} - -template -bool contains(const J& root, const typename J::string_view_type& path) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - std::error_code ec; - evaluator.get(root, path, ec); - return !ec ? true : false; -} - -template -void insert_or_assign(J& root, const typename J::string_view_type& path, const J& value) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - std::error_code ec; - evaluator.insert_or_assign(root, path, value, ec); - if (ec) - { - JSONCONS_THROW(jsonpointer_error(ec)); - } -} - -template -void insert_or_assign(J& root, const typename J::string_view_type& path, const J& value, std::error_code& ec) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - evaluator.insert_or_assign(root, path, value, ec); -} - -template -void insert(J& root, const typename J::string_view_type& path, const J& value) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - std::error_code ec; - evaluator.insert(root, path, value, ec); - if (ec) - { - JSONCONS_THROW(jsonpointer_error(ec)); - } -} - -template -void insert(J& root, const typename J::string_view_type& path, const J& value, std::error_code& ec) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - evaluator.insert(root, path, value, ec); -} - -template -void remove(J& root, const typename J::string_view_type& path) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - std::error_code ec; - evaluator.remove(root, path, ec); - if (ec) - { - JSONCONS_THROW(jsonpointer_error(ec)); - } -} - -template -void remove(J& root, const typename J::string_view_type& path, std::error_code& ec) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - evaluator.remove(root, path, ec); -} - -template -void replace(J& root, const typename J::string_view_type& path, const J& value) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - std::error_code ec; - evaluator.replace(root, path, value, ec); - if (ec) - { - JSONCONS_THROW(jsonpointer_error(ec)); - } -} - -template -void replace(J& root, const typename J::string_view_type& path, const J& value, std::error_code& ec) -{ - jsoncons::jsonpointer::detail::jsonpointer_evaluator evaluator; - - evaluator.replace(root, path, value, ec); -} - -template -void escape(const String& s, std::basic_ostringstream& os) -{ - for (auto c : s) - { - if (c == '~') - { - os.put('~'); - os.put('0'); - } - else if (c == '/') - { - os.put('~'); - os.put('1'); - } - else - { - os.put(c); - } - } -} - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpointer/jsonpointer_error.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpointer/jsonpointer_error.hpp deleted file mode 100644 index dfd26184c7..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/jsonpointer/jsonpointer_error.hpp +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_JSONPOINTER_JSONPOINTER_ERROR_HPP -#define JSONCONS_JSONPOINTER_JSONPOINTER_ERROR_HPP - -#include -#include - -namespace jsoncons { namespace jsonpointer { - -class jsonpointer_error : public std::system_error, public virtual json_exception -{ -public: - jsonpointer_error(const std::error_code& ec) - : std::system_error(ec) - { - } - jsonpointer_error(const std::error_code& ec, const std::string& what_arg) - : std::system_error(ec, what_arg) - { - } - jsonpointer_error(const std::error_code& ec, const char* what_arg) - : std::system_error(ec, what_arg) - { - } - jsonpointer_error(const jsonpointer_error& other) = default; - - jsonpointer_error(jsonpointer_error&& other) = default; - - const char* what() const noexcept override - { - return std::system_error::what(); - } -}; - -enum class jsonpointer_errc -{ - ok = 0, - expected_slash = 1, - index_exceeds_array_size, - expected_0_or_1, - invalid_index, - name_not_found, - key_already_exists, - expected_object_or_array, - end_of_input -}; - -class jsonpointer_error_category_impl - : public std::error_category -{ -public: - const char* name() const noexcept override - { - return "jsoncons/jsonpointer"; - } - std::string message(int ev) const override - { - switch (static_cast(ev)) - { - case jsonpointer_errc::expected_slash: - return "Expected /"; - case jsonpointer_errc::index_exceeds_array_size: - return "Index exceeds array size"; - case jsonpointer_errc::expected_0_or_1: - return "Expected '0' or '1' after escape character '~'"; - case jsonpointer_errc::name_not_found: - return "Name not found"; - case jsonpointer_errc::invalid_index: - return "Invalid array index"; - case jsonpointer_errc::key_already_exists: - return "Key already exists"; - case jsonpointer_errc::expected_object_or_array: - return "Expected object or array"; - case jsonpointer_errc::end_of_input: - return "Unexpected end of input"; - default: - return "Unknown jsonpointer error"; - } - } -}; - -inline -const std::error_category& jsonpointer_error_category() -{ - static jsonpointer_error_category_impl instance; - return instance; -} - -inline -std::error_code make_error_code(jsonpointer_errc result) -{ - return std::error_code(static_cast(result),jsonpointer_error_category()); -} - -}} - -namespace std { - template<> - struct is_error_code_enum : public true_type - { - }; -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack.hpp deleted file mode 100644 index d5c8360edb..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack.hpp +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_MSGPACK_MSGPACK_HPP -#define JSONCONS_MSGPACK_MSGPACK_HPP - -#include -#include -#include -#include // std::enable_if -#include // std::basic_istream -#include -#include -#include -#include - -namespace jsoncons { namespace msgpack { - -// encode_msgpack - -template -typename std::enable_if::value,void>::type -encode_msgpack(const T& j, std::vector& v) -{ - typedef typename T::char_type char_type; - msgpack_bytes_encoder encoder(v); - auto adaptor = make_json_content_handler_adaptor>(encoder); - j.dump(adaptor); -} - -template -typename std::enable_if::value,void>::type -encode_msgpack(const T& val, std::vector& v) -{ - msgpack_bytes_encoder encoder(v); - write_to(json(), val, encoder); -} - -template -typename std::enable_if::value,void>::type -encode_msgpack(const T& j, std::ostream& os) -{ - typedef typename T::char_type char_type; - msgpack_encoder encoder(os); - auto adaptor = make_json_content_handler_adaptor>(encoder); - j.dump(adaptor); -} - -template -typename std::enable_if::value,void>::type -encode_msgpack(const T& val, std::ostream& os) -{ - msgpack_encoder encoder(os); - write_to(json(), val, encoder); -} - -// decode_msgpack - -template -typename std::enable_if::value,T>::type -decode_msgpack(const std::vector& v) -{ - jsoncons::json_decoder decoder; - auto adaptor = make_json_content_handler_adaptor(decoder); - basic_msgpack_reader reader(v, adaptor); - std::error_code ec; - reader.read(ec); - if (ec) - { - throw ser_error(ec,reader.line(),reader.column()); - } - return decoder.get_result(); -} - -template -typename std::enable_if::value,T>::type -decode_msgpack(const std::vector& v) -{ - jsoncons::json_decoder decoder; - basic_msgpack_reader reader(v, decoder); - reader.read(); - return decoder.get_result().template as(); -} - -template -typename std::enable_if::value,T>::type -decode_msgpack(std::istream& is) -{ - jsoncons::json_decoder decoder; - auto adaptor = make_json_content_handler_adaptor(decoder); - msgpack_reader reader(is, adaptor); - std::error_code ec; - reader.read(ec); - if (ec) - { - throw ser_error(ec,reader.line(),reader.column()); - } - return decoder.get_result(); -} - -template -typename std::enable_if::value,T>::type -decode_msgpack(std::istream& is) -{ - jsoncons::json_decoder decoder; - msgpack_reader reader(is, decoder); - reader.read(); - return decoder.get_result(); -} - -#if !defined(JSONCONS_NO_DEPRECATED) -template -std::vector encode_msgpack(const Json& j) -{ - std::vector v; - encode_msgpack(j, v); - return v; -} -#endif - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack_detail.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack_detail.hpp deleted file mode 100644 index 75f8ce0702..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack_detail.hpp +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_MSGPACK_MSGPACK_DETAIL_HPP -#define JSONCONS_MSGPACK_MSGPACK_DETAIL_HPP - -#include -#include -#include - -namespace jsoncons { namespace msgpack { namespace detail { - -namespace msgpack_format -{ - const uint8_t positive_fixint_base_cd = 0x00; - const uint8_t nil_cd = 0xc0; - const uint8_t false_cd = 0xc2; - const uint8_t true_cd = 0xc3; - const uint8_t float32_cd = 0xca; - const uint8_t float64_cd = 0xcb; - const uint8_t uint8_cd = 0xcc; - const uint8_t uint16_cd = 0xcd; - const uint8_t uint32_cd = 0xce; - const uint8_t uint64_cd = 0xcf; - const uint8_t int8_cd = 0xd0; - const uint8_t int16_cd = 0xd1; - const uint8_t int32_cd = 0xd2; - const uint8_t int64_cd = 0xd3; - const uint8_t fixmap_base_cd = 0x80; - const uint8_t fixarray_base_cd = 0x90; - const uint8_t fixstr_base_cd = 0xa0; - const uint8_t str8_cd = 0xd9; - const uint8_t str16_cd = 0xda; - const uint8_t str32_cd = 0xdb; - const uint8_t bin8_cd = 0xc7; - const uint8_t bin16_cd = 0xc8; - const uint8_t bin32_cd = 0xc9; - const uint8_t array16_cd = 0xdc; - const uint8_t array32_cd = 0xdd; - const uint8_t map16_cd = 0xde; - const uint8_t map32_cd = 0xdf; - const uint8_t negative_fixint_base_cd = 0xe0; -} - -}}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack_encoder.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack_encoder.hpp deleted file mode 100644 index 6240156692..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack_encoder.hpp +++ /dev/null @@ -1,459 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_MSGPACK_MSGPACK_ENCODER_HPP -#define JSONCONS_MSGPACK_MSGPACK_ENCODER_HPP - -#include -#include -#include // std::numeric_limits -#include -#include // std::move -#include -#include -#include -#include -#include -#include -#include - -namespace jsoncons { namespace msgpack { - -enum class msgpack_container_type {object, indefinite_length_object, array, indefinite_length_array}; - -template -class basic_msgpack_encoder final : public basic_json_content_handler -{ - enum class decimal_parse_state { start, integer, exp1, exp2, fraction1 }; -public: - typedef char char_type; - using typename basic_json_content_handler::string_view_type; - typedef Result result_type; - -private: - struct stack_item - { - msgpack_container_type type_; - size_t length_; - size_t count_; - - stack_item(msgpack_container_type type, size_t length = 0) - : type_(type), length_(length), count_(0) - { - } - - size_t length() const - { - return length_; - } - - size_t count() const - { - return count_; - } - - bool is_object() const - { - return type_ == msgpack_container_type::object || type_ == msgpack_container_type::indefinite_length_object; - } - - bool is_indefinite_length() const - { - return type_ == msgpack_container_type::indefinite_length_array || type_ == msgpack_container_type::indefinite_length_object; - } - - }; - std::vector stack_; - Result result_; - - // Noncopyable and nonmoveable - basic_msgpack_encoder(const basic_msgpack_encoder&) = delete; - basic_msgpack_encoder& operator=(const basic_msgpack_encoder&) = delete; -public: - explicit basic_msgpack_encoder(result_type result) - : result_(std::move(result)) - { - } - - ~basic_msgpack_encoder() - { - try - { - result_.flush(); - } - catch (...) - { - } - } - -private: - // Implementing methods - - void do_flush() override - { - result_.flush(); - } - - bool do_begin_object(semantic_tag, const ser_context&) override - { - throw ser_error(msgpack_errc::object_length_required); - } - - bool do_begin_object(size_t length, semantic_tag, const ser_context&) override - { - stack_.push_back(stack_item(msgpack_container_type::object, length)); - - if (length <= 15) - { - // fixmap - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::fixmap_base_cd | (length & 0xf)), - std::back_inserter(result_)); - } - else if (length <= 65535) - { - // map 16 - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::map16_cd), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), - std::back_inserter(result_)); - } - else if (length <= 4294967295) - { - // map 32 - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::map32_cd), - std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), - std::back_inserter(result_)); - } - - return true; - } - - bool do_end_object(const ser_context&) override - { - JSONCONS_ASSERT(!stack_.empty()); - - if (stack_.back().count() < stack_.back().length()) - { - throw ser_error( msgpack_errc::too_few_items); - } - else if (stack_.back().count() > stack_.back().length()) - { - throw ser_error( msgpack_errc::too_many_items); - } - - stack_.pop_back(); - end_value(); - return true; - } - - bool do_begin_array(semantic_tag, const ser_context&) override - { - throw ser_error(msgpack_errc::array_length_required); - } - - bool do_begin_array(size_t length, semantic_tag, const ser_context&) override - { - stack_.push_back(stack_item(msgpack_container_type::array, length)); - if (length <= 15) - { - // fixarray - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::fixarray_base_cd | (length & 0xf)), std::back_inserter(result_)); - } - else if (length <= (std::numeric_limits::max)()) - { - // array 16 - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::array16_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length),std::back_inserter(result_)); - } - else if (length <= (std::numeric_limits::max)()) - { - // array 32 - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::array32_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length),std::back_inserter(result_)); - } - return true; - } - - bool do_end_array(const ser_context&) override - { - JSONCONS_ASSERT(!stack_.empty()); - - if (stack_.back().count() < stack_.back().length()) - { - throw ser_error(msgpack_errc::too_few_items); - } - else if (stack_.back().count() > stack_.back().length()) - { - throw ser_error(msgpack_errc::too_many_items); - } - - stack_.pop_back(); - end_value(); - return true; - } - - bool do_name(const string_view_type& name, const ser_context&) override - { - write_string_value(name); - return true; - } - - bool do_null_value(semantic_tag, const ser_context&) override - { - // nil - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::nil_cd), std::back_inserter(result_)); - end_value(); - return true; - } - - bool do_string_value(const string_view_type& sv, semantic_tag, const ser_context&) override - { - write_string_value(sv); - end_value(); - return true; - } - - void write_string_value(const string_view_type& sv) - { - auto result = unicons::validate(sv.begin(), sv.end()); - if (result.ec != unicons::conv_errc()) - { - throw ser_error(msgpack_errc::invalid_utf8_text_string); - } - - const size_t length = sv.length(); - if (length <= 31) - { - // fixstr stores a byte array whose length is upto 31 bytes - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::fixstr_base_cd | length), std::back_inserter(result_)); - } - else if (length <= (std::numeric_limits::max)()) - { - // str 8 stores a byte array whose length is upto (2^8)-1 bytes - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::str8_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), std::back_inserter(result_)); - } - else if (length <= (std::numeric_limits::max)()) - { - // str 16 stores a byte array whose length is upto (2^16)-1 bytes - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::str16_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), std::back_inserter(result_)); - } - else if (length <= (std::numeric_limits::max)()) - { - // str 32 stores a byte array whose length is upto (2^32)-1 bytes - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::str32_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length),std::back_inserter(result_)); - } - - for (auto c : sv) - { - result_.push_back(c); - } - } - - bool do_byte_string_value(const byte_string_view& b, - semantic_tag, - const ser_context&) override - { - - const size_t length = b.length(); - if (length <= (std::numeric_limits::max)()) - { - // str 8 stores a byte array whose length is upto (2^8)-1 bytes - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::bin8_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), std::back_inserter(result_)); - } - else if (length <= (std::numeric_limits::max)()) - { - // str 16 stores a byte array whose length is upto (2^16)-1 bytes - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::bin16_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length), std::back_inserter(result_)); - } - else if (length <= (std::numeric_limits::max)()) - { - // str 32 stores a byte array whose length is upto (2^32)-1 bytes - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::bin32_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(length),std::back_inserter(result_)); - } - - for (auto c : b) - { - result_.push_back(c); - } - - end_value(); - return true; - } - - bool do_double_value(double val, - semantic_tag, - const ser_context&) override - { - float valf = (float)val; - if ((double)valf == val) - { - // float 32 - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::float32_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(valf,std::back_inserter(result_)); - } - else - { - // float 64 - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::float64_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(val,std::back_inserter(result_)); - } - - // write double - - end_value(); - return true; - } - - bool do_int64_value(int64_t val, - semantic_tag, - const ser_context&) override - { - if (val >= 0) - { - if (val <= 0x7f) - { - // positive fixnum stores 7-bit positive integer - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - else if (val <= (std::numeric_limits::max)()) - { - // uint 8 stores a 8-bit unsigned integer - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::uint8_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - else if (val <= (std::numeric_limits::max)()) - { - // uint 16 stores a 16-bit big-endian unsigned integer - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::uint16_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - else if (val <= (std::numeric_limits::max)()) - { - // uint 32 stores a 32-bit big-endian unsigned integer - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::uint32_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - else if (val <= (std::numeric_limits::max)()) - { - // int 64 stores a 64-bit big-endian signed integer - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::uint64_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - } - else - { - if (val >= -32) - { - // negative fixnum stores 5-bit negative integer - jsoncons::detail::to_big_endian(static_cast(val), std::back_inserter(result_)); - } - else if (val >= (std::numeric_limits::lowest)()) - { - // int 8 stores a 8-bit signed integer - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::int8_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - else if (val >= (std::numeric_limits::lowest)()) - { - // int 16 stores a 16-bit big-endian signed integer - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::int16_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - else if (val >= (std::numeric_limits::lowest)()) - { - // int 32 stores a 32-bit big-endian signed integer - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::int32_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - else if (val >= (std::numeric_limits::lowest)()) - { - // int 64 stores a 64-bit big-endian signed integer - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::int64_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - } - end_value(); - return true; - } - - bool do_uint64_value(uint64_t val, - semantic_tag, - const ser_context&) override - { - if (val <= (std::numeric_limits::max)()) - { - // positive fixnum stores 7-bit positive integer - jsoncons::detail::to_big_endian(static_cast(val), std::back_inserter(result_)); - } - else if (val <= (std::numeric_limits::max)()) - { - // uint 8 stores a 8-bit unsigned integer - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::uint8_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(val), std::back_inserter(result_)); - } - else if (val <= (std::numeric_limits::max)()) - { - // uint 16 stores a 16-bit big-endian unsigned integer - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::uint16_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - else if (val <= (std::numeric_limits::max)()) - { - // uint 32 stores a 32-bit big-endian unsigned integer - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::uint32_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - else if (val <= (std::numeric_limits::max)()) - { - // uint 64 stores a 64-bit big-endian unsigned integer - jsoncons::detail::to_big_endian(static_cast(jsoncons::msgpack::detail::msgpack_format ::uint64_cd), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - end_value(); - return true; - } - - bool do_bool_value(bool val, semantic_tag, const ser_context&) override - { - // true and false - result_.push_back(static_cast(val ? jsoncons::msgpack::detail::msgpack_format ::true_cd : jsoncons::msgpack::detail::msgpack_format ::false_cd)); - //jsoncons::detail::to_big_endian(static_cast(val ? jsoncons::msgpack::detail::msgpack_format ::true_cd : jsoncons::msgpack::detail::msgpack_format ::false_cd), std::back_inserter(result_)); - - end_value(); - return true; - } - - void end_value() - { - if (!stack_.empty()) - { - ++stack_.back().count_; - } - } -}; - -typedef basic_msgpack_encoder msgpack_encoder; -typedef basic_msgpack_encoder msgpack_bytes_encoder; - -#if !defined(JSONCONS_NO_DEPRECATED) -typedef basic_msgpack_encoder msgpack_bytes_serializer; - -template -using basic_msgpack_serializer = basic_msgpack_encoder; - -typedef basic_msgpack_serializer msgpack_serializer; -typedef basic_msgpack_serializer msgpack_buffer_serializer; -#endif - -}} -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack_error.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack_error.hpp deleted file mode 100644 index 3b2d1d7b05..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack_error.hpp +++ /dev/null @@ -1,82 +0,0 @@ -/// Copyright 2018 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_MSGPACK_MSGPACK_ERROR_HPP -#define JSONCONS_MSGPACK_MSGPACK_ERROR_HPP - -#include -#include - -namespace jsoncons { namespace msgpack { - -enum class msgpack_errc -{ - ok = 0, - unexpected_eof = 1, - source_error, - invalid_utf8_text_string, - array_length_required, - object_length_required, - too_many_items, - too_few_items -}; - -class msgpack_error_category_impl - : public std::error_category -{ -public: - const char* name() const noexcept override - { - return "jsoncons/msgpack"; - } - std::string message(int ev) const override - { - switch (static_cast(ev)) - { - case msgpack_errc::unexpected_eof: - return "Unexpected end of file"; - case msgpack_errc::source_error: - return "Source error"; - case msgpack_errc::invalid_utf8_text_string: - return "Illegal UTF-8 encoding in text string"; - case msgpack_errc::array_length_required: - return "MessagePack encoder requires array length"; - case msgpack_errc::object_length_required: - return "MessagePack encoder requires object length"; - case msgpack_errc::too_many_items: - return "Too many items were added to a MessagePack object or array"; - case msgpack_errc::too_few_items: - return "Too few items were added to a MessagePack object or array"; - default: - return "Unknown MessagePack parser error"; - } - } -}; - -inline -const std::error_category& msgpack_error_category() -{ - static msgpack_error_category_impl instance; - return instance; -} - -inline -std::error_code make_error_code(msgpack_errc e) -{ - return std::error_code(static_cast(e),msgpack_error_category()); -} - - -}} - -namespace std { - template<> - struct is_error_code_enum : public true_type - { - }; -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack_reader.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack_reader.hpp deleted file mode 100644 index 8970675855..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/msgpack/msgpack_reader.hpp +++ /dev/null @@ -1,757 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_MSGPACK_MSGPACK_READER_HPP -#define JSONCONS_MSGPACK_MSGPACK_READER_HPP - -#include -#include -#include -#include // std::move -#include -#include -#include -#include -#include -#include - -namespace jsoncons { namespace msgpack { - -template -class basic_msgpack_reader : public ser_context -{ - Src source_; - json_content_handler& handler_; - size_t nesting_depth_; - std::string buffer_; -public: - template - basic_msgpack_reader(Source&& source, json_content_handler& handler) - : source_(std::forward(source)), - handler_(handler), - nesting_depth_(0) - { - } - - void read() - { - std::error_code ec; - read(ec); - if (ec) - { - throw ser_error(ec,line(),column()); - } - } - - void read(std::error_code& ec) - { - try - { - read_internal(ec); - } - catch (const ser_error& e) - { - ec = e.code(); - } - } - - size_t line() const override - { - return 0; - } - - size_t column() const override - { - return source_.position(); - } -private: - - void read_internal(std::error_code& ec) - { - if (source_.is_error()) - { - ec = msgpack_errc::source_error; - return; - } - //const uint8_t* pos = input_ptr_++; - - uint8_t type{}; - source_.get(type); - - if (type <= 0xbf) - { - if (type <= 0x7f) - { - // positive fixint - handler_.uint64_value(type, semantic_tag::none, *this); - } - else if (type <= 0x8f) - { - // fixmap - const size_t len = type & 0x0f; - handler_.begin_object(len, semantic_tag::none, *this); - ++nesting_depth_; - for (size_t i = 0; i < len; ++i) - { - parse_name(ec); - if (ec) - { - return; - } - read(ec); - if (ec) - { - return; - } - } - handler_.end_object(*this); - --nesting_depth_; - } - else if (type <= 0x9f) - { - // fixarray - const size_t len = type & 0x0f; - handler_.begin_array(len, semantic_tag::none, *this); - ++nesting_depth_; - for (size_t i = 0; i < len; ++i) - { - read(ec); - if (ec) - { - return; - } - } - handler_.end_array(*this); - --nesting_depth_; - } - else - { - // fixstr - const size_t len = type & 0x1f; - - std::basic_string s; - source_.read(std::back_inserter(s), len); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - - auto result = unicons::validate(s.begin(),s.end()); - if (result.ec != unicons::conv_errc()) - { - ec = msgpack_errc::invalid_utf8_text_string; - return; - } - handler_.string_value(basic_string_view(s.data(),s.length()), semantic_tag::none, *this); - } - } - else if (type >= 0xe0) - { - // negative fixint - handler_.int64_value(static_cast(type), semantic_tag::none, *this); - } - else - { - switch (type) - { - case jsoncons::msgpack::detail::msgpack_format ::nil_cd: - { - handler_.null_value(semantic_tag::none, *this); - break; - } - case jsoncons::msgpack::detail::msgpack_format ::true_cd: - { - handler_.bool_value(true, semantic_tag::none, *this); - break; - } - case jsoncons::msgpack::detail::msgpack_format ::false_cd: - { - handler_.bool_value(false, semantic_tag::none, *this); - break; - } - case jsoncons::msgpack::detail::msgpack_format ::float32_cd: - { - uint8_t buf[sizeof(float)]; - source_.read(buf, sizeof(float)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - float val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - handler_.double_value(val, semantic_tag::none, *this); - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::float64_cd: - { - uint8_t buf[sizeof(double)]; - source_.read(buf, sizeof(double)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - double val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - handler_.double_value(val, semantic_tag::none, *this); - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::uint8_cd: - { - uint8_t val{}; - source_.get(val); - handler_.uint64_value(val, semantic_tag::none, *this); - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::uint16_cd: - { - uint8_t buf[sizeof(uint16_t)]; - source_.read(buf, sizeof(uint16_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - uint16_t val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - handler_.uint64_value(val, semantic_tag::none, *this); - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::uint32_cd: - { - uint8_t buf[sizeof(uint32_t)]; - source_.read(buf, sizeof(uint32_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - uint32_t val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - handler_.uint64_value(val, semantic_tag::none, *this); - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::uint64_cd: - { - uint8_t buf[sizeof(uint64_t)]; - source_.read(buf, sizeof(uint64_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - uint64_t val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - handler_.uint64_value(val, semantic_tag::none, *this); - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::int8_cd: - { - uint8_t buf[sizeof(int8_t)]; - source_.read(buf, sizeof(int8_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int8_t val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - handler_.int64_value(val, semantic_tag::none, *this); - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::int16_cd: - { - uint8_t buf[sizeof(int16_t)]; - source_.read(buf, sizeof(int16_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int16_t val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - handler_.int64_value(val, semantic_tag::none, *this); - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::int32_cd: - { - uint8_t buf[sizeof(int32_t)]; - source_.read(buf, sizeof(int32_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int32_t val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - handler_.int64_value(val, semantic_tag::none, *this); - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::int64_cd: - { - uint8_t buf[sizeof(int64_t)]; - source_.read(buf, sizeof(int64_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int64_t val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - handler_.int64_value(val, semantic_tag::none, *this); - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::str8_cd: - { - uint8_t buf[sizeof(int8_t)]; - source_.read(buf, sizeof(int8_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int8_t len = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - - std::basic_string s; - source_.read(std::back_inserter(s), len); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - auto result = unicons::validate(s.begin(),s.end()); - if (result.ec != unicons::conv_errc()) - { - ec = msgpack_errc::invalid_utf8_text_string; - return; - } - handler_.string_value(basic_string_view(s.data(),s.length()), semantic_tag::none, *this); - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::str16_cd: - { - uint8_t buf[sizeof(int16_t)]; - source_.read(buf, sizeof(int16_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int16_t len = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - - std::basic_string s; - source_.read(std::back_inserter(s), len); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - - auto result = unicons::validate(s.begin(),s.end()); - if (result.ec != unicons::conv_errc()) - { - ec = msgpack_errc::invalid_utf8_text_string; - return; - } - handler_.string_value(basic_string_view(s.data(),s.length()), semantic_tag::none, *this); - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::str32_cd: - { - uint8_t buf[sizeof(int32_t)]; - source_.read(buf, sizeof(int32_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int32_t len = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - - std::basic_string s; - source_.read(std::back_inserter(s), len); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - - auto result = unicons::validate(s.begin(),s.end()); - if (result.ec != unicons::conv_errc()) - { - ec = msgpack_errc::invalid_utf8_text_string; - return; - } - handler_.string_value(basic_string_view(s.data(),s.length()), semantic_tag::none, *this); - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::bin8_cd: - { - uint8_t buf[sizeof(int8_t)]; - source_.read(buf, sizeof(int8_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int8_t len = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - - std::vector v; - v.reserve(len); - source_.read(std::back_inserter(v), len); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - - handler_.byte_string_value(byte_string_view(v.data(),v.size()), - semantic_tag::none, - *this); - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::bin16_cd: - { - uint8_t buf[sizeof(int16_t)]; - source_.read(buf, sizeof(int16_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int16_t len = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - - std::vector v; - v.reserve(len); - source_.read(std::back_inserter(v), len); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - - handler_.byte_string_value(byte_string_view(v.data(),v.size()), - semantic_tag::none, - *this); - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::bin32_cd: - { - uint8_t buf[sizeof(int32_t)]; - source_.read(buf, sizeof(int32_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int32_t len = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - - std::vector v; - v.reserve(len); - source_.read(std::back_inserter(v), len); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - - handler_.byte_string_value(byte_string_view(v.data(),v.size()), - semantic_tag::none, - *this); - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::array16_cd: - { - uint8_t buf[sizeof(int16_t)]; - source_.read(buf, sizeof(int16_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int16_t len = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - - handler_.begin_array(len, semantic_tag::none, *this); - ++nesting_depth_; - for (int16_t i = 0; i < len; ++i) - { - read(ec); - if (ec) - { - return; - } - } - handler_.end_array(*this); - --nesting_depth_; - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::array32_cd: - { - uint8_t buf[sizeof(int32_t)]; - source_.read(buf, sizeof(int32_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int32_t len = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - - handler_.begin_array(len, semantic_tag::none, *this); - ++nesting_depth_; - for (int32_t i = 0; i < len; ++i) - { - read(ec); - if (ec) - { - return; - } - } - handler_.end_array(*this); - --nesting_depth_; - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::map16_cd : - { - uint8_t buf[sizeof(int16_t)]; - source_.read(buf, sizeof(int16_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int16_t len = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - - handler_.begin_object(len, semantic_tag::none, *this); - ++nesting_depth_; - for (int16_t i = 0; i < len; ++i) - { - parse_name(ec); - if (ec) - { - return; - } - read(ec); - if (ec) - { - return; - } - } - handler_.end_object(*this); - --nesting_depth_; - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::map32_cd : - { - uint8_t buf[sizeof(int32_t)]; - source_.read(buf, sizeof(int32_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int32_t len = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - - handler_.begin_object(len, semantic_tag::none, *this); - ++nesting_depth_; - for (int32_t i = 0; i < len; ++i) - { - parse_name(ec); - if (ec) - { - return; - } - read(ec); - if (ec) - { - return; - } - } - handler_.end_object(*this); - --nesting_depth_; - break; - } - - default: - { - //error - } - } - } - } - - void parse_name(std::error_code& ec) - { - uint8_t type{}; - source_.get(type); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - - //const uint8_t* pos = input_ptr_++; - if (type >= 0xa0 && type <= 0xbf) - { - // fixstr - const size_t len = type & 0x1f; - - std::basic_string s; - source_.read(std::back_inserter(s), len); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - auto result = unicons::validate(s.begin(),s.end()); - if (result.ec != unicons::conv_errc()) - { - ec = msgpack_errc::invalid_utf8_text_string; - return; - } - handler_.name(basic_string_view(s.data(),s.length()), *this); - } - else - { - switch (type) - { - case jsoncons::msgpack::detail::msgpack_format ::str8_cd: - { - uint8_t buf[sizeof(int8_t)]; - source_.read(buf, sizeof(int8_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int8_t len = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - - std::basic_string s; - source_.read(std::back_inserter(s), len); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - - auto result = unicons::validate(s.begin(),s.end()); - if (result.ec != unicons::conv_errc()) - { - ec = msgpack_errc::invalid_utf8_text_string; - return; - } - handler_.name(basic_string_view(s.data(),s.length()), *this); - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::str16_cd: - { - uint8_t buf[sizeof(int16_t)]; - source_.read(buf, sizeof(int16_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int16_t len = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - - std::basic_string s; - source_.read(std::back_inserter(s), len); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - - //std::basic_string s; - //auto result = unicons::convert( - // first, last,std::back_inserter(s),unicons::conv_flags::strict); - //if (result.ec != unicons::conv_errc()) - //{ - // JSONCONS_THROW(json_runtime_error("Illegal unicode")); - //} - handler_.name(basic_string_view(s.data(),s.length()), *this); - break; - } - - case jsoncons::msgpack::detail::msgpack_format ::str32_cd: - { - uint8_t buf[sizeof(int32_t)]; - source_.read(buf, sizeof(int32_t)); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int32_t len = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - - std::basic_string s; - source_.read(std::back_inserter(s), len); - if (source_.eof()) - { - ec = msgpack_errc::unexpected_eof; - return; - } - - //std::basic_string s; - //auto result = unicons::convert( - // first, last,std::back_inserter(s),unicons::conv_flags::strict); - //if (result.ec != unicons::conv_errc()) - //{ - // JSONCONS_THROW(json_runtime_error("Illegal unicode")); - //} - handler_.name(basic_string_view(s.data(),s.length()), *this); - break; - } - } - - } - } -}; - -typedef basic_msgpack_reader msgpack_reader; - -typedef basic_msgpack_reader msgpack_bytes_reader; - -#if !defined(JSONCONS_NO_DEPRECATED) -typedef msgpack_bytes_reader msgpack_buffer_reader; -#endif - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson.hpp deleted file mode 100644 index d30c562c41..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson.hpp +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_UBJSON_UBJSON_HPP -#define JSONCONS_UBJSON_UBJSON_HPP - -#include -#include -#include -#include // std::enable_if -#include // std::basic_istream -#include -#include -#include -#include - -namespace jsoncons { namespace ubjson { - -// encode_ubjson - -template -typename std::enable_if::value,void>::type -encode_ubjson(const T& j, std::vector& v) -{ - typedef typename T::char_type char_type; - ubjson_bytes_encoder encoder(v); - auto adaptor = make_json_content_handler_adaptor>(encoder); - j.dump(adaptor); -} - -template -typename std::enable_if::value,void>::type -encode_ubjson(const T& val, std::vector& v) -{ - ubjson_bytes_encoder encoder(v); - write_to(json(), val, encoder); -} - -template -typename std::enable_if::value,void>::type -encode_ubjson(const T& j, std::ostream& os) -{ - typedef typename T::char_type char_type; - ubjson_encoder encoder(os); - auto adaptor = make_json_content_handler_adaptor>(encoder); - j.dump(adaptor); -} - -template -typename std::enable_if::value,void>::type -encode_ubjson(const T& val, std::ostream& os) -{ - ubjson_encoder encoder(os); - write_to(json(), val, encoder); -} - -// decode_ubjson - -template -typename std::enable_if::value,T>::type -decode_ubjson(const std::vector& v) -{ - jsoncons::json_decoder decoder; - auto adaptor = make_json_content_handler_adaptor(decoder); - basic_ubjson_reader reader(v, adaptor); - reader.read(); - return decoder.get_result(); -} - -template -typename std::enable_if::value,T>::type -decode_ubjson(const std::vector& v) -{ - jsoncons::json_decoder decoder; - basic_ubjson_reader reader(v, decoder); - reader.read(); - return decoder.get_result().template as(); -} - -template -typename std::enable_if::value,T>::type -decode_ubjson(std::istream& is) -{ - jsoncons::json_decoder decoder; - auto adaptor = make_json_content_handler_adaptor(decoder); - ubjson_reader reader(is, adaptor); - reader.read(); - return decoder.get_result(); -} - -template -typename std::enable_if::value,T>::type -decode_ubjson(std::istream& is) -{ - jsoncons::json_decoder decoder; - ubjson_reader reader(is, decoder); - reader.read(); - return decoder.get_result(); -} - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson_detail.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson_detail.hpp deleted file mode 100644 index 99f2bf922c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson_detail.hpp +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_UBJSON_UBJSON_DETAIL_HPP -#define JSONCONS_UBJSON_UBJSON_DETAIL_HPP - -#include -#include -#include - -namespace jsoncons { namespace ubjson { namespace detail { - -namespace ubjson_format -{ - const uint8_t null_type = 'Z'; - const uint8_t no_op_type = 'N'; - const uint8_t true_type = 'T'; - const uint8_t false_type = 'F'; - const uint8_t int8_type = 'i'; - const uint8_t uint8_type = 'U'; - const uint8_t int16_type = 'I'; - const uint8_t int32_type = 'l'; - const uint8_t int64_type = 'L'; - const uint8_t float32_type = 'd'; - const uint8_t float64_type = 'D'; - const uint8_t high_precision_number_type = 'H'; - const uint8_t char_type = 'C'; - const uint8_t string_type = 'S'; - const uint8_t start_array_marker = '['; - const uint8_t end_array_marker = ']'; - const uint8_t start_object_marker = '{'; - const uint8_t end_object_marker = '}'; - const uint8_t type_marker = '$'; - const uint8_t count_marker = '#'; -} - -}}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson_encoder.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson_encoder.hpp deleted file mode 100644 index 10085420c8..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson_encoder.hpp +++ /dev/null @@ -1,433 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_UBJSON_UBJSON_ENCODER_HPP -#define JSONCONS_UBJSON_UBJSON_ENCODER_HPP - -#include -#include -#include // std::numeric_limits -#include -#include // std::move -#include -#include -#include -#include -#include -#include -#include - -namespace jsoncons { namespace ubjson { - -enum class ubjson_container_type {object, indefinite_length_object, array, indefinite_length_array}; - -template -class basic_ubjson_encoder final : public basic_json_content_handler -{ - - enum class decimal_parse_state { start, integer, exp1, exp2, fraction1 }; -public: - using typename basic_json_content_handler::string_view_type; - typedef Result result_type; - -private: - struct stack_item - { - ubjson_container_type type_; - size_t length_; - size_t count_; - - stack_item(ubjson_container_type type, size_t length = 0) - : type_(type), length_(length), count_(0) - { - } - - size_t length() const - { - return length_; - } - - size_t count() const - { - return count_; - } - - bool is_object() const - { - return type_ == ubjson_container_type::object || type_ == ubjson_container_type::indefinite_length_object; - } - - bool is_indefinite_length() const - { - return type_ == ubjson_container_type::indefinite_length_array || type_ == ubjson_container_type::indefinite_length_object; - } - - }; - std::vector stack_; - Result result_; - - // Noncopyable and nonmoveable - basic_ubjson_encoder(const basic_ubjson_encoder&) = delete; - basic_ubjson_encoder& operator=(const basic_ubjson_encoder&) = delete; -public: - basic_ubjson_encoder(result_type result) - : result_(std::move(result)) - { - } - - ~basic_ubjson_encoder() - { - try - { - result_.flush(); - } - catch (...) - { - } - } - -private: - // Implementing methods - - void do_flush() override - { - result_.flush(); - } - - bool do_begin_object(semantic_tag, const ser_context&) override - { - stack_.push_back(stack_item(ubjson_container_type::indefinite_length_object)); - result_.push_back(jsoncons::ubjson::detail::ubjson_format::start_object_marker); - - return true; - } - - bool do_begin_object(size_t length, semantic_tag, const ser_context&) override - { - stack_.push_back(stack_item(ubjson_container_type::object, length)); - result_.push_back(jsoncons::ubjson::detail::ubjson_format::start_object_marker); - result_.push_back(jsoncons::ubjson::detail::ubjson_format::count_marker); - put_length(length); - - return true; - } - - bool do_end_object(const ser_context&) override - { - JSONCONS_ASSERT(!stack_.empty()); - if (stack_.back().is_indefinite_length()) - { - result_.push_back(jsoncons::ubjson::detail::ubjson_format::end_object_marker); - } - else - { - if (stack_.back().count() < stack_.back().length()) - { - throw ser_error(ubjson_errc::too_few_items); - } - if (stack_.back().count() > stack_.back().length()) - { - throw ser_error(ubjson_errc::too_many_items); - } - } - stack_.pop_back(); - end_value(); - return true; - } - - bool do_begin_array(semantic_tag, const ser_context&) override - { - stack_.push_back(stack_item(ubjson_container_type::indefinite_length_array)); - result_.push_back(jsoncons::ubjson::detail::ubjson_format::start_array_marker); - - return true; - } - - bool do_begin_array(size_t length, semantic_tag, const ser_context&) override - { - stack_.push_back(stack_item(ubjson_container_type::array, length)); - result_.push_back(jsoncons::ubjson::detail::ubjson_format::start_array_marker); - result_.push_back(jsoncons::ubjson::detail::ubjson_format::count_marker); - put_length(length); - - return true; - } - - bool do_end_array(const ser_context&) override - { - JSONCONS_ASSERT(!stack_.empty()); - if (stack_.back().is_indefinite_length()) - { - result_.push_back(jsoncons::ubjson::detail::ubjson_format::end_array_marker); - } - else - { - if (stack_.back().count() < stack_.back().length()) - { - throw ser_error(ubjson_errc::too_few_items); - } - if (stack_.back().count() > stack_.back().length()) - { - throw ser_error(ubjson_errc::too_many_items); - } - } - stack_.pop_back(); - end_value(); - return true; - } - - bool do_name(const string_view_type& name, const ser_context&) override - { - auto result = unicons::validate(name.begin(), name.end()); - if (result.ec != unicons::conv_errc()) - { - throw ser_error(ubjson_errc::invalid_utf8_text_string); - } - - put_length(name.length()); - - for (auto c : name) - { - result_.push_back(c); - } - return true; - } - - bool do_null_value(semantic_tag, const ser_context&) override - { - // nil - jsoncons::detail::to_big_endian(static_cast(jsoncons::ubjson::detail::ubjson_format::null_type), std::back_inserter(result_)); - end_value(); - return true; - } - - bool do_string_value(const string_view_type& sv, semantic_tag tag, const ser_context&) override - { - switch (tag) - { - case semantic_tag::bigint: - case semantic_tag::bigdec: - { - result_.push_back(jsoncons::ubjson::detail::ubjson_format::high_precision_number_type); - break; - } - default: - { - result_.push_back(jsoncons::ubjson::detail::ubjson_format::string_type); - break; - } - } - - auto result = unicons::validate(sv.begin(), sv.end()); - if (result.ec != unicons::conv_errc()) - { - JSONCONS_THROW(json_runtime_error("Illegal unicode")); - } - - put_length(sv.length()); - - for (auto c : sv) - { - result_.push_back(c); - } - - end_value(); - return true; - } - - void put_length(size_t length) - { - if (length <= (std::numeric_limits::max)()) - { - result_.push_back('U'); - jsoncons::detail::to_big_endian(static_cast(length), std::back_inserter(result_)); - } - else if (length <= (size_t)(std::numeric_limits::max)()) - { - result_.push_back('I'); - jsoncons::detail::to_big_endian(static_cast(length), std::back_inserter(result_)); - } - else if (length <= (uint32_t)(std::numeric_limits::max)()) - { - result_.push_back('l'); - jsoncons::detail::to_big_endian(static_cast(length),std::back_inserter(result_)); - } - else if (length <= (uint64_t)(std::numeric_limits::max)()) - { - result_.push_back('L'); - jsoncons::detail::to_big_endian(static_cast(length),std::back_inserter(result_)); - } - } - - bool do_byte_string_value(const byte_string_view& b, - semantic_tag, - const ser_context&) override - { - - const size_t length = b.length(); - result_.push_back(jsoncons::ubjson::detail::ubjson_format::start_array_marker); - jsoncons::detail::to_big_endian(static_cast(jsoncons::ubjson::detail::ubjson_format::type_marker), std::back_inserter(result_)); - jsoncons::detail::to_big_endian(static_cast(jsoncons::ubjson::detail::ubjson_format::uint8_type), std::back_inserter(result_)); - put_length(length); - - for (auto c : b) - { - result_.push_back(c); - } - - end_value(); - return true; - } - - bool do_double_value(double val, - semantic_tag, - const ser_context&) override - { - float valf = (float)val; - if ((double)valf == val) - { - // float 32 - result_.push_back(static_cast(jsoncons::ubjson::detail::ubjson_format::float32_type)); - jsoncons::detail::to_big_endian(valf,std::back_inserter(result_)); - } - else - { - // float 64 - result_.push_back(static_cast(jsoncons::ubjson::detail::ubjson_format::float64_type)); - jsoncons::detail::to_big_endian(val,std::back_inserter(result_)); - } - - // write double - - end_value(); - return true; - } - - bool do_int64_value(int64_t val, - semantic_tag, - const ser_context&) override - { - if (val >= 0) - { - if (val <= (std::numeric_limits::max)()) - { - // uint 8 stores a 8-bit unsigned integer - result_.push_back(jsoncons::ubjson::detail::ubjson_format::uint8_type); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - else if (val <= (std::numeric_limits::max)()) - { - // uint 16 stores a 16-bit big-endian unsigned integer - result_.push_back(jsoncons::ubjson::detail::ubjson_format::int16_type); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - else if (val <= (std::numeric_limits::max)()) - { - // uint 32 stores a 32-bit big-endian unsigned integer - result_.push_back(jsoncons::ubjson::detail::ubjson_format::int32_type); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - else if (val <= (std::numeric_limits::max)()) - { - // int 64 stores a 64-bit big-endian signed integer - result_.push_back(jsoncons::ubjson::detail::ubjson_format::int64_type); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - else - { - // big integer - } - } - else - { - if (val >= (std::numeric_limits::lowest)()) - { - // int 8 stores a 8-bit signed integer - result_.push_back(jsoncons::ubjson::detail::ubjson_format::int8_type); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - else if (val >= (std::numeric_limits::lowest)()) - { - // int 16 stores a 16-bit big-endian signed integer - result_.push_back(jsoncons::ubjson::detail::ubjson_format::int16_type); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - else if (val >= (std::numeric_limits::lowest)()) - { - // int 32 stores a 32-bit big-endian signed integer - result_.push_back(jsoncons::ubjson::detail::ubjson_format::int32_type); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - else if (val >= (std::numeric_limits::lowest)()) - { - // int 64 stores a 64-bit big-endian signed integer - result_.push_back(jsoncons::ubjson::detail::ubjson_format::int64_type); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - } - end_value(); - return true; - } - - bool do_uint64_value(uint64_t val, - semantic_tag, - const ser_context&) override - { - if (val <= (std::numeric_limits::max)()) - { - result_.push_back(jsoncons::ubjson::detail::ubjson_format::uint8_type); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - else if (val <= (std::numeric_limits::max)()) - { - result_.push_back(jsoncons::ubjson::detail::ubjson_format::int16_type); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - else if (val <= (std::numeric_limits::max)()) - { - result_.push_back(jsoncons::ubjson::detail::ubjson_format::int32_type); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - else if (val <= (uint64_t)(std::numeric_limits::max)()) - { - result_.push_back(jsoncons::ubjson::detail::ubjson_format::int64_type); - jsoncons::detail::to_big_endian(static_cast(val),std::back_inserter(result_)); - } - end_value(); - return true; - } - - bool do_bool_value(bool val, semantic_tag, const ser_context&) override - { - // true and false - result_.push_back(static_cast(val ? jsoncons::ubjson::detail::ubjson_format::true_type : jsoncons::ubjson::detail::ubjson_format::false_type)); - - end_value(); - return true; - } - - void end_value() - { - if (!stack_.empty()) - { - ++stack_.back().count_; - } - } -}; - -typedef basic_ubjson_encoder ubjson_encoder; -typedef basic_ubjson_encoder ubjson_bytes_encoder; - -#if !defined(JSONCONS_NO_DEPRECATED) -template -using basic_ubjson_serializer = basic_ubjson_encoder; - -typedef basic_ubjson_encoder ubjson_serializer; -typedef basic_ubjson_encoder ubjson_buffer_serializer; -#endif - -}} -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson_error.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson_error.hpp deleted file mode 100644 index dc11094ed9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson_error.hpp +++ /dev/null @@ -1,89 +0,0 @@ -/// Copyright 2018 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_UBJSON_UBJSON_ERROR_HPP -#define JSONCONS_UBJSON_UBJSON_ERROR_HPP - -#include -#include - -namespace jsoncons { namespace ubjson { - -enum class ubjson_errc -{ - ok = 0, - unexpected_eof = 1, - source_error, - count_required_after_type, - length_cannot_be_negative, - length_must_be_integer, - unknown_type, - invalid_utf8_text_string, - too_many_items, - too_few_items, - number_too_large -}; - -class ubjson_error_category_impl - : public std::error_category -{ -public: - const char* name() const noexcept override - { - return "jsoncons/ubjson"; - } - std::string message(int ev) const override - { - switch (static_cast(ev)) - { - case ubjson_errc::unexpected_eof: - return "Unexpected end of file"; - case ubjson_errc::source_error: - return "Source error"; - case ubjson_errc::count_required_after_type: - return "Type is specified for container, but count is not specified"; - case ubjson_errc::length_cannot_be_negative: - return "Length cannot be negative"; - case ubjson_errc::length_must_be_integer: - return "Length must be a integer numeric type (int8, uint8, int16, int32, int64)"; - case ubjson_errc::invalid_utf8_text_string: - return "Illegal UTF-8 encoding in text string"; - case ubjson_errc::too_many_items: - return "Too many items were added to a UBJSON object or array of known length"; - case ubjson_errc::too_few_items: - return "Too few items were added to a UBJSON object or array of known length"; - case ubjson_errc::number_too_large: - return "Number exceeds implementation limits"; - default: - return "Unknown UBJSON parser error"; - } - } -}; - -inline -const std::error_category& ubjson_error_category() -{ - static ubjson_error_category_impl instance; - return instance; -} - -inline -std::error_code make_error_code(ubjson_errc e) -{ - return std::error_code(static_cast(e),ubjson_error_category()); -} - - -}} - -namespace std { - template<> - struct is_error_code_enum : public true_type - { - }; -} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson_reader.hpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson_reader.hpp deleted file mode 100644 index 540a588292..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/include/jsoncons_ext/ubjson/ubjson_reader.hpp +++ /dev/null @@ -1,589 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under the Boost license, Version 1.0. -// (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -// See https://github.com/danielaparker/jsoncons for latest version - -#ifndef JSONCONS_UBJSON_UBJSON_READER_HPP -#define JSONCONS_UBJSON_UBJSON_READER_HPP - -#include -#include -#include // std::move -#include -#include -#include -#include -#include -#include - -namespace jsoncons { namespace ubjson { - -template -class basic_ubjson_reader : public ser_context -{ - Src source_; - json_content_handler& handler_; - size_t nesting_depth_; - std::string buffer_; -public: - template - basic_ubjson_reader(Source&& source, json_content_handler& handler) - : source_(std::forward(source)), - handler_(handler), - nesting_depth_(0) - { - } - - void read() - { - std::error_code ec; - read(ec); - if (ec) - { - throw ser_error(ec,line(),column()); - } - } - - void read(std::error_code& ec) - { - try - { - read_internal(ec); - } - catch (const ser_error& e) - { - ec = e.code(); - } - } - - size_t line() const override - { - return 0; - } - - size_t column() const override - { - return source_.position(); - } -private: - - void read_internal(std::error_code& ec) - { - if (source_.is_error()) - { - ec = ubjson_errc::source_error; - return; - } - //const uint8_t* pos = input_ptr_++; - - uint8_t type{}; - if (source_.get(type) == 0) - { - ec = ubjson_errc::unexpected_eof; - return; - } - read_value(type, ec); - } - - void read_value(uint8_t type, std::error_code& ec) - { - switch (type) - { - case jsoncons::ubjson::detail::ubjson_format::null_type: - { - handler_.null_value(semantic_tag::none, *this); - break; - } - case jsoncons::ubjson::detail::ubjson_format::no_op_type: - { - break; - } - case jsoncons::ubjson::detail::ubjson_format::true_type: - { - handler_.bool_value(true, semantic_tag::none, *this); - break; - } - case jsoncons::ubjson::detail::ubjson_format::false_type: - { - handler_.bool_value(false, semantic_tag::none, *this); - break; - } - case jsoncons::ubjson::detail::ubjson_format::int8_type: - { - uint8_t buf[sizeof(int8_t)]; - source_.read(buf, sizeof(int8_t)); - if (source_.eof()) - { - ec = ubjson_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int8_t val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - handler_.int64_value(val, semantic_tag::none, *this); - break; - } - case jsoncons::ubjson::detail::ubjson_format::uint8_type: - { - uint8_t val{}; - if (source_.get(val) == 0) - { - ec = ubjson_errc::unexpected_eof; - return; - } - handler_.uint64_value(val, semantic_tag::none, *this); - break; - } - case jsoncons::ubjson::detail::ubjson_format::int16_type: - { - uint8_t buf[sizeof(int16_t)]; - source_.read(buf, sizeof(int16_t)); - if (source_.eof()) - { - ec = ubjson_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int16_t val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - handler_.int64_value(val, semantic_tag::none, *this); - break; - } - case jsoncons::ubjson::detail::ubjson_format::int32_type: - { - uint8_t buf[sizeof(int32_t)]; - source_.read(buf, sizeof(int32_t)); - if (source_.eof()) - { - ec = ubjson_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int32_t val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - handler_.int64_value(val, semantic_tag::none, *this); - break; - } - case jsoncons::ubjson::detail::ubjson_format::int64_type: - { - uint8_t buf[sizeof(int64_t)]; - source_.read(buf, sizeof(int64_t)); - if (source_.eof()) - { - ec = ubjson_errc::unexpected_eof; - return; - } - const uint8_t* endp; - int64_t val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - handler_.int64_value(val, semantic_tag::none, *this); - break; - } - case jsoncons::ubjson::detail::ubjson_format::float32_type: - { - uint8_t buf[sizeof(float)]; - source_.read(buf, sizeof(float)); - if (source_.eof()) - { - ec = ubjson_errc::unexpected_eof; - return; - } - const uint8_t* endp; - float val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - handler_.double_value(val, semantic_tag::none, *this); - break; - } - case jsoncons::ubjson::detail::ubjson_format::float64_type: - { - uint8_t buf[sizeof(double)]; - source_.read(buf, sizeof(double)); - if (source_.eof()) - { - ec = ubjson_errc::unexpected_eof; - return; - } - const uint8_t* endp; - double val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - handler_.double_value(val, semantic_tag::none, *this); - break; - } - case jsoncons::ubjson::detail::ubjson_format::char_type: - { - uint8_t buf[sizeof(char)]; - source_.read(buf, sizeof(char)); - if (source_.eof()) - { - ec = ubjson_errc::unexpected_eof; - return; - } - const uint8_t* endp; - char c = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - auto result = unicons::validate(&c,&c+1); - if (result.ec != unicons::conv_errc()) - { - ec = ubjson_errc::invalid_utf8_text_string; - return; - } - handler_.string_value(basic_string_view(&c,1), semantic_tag::none, *this); - break; - } - case jsoncons::ubjson::detail::ubjson_format::string_type: - { - size_t length = get_length(ec); - if (ec) - { - return; - } - std::string s; - source_.read(std::back_inserter(s), length); - if (source_.eof()) - { - ec = ubjson_errc::unexpected_eof; - return; - } - auto result = unicons::validate(s.begin(),s.end()); - if (result.ec != unicons::conv_errc()) - { - ec = ubjson_errc::invalid_utf8_text_string; - return; - } - handler_.string_value(basic_string_view(s.data(),s.length()), semantic_tag::none, *this); - break; - } - case jsoncons::ubjson::detail::ubjson_format::high_precision_number_type: - { - size_t length = get_length(ec); - if (ec) - { - return; - } - std::string s; - source_.read(std::back_inserter(s), length); - if (source_.eof()) - { - ec = ubjson_errc::unexpected_eof; - return; - } - if (jsoncons::detail::is_integer(s.data(),s.length())) - { - handler_.string_value(basic_string_view(s.data(),s.length()), semantic_tag::bigint, *this); - } - else - { - handler_.string_value(basic_string_view(s.data(),s.length()), semantic_tag::bigdec, *this); - } - break; - } - case jsoncons::ubjson::detail::ubjson_format::start_array_marker: - { - if (source_.peek() == jsoncons::ubjson::detail::ubjson_format::type_marker) - { - source_.ignore(1); - uint8_t item_type{}; - if (source_.get(item_type) == 0) - { - ec = ubjson_errc::unexpected_eof; - return; - } - if (source_.peek() == jsoncons::ubjson::detail::ubjson_format::count_marker) - { - source_.ignore(1); - size_t length = get_length(ec); - handler_.begin_array(length, semantic_tag::none, *this); - for (size_t i = 0; i < length; ++i) - { - read_value(item_type, ec); - if (ec) - { - return; - } - } - handler_.end_array(*this); - } - else - { - ec = ubjson_errc::count_required_after_type; - return; - } - } - else if (source_.peek() == jsoncons::ubjson::detail::ubjson_format::count_marker) - { - source_.ignore(1); - size_t length = get_length(ec); - handler_.begin_array(length, semantic_tag::none, *this); - for (size_t i = 0; i < length; ++i) - { - read(ec); - if (ec) - { - return; - } - } - handler_.end_array(*this); - } - else - { - handler_.begin_array(semantic_tag::none, *this); - while (source_.peek() != jsoncons::ubjson::detail::ubjson_format::end_array_marker) - { - read(ec); - if (ec) - { - return; - } - } - handler_.end_array(*this); - source_.ignore(1); - } - break; - } - case jsoncons::ubjson::detail::ubjson_format::start_object_marker: - { - if (source_.peek() == jsoncons::ubjson::detail::ubjson_format::type_marker) - { - source_.ignore(1); - uint8_t item_type{}; - if (source_.get(item_type) == 0) - { - ec = ubjson_errc::unexpected_eof; - return; - } - if (source_.peek() == jsoncons::ubjson::detail::ubjson_format::count_marker) - { - source_.ignore(1); - size_t length = get_length(ec); - handler_.begin_object(length, semantic_tag::none, *this); - for (size_t i = 0; i < length; ++i) - { - read_name(ec); - if (ec) - { - return; - } - read_value(item_type, ec); - if (ec) - { - return; - } - } - handler_.end_object(*this); - } - else - { - ec = ubjson_errc::count_required_after_type; - return; - } - } - else - { - if (source_.peek() == jsoncons::ubjson::detail::ubjson_format::count_marker) - { - source_.ignore(1); - size_t length = get_length(ec); - handler_.begin_object(length, semantic_tag::none, *this); - for (size_t i = 0; i < length; ++i) - { - read_name(ec); - if (ec) - { - return; - } - read(ec); - if (ec) - { - return; - } - } - handler_.end_object(*this); - } - else - { - handler_.begin_object(semantic_tag::none, *this); - while (source_.peek() != jsoncons::ubjson::detail::ubjson_format::end_object_marker) - { - read_name(ec); - if (ec) - { - return; - } - read(ec); - if (ec) - { - return; - } - } - handler_.end_object(*this); - source_.ignore(1); - } - } - break; - } - default: - { - ec = ubjson_errc::unknown_type; - return; - } - } - } - - size_t get_length(std::error_code& ec) - { - size_t length = 0; - if (JSONCONS_UNLIKELY(source_.eof())) - { - ec = ubjson_errc::unexpected_eof; - return length; - } - uint8_t type{}; - if (source_.get(type) == 0) - { - ec = ubjson_errc::unexpected_eof; - return length; - } - switch (type) - { - case jsoncons::ubjson::detail::ubjson_format::int8_type: - { - uint8_t buf[sizeof(int8_t)]; - source_.read(buf, sizeof(int8_t)); - if (source_.eof()) - { - ec = ubjson_errc::unexpected_eof; - return length; - } - const uint8_t* endp; - int8_t val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - if (val >= 0) - { - length = val; - } - else - { - ec = ubjson_errc::length_cannot_be_negative; - return length; - } - break; - } - case jsoncons::ubjson::detail::ubjson_format::uint8_type: - { - uint8_t val{}; - if (source_.get(val) == 0) - { - ec = ubjson_errc::unexpected_eof; - return length; - } - length = val; - break; - } - case jsoncons::ubjson::detail::ubjson_format::int16_type: - { - uint8_t buf[sizeof(int16_t)]; - source_.read(buf, sizeof(int16_t)); - if (source_.eof()) - { - ec = ubjson_errc::unexpected_eof; - return length; - } - const uint8_t* endp; - int16_t val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - if (val >= 0) - { - length = val; - } - else - { - ec = ubjson_errc::length_cannot_be_negative; - return length; - } - break; - } - case jsoncons::ubjson::detail::ubjson_format::int32_type: - { - uint8_t buf[sizeof(int32_t)]; - source_.read(buf, sizeof(int32_t)); - if (source_.eof()) - { - ec = ubjson_errc::unexpected_eof; - return length; - } - const uint8_t* endp; - int32_t val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - if (val >= 0) - { - length = val; - } - else - { - ec = ubjson_errc::length_cannot_be_negative; - return length; - } - break; - } - case jsoncons::ubjson::detail::ubjson_format::int64_type: - { - uint8_t buf[sizeof(int64_t)]; - source_.read(buf, sizeof(int64_t)); - if (source_.eof()) - { - ec = ubjson_errc::unexpected_eof; - return length; - } - const uint8_t* endp; - int64_t val = jsoncons::detail::from_big_endian(buf,buf+sizeof(buf),&endp); - if (val >= 0) - { - length = (size_t)val; - if (length != (uint64_t)val) - { - ec = ubjson_errc::number_too_large; - return length; - } - } - else - { - ec = ubjson_errc::length_cannot_be_negative; - return length; - } - break; - } - default: - { - ec = ubjson_errc::length_must_be_integer; - return length; - } - } - return length; - } - - void read_name(std::error_code& ec) - { - size_t length = get_length(ec); - if (ec) - { - return; - } - std::string s; - source_.read(std::back_inserter(s), length); - if (source_.eof()) - { - ec = ubjson_errc::unexpected_eof; - return; - } - auto result = unicons::validate(s.begin(),s.end()); - if (result.ec != unicons::conv_errc()) - { - ec = ubjson_errc::invalid_utf8_text_string; - return; - } - handler_.name(basic_string_view(s.data(),s.length()), *this); - } -}; - -typedef basic_ubjson_reader ubjson_reader; - -typedef basic_ubjson_reader ubjson_bytes_reader; - -#if !defined(JSONCONS_NO_DEPRECATED) -typedef ubjson_bytes_reader ubjson_buffer_reader; -#endif - -}} - -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/CMakeLists.txt b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/CMakeLists.txt deleted file mode 100644 index 241057fc7c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/CMakeLists.txt +++ /dev/null @@ -1,99 +0,0 @@ -cmake_minimum_required(VERSION 3.1) - -if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) - project(jsoncons-test) - - find_package(jsoncons REQUIRED CONFIG) - set(JSONCONS_INCLUDE_DIR ${jsoncons_INCLUDE_DIRS}) -endif () - -if(NOT CMAKE_BUILD_TYPE) -message(STATUS "Forcing tests build type to Release") -set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE) -endif() - -include(CheckCXXCompilerFlag) - -string(TOUPPER "${CMAKE_BUILD_TYPE}" U_CMAKE_BUILD_TYPE) - -################ -# ARM SETTINGS # -################ - -OPTION(CROSS_COMPILE_ARM "cross compile for ARM targets" OFF) -OPTION(JSONCONS_SANITIZE "sanitize" OFF) -option(JSONCONS_VALGRIND "Execute tests with valgrind" OFF) - -if(JSONCONS_VALGRIND) - find_program(CMAKE_MEMORYCHECK_COMMAND valgrind) - message(STATUS "Executing test suite with Valgrind (${CMAKE_MEMORYCHECK_COMMAND})") - set(memcheck_command "${CMAKE_MEMORYCHECK_COMMAND} ${CMAKE_MEMORYCHECK_COMMAND_OPTIONS} --error-exitcode=1 --leak-check=full") - separate_arguments(memcheck_command) -endif() - -include (CTest) - -# Note: to compile on ARM (or cross compile), you may need to add the following: -# -DTARGET_ARCH="armv8-a -mfpu=neon -mfloat-abi=softfp -target arm-linux-gnueabi" -set(TARGET_ARCH "native" CACHE STRING "Target architecture arguments") -set(ARM_ARCH_DIRECTORY "arm-linux-gnueabi" CACHE STRING "ARM arch header dir") -set(ARM_GCC_VER "4.7.3" CACHE STRING "ARM GCC header dir") - -# load per-platform configuration -include (${JSONCONS_PROJECT_DIR}/build_files/cmake/${CMAKE_SYSTEM_NAME}.cmake) - -if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Intel") - if (CROSS_COMPILE_ARM) - # We're cross-compiling with clang++ on Travis, this is all pretty specific and just for testing - set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS) - set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS) - set(CMAKE_THREAD_LIBS_INIT) - - include_directories(/usr/${ARM_ARCH_DIRECTORY}/include/c++/${ARM_GCC_VER}/${ARM_ARCH_DIRECTORY}/) - include_directories(/usr/${ARM_ARCH_DIRECTORY}/include/c++/${ARM_GCC_VER}/) - include_directories(/usr/${ARM_ARCH_DIRECTORY}/include/) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=${TARGET_ARCH} -Wunused-parameter -Wextra -Wreorder -std=c++11") - endif() -endif() -if (JSONCONS_SANITIZE) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O2 -fsanitize=address -fsanitize=undefined -fno-omit-frame-pointer") -endif() - -if(MSVC) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /MP /bigobj") - set(CMAKE_EXE_LINKER_FLAGS /MANIFEST:NO) -endif() - -set(CTEST_OUTPUT_ON_FAILURE ON) - -set(JSONCONS_TESTS_DIR ${JSONCONS_PROJECT_DIR}/tests) -set(JSONCONS_TESTS_SOURCE_DIR ${JSONCONS_PROJECT_DIR}/tests/src) -set(JSONCONS_INCLUDE_DIR ${JSONCONS_PROJECT_DIR}/include) -set(JSONCONS_THIRD_PARTY_INCLUDE_DIR ${JSONCONS_PROJECT_DIR}/third_party) - -set(CATCH_INCLUDE_DIR ${JSONCONS_THIRD_PARTY_INCLUDE_DIR}/catch) -add_library(Catch INTERFACE) -target_include_directories (Catch INTERFACE ${CATCH_INCLUDE_DIR}) - -file(GLOB_RECURSE JSONCONS_TESTS_SOURCES ${JSONCONS_TESTS_SOURCE_DIR}/*.cpp) - -#set(JSONCONS_TESTS_SOURCES -# ${JSONCONS_TESTS_SOURCE_DIR}/tests_main.cpp -# ${JSONCONS_TESTS_SOURCE_DIR}/jsonpath/jsonpath_tests.cpp -# ) -set(JSONCONS_TARGET test_jsoncons) -add_executable(${JSONCONS_TARGET} EXCLUDE_FROM_ALL ${JSONCONS_TESTS_SOURCES} ${JSONCONS_HEADERS}) - -add_test(example_test ${JSONCONS_TARGET}) - -target_include_directories (${JSONCONS_TARGET} PUBLIC ${JSONCONS_INCLUDE_DIR} - PUBLIC ${JSONCONS_THIRD_PARTY_INCLUDE_DIR}) - -target_link_libraries(${JSONCONS_TARGET} Catch) - -if (CROSS_COMPILE_ARM) - add_custom_target(jtest COMMAND qemu-arm -L /usr/arm-linux-gnueabi/ test_jsoncons DEPENDS ${JSONCONS_TARGET}) -else() - add_custom_target(jtest COMMAND test_jsoncons DEPENDS ${JSONCONS_TARGET}) -endif() - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/Readme.txt b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/Readme.txt deleted file mode 100644 index 1674acf256..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/Readme.txt +++ /dev/null @@ -1,37 +0,0 @@ - -Build instructions - -Microsoft Visual Studio -======================= -To build the test suite with Visual C++ version 10 SP1 or later: - -Set BOOST_ROOT to the root directory containing Boost -Set BOOST_LIB to the directory containing Boost static libraries - -CMake -===== -* set current directory to jsonsons/test_suite/build/cmake - -* On Windows, set BOOST_ROOT and BOOST_LIBRARYDIR to enable cmake to locate boost stuff. - -* Generate build scripts of your choise: cmake -G - where can be (other choises are possible): - * Windows: "Visual Studio 10" or "NMake Makefiles" - * Unix: "Ninja" or "Unix Makefiles" - -* Compilation: use appropriate actions depending of generated files. - -* Tests execution: - * Windows: ./run_tests.bat - * Unix: ./run_tests.sh - -SCons -===== -WARNING: Only usable on Unix with gcc as compiler - -* set current directory to jsonsons/test_suite/build/scons - -* compilation: scons - -* Tests execution: - * Unix: ./run_tests.sh diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/document.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/document.json deleted file mode 100644 index 216f518811..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/document.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "store": { - "book": [ - { "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - }, - { "category": "fiction", - "author": "Herman Melville", - "title": "Moby Dick", - "isbn": "0-553-21311-3", - "price": 8.99 - }, - { "category": "fiction", - "author": "J. R. R. Tolkien", - "title": "The Lord of the Rings", - "isbn": "0-395-19395-8", - "price": 22.99 - } - ], - "bicycle": [{ - "color": "black/green", - "price": 97.50, - "options" : [{ - "color": "black/grey", - "price": 125.99 - }] - }] - }, - "boutique": { - "book": [ - { "category": "fiction", - "author": "Victor Hugo", - "title": "Les Miserables", - "price": 6.02 - }, - { "category": "cooking", - "author": "Paula Haney", - "title": "The Hoosier Mama Book of Pie", - "price": 13.77 - }, - { "category": "fiction", - "author": "David Lindsay", - "title": "A Voyage to Arcturus", - "price": 6.99 - }, - { "category": "travel", - "author": "Gillian Price", - "title": "Trekking in the Dolomites", - "isbn": "978 1 85284 563 6", - "price": 18.55 - } - ], - "bicycle": [{ - "color": "black/green", - "price": 97.50 - }] - } -} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query1.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query1.json deleted file mode 100644 index d8c0fd6a38..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query1.json +++ /dev/null @@ -1,30 +0,0 @@ -[ - [ - { - "category" : "reference", - "author" : "Nigel Rees", - "title" : "Sayings of the Century", - "price" : 8.95 - }, - { - "category" : "fiction", - "author" : "Evelyn Waugh", - "title" : "Sword of Honour", - "price" : 12.99 - }, - { - "category" : "fiction", - "author" : "Herman Melville", - "title" : "Moby Dick", - "isbn" : "0-553-21311-3", - "price" : 8.99 - }, - { - "category" : "fiction", - "author" : "J. R. R. Tolkien", - "title" : "The Lord of the Rings", - "isbn" : "0-395-19395-8", - "price" : 22.99 - } - ] -] diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query1.jsonpath b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query1.jsonpath deleted file mode 100644 index b3e3b6a887..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query1.jsonpath +++ /dev/null @@ -1 +0,0 @@ -$['store']['book'] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query2.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query2.json deleted file mode 100644 index eff4bfb852..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query2.json +++ /dev/null @@ -1,4 +0,0 @@ -[ - 97.5, - 97.5 -] diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query2.jsonpath b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query2.jsonpath deleted file mode 100644 index 5f19690b9a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query2.jsonpath +++ /dev/null @@ -1 +0,0 @@ -$..['bicycle'][*].price \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query3.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query3.json deleted file mode 100644 index 1c09ec426e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query3.json +++ /dev/null @@ -1,5 +0,0 @@ -[ - 97.5, - 125.99, - 97.5 -] diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query3.jsonpath b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query3.jsonpath deleted file mode 100644 index 460c5fda6b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query3.jsonpath +++ /dev/null @@ -1 +0,0 @@ -$..['bicycle']..price \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query4.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query4.json deleted file mode 100644 index 1e490977dd..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query4.json +++ /dev/null @@ -1,16 +0,0 @@ -[ - { - "color" : "black/green", - "price" : 97.5, - "options" : [ - { - "color" : "black/grey", - "price" : 125.99 - } - ] - }, - { - "color" : "black/green", - "price" : 97.5 - } -] diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query4.jsonpath b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query4.jsonpath deleted file mode 100644 index 700e61a08c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query4.jsonpath +++ /dev/null @@ -1 +0,0 @@ -$..bicycle[?(@.price>=97)] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query5.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query5.json deleted file mode 100644 index 3916f84801..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query5.json +++ /dev/null @@ -1,6 +0,0 @@ -[ - "Evelyn Waugh", - "J. R. R. Tolkien", - "Paula Haney", - "Gillian Price" -] diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query5.jsonpath b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query5.jsonpath deleted file mode 100644 index 9c98cc02b7..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query5.jsonpath +++ /dev/null @@ -1 +0,0 @@ -$..book[1:4:2].author \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query6.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query6.json deleted file mode 100644 index cbb06d3c0f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query6.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - "Paula Haney", - "Evelyn Waugh", - "J. R. R. Tolkien", - "Victor Hugo", - "Nigel Rees", - "Gillian Price" -] diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query6.jsonpath b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query6.jsonpath deleted file mode 100644 index a222d3d5eb..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query6.jsonpath +++ /dev/null @@ -1 +0,0 @@ -$..book[1:4:2,0].author \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query7.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query7.json deleted file mode 100644 index 2ab89dffb6..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query7.json +++ /dev/null @@ -1,15 +0,0 @@ -[ - { - "category" : "fiction", - "author" : "Herman Melville", - "title" : "Moby Dick", - "isbn" : "0-553-21311-3", - "price" : 8.99 - }, - { - "category" : "fiction", - "author" : "David Lindsay", - "title" : "A Voyage to Arcturus", - "price" : 6.99 - } -] diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query7.jsonpath b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query7.jsonpath deleted file mode 100644 index 5ff10b0e40..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query7.jsonpath +++ /dev/null @@ -1 +0,0 @@ -$..book[(@.length - 2)] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query8.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query8.json deleted file mode 100644 index 38a4b6d933..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query8.json +++ /dev/null @@ -1,16 +0,0 @@ -[ - { - "category" : "fiction", - "author" : "J. R. R. Tolkien", - "title" : "The Lord of the Rings", - "isbn" : "0-395-19395-8", - "price" : 22.99 - }, - { - "category" : "travel", - "author" : "Gillian Price", - "title" : "Trekking in the Dolomites", - "isbn" : "978 1 85284 563 6", - "price" : 18.55 - } -] diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query8.jsonpath b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query8.jsonpath deleted file mode 100644 index e252019011..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query8.jsonpath +++ /dev/null @@ -1 +0,0 @@ -$..book[-1:] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query9.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query9.json deleted file mode 100644 index 9648d8c4cf..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query9.json +++ /dev/null @@ -1,6 +0,0 @@ -[ - "A Voyage to Arcturus","David Lindsay","Evelyn Waugh","Gillian Price", - "Herman Melville","J. R. R. Tolkien","Les Miserables","Moby Dick", - "Nigel Rees","Paula Haney","Sayings of the Century","Sword of Honour", - "The Hoosier Mama Book of Pie","The Lord of the Rings","Trekking in the Dolomites","Victor Hugo" -] diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query9.jsonpath b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query9.jsonpath deleted file mode 100644 index ec276f3040..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONPathTestSuite/query9.jsonpath +++ /dev/null @@ -1 +0,0 @@ -$..book[*]['author','title'] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_double_huge_neg_exp.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_double_huge_neg_exp.json deleted file mode 100644 index ae4c7b71f6..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_double_huge_neg_exp.json +++ /dev/null @@ -1 +0,0 @@ -[123.456e-789] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_huge_exp.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_huge_exp.json deleted file mode 100644 index 9b5efa2368..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_huge_exp.json +++ /dev/null @@ -1 +0,0 @@ -[0.4e00669999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999969999999006] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_neg_int_huge_exp.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_neg_int_huge_exp.json deleted file mode 100644 index 3abd58a5c4..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_neg_int_huge_exp.json +++ /dev/null @@ -1 +0,0 @@ -[-1e+9999] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_pos_double_huge_exp.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_pos_double_huge_exp.json deleted file mode 100644 index e10a7eb62a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_pos_double_huge_exp.json +++ /dev/null @@ -1 +0,0 @@ -[1.5e+9999] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_real_neg_overflow.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_real_neg_overflow.json deleted file mode 100644 index 3d628a9943..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_real_neg_overflow.json +++ /dev/null @@ -1 +0,0 @@ -[-123123e100000] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_real_pos_overflow.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_real_pos_overflow.json deleted file mode 100644 index 54d7d3dcdb..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_real_pos_overflow.json +++ /dev/null @@ -1 +0,0 @@ -[123123e100000] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_real_underflow.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_real_underflow.json deleted file mode 100644 index c5236eb26b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_real_underflow.json +++ /dev/null @@ -1 +0,0 @@ -[123e-10000000] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_too_big_neg_int.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_too_big_neg_int.json deleted file mode 100644 index dfa3846197..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_too_big_neg_int.json +++ /dev/null @@ -1 +0,0 @@ -[-123123123123123123123123123123] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_too_big_pos_int.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_too_big_pos_int.json deleted file mode 100644 index 338a8c3c0b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_too_big_pos_int.json +++ /dev/null @@ -1 +0,0 @@ -[100000000000000000000] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_very_big_negative_int.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_very_big_negative_int.json deleted file mode 100644 index e2d9738c23..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_number_very_big_negative_int.json +++ /dev/null @@ -1 +0,0 @@ -[-237462374673276894279832749832423479823246327846] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_object_key_lone_2nd_surrogate.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_object_key_lone_2nd_surrogate.json deleted file mode 100644 index 5be7ebaf98..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_object_key_lone_2nd_surrogate.json +++ /dev/null @@ -1 +0,0 @@ -{"\uDFAA":0} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_1st_surrogate_but_2nd_missing.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_1st_surrogate_but_2nd_missing.json deleted file mode 100644 index 3b9e37c67a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_1st_surrogate_but_2nd_missing.json +++ /dev/null @@ -1 +0,0 @@ -["\uDADA"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_1st_valid_surrogate_2nd_invalid.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_1st_valid_surrogate_2nd_invalid.json deleted file mode 100644 index 487592832e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_1st_valid_surrogate_2nd_invalid.json +++ /dev/null @@ -1 +0,0 @@ -["\uD888\u1234"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_UTF-16LE_with_BOM.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_UTF-16LE_with_BOM.json deleted file mode 100644 index 2a79c0629a49133d8f715bddbef19cfb3ef025bd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 ScmezWFPcG#;Uy5qG5`P~Km+3d diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_UTF-8_invalid_sequence.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_UTF-8_invalid_sequence.json deleted file mode 100644 index e2a968a159..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_UTF-8_invalid_sequence.json +++ /dev/null @@ -1 +0,0 @@ -["日ш"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_UTF8_surrogate_U+D800.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_UTF8_surrogate_U+D800.json deleted file mode 100644 index 916bff920f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_UTF8_surrogate_U+D800.json +++ /dev/null @@ -1 +0,0 @@ -[""] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_incomplete_surrogate_and_escape_valid.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_incomplete_surrogate_and_escape_valid.json deleted file mode 100644 index 3cb11d2294..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_incomplete_surrogate_and_escape_valid.json +++ /dev/null @@ -1 +0,0 @@ -["\uD800\n"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_incomplete_surrogate_pair.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_incomplete_surrogate_pair.json deleted file mode 100644 index 38ec23bb0a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_incomplete_surrogate_pair.json +++ /dev/null @@ -1 +0,0 @@ -["\uDd1ea"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_incomplete_surrogates_escape_valid.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_incomplete_surrogates_escape_valid.json deleted file mode 100644 index c9cd6f6c31..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_incomplete_surrogates_escape_valid.json +++ /dev/null @@ -1 +0,0 @@ -["\uD800\uD800\n"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_invalid_lonely_surrogate.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_invalid_lonely_surrogate.json deleted file mode 100644 index 3abbd8d8d7..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_invalid_lonely_surrogate.json +++ /dev/null @@ -1 +0,0 @@ -["\ud800"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_invalid_surrogate.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_invalid_surrogate.json deleted file mode 100644 index ffddc04f52..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_invalid_surrogate.json +++ /dev/null @@ -1 +0,0 @@ -["\ud800abc"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_invalid_utf-8.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_invalid_utf-8.json deleted file mode 100644 index 8e45a7ecae..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_invalid_utf-8.json +++ /dev/null @@ -1 +0,0 @@ -[""] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_inverted_surrogates_U+1D11E.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_inverted_surrogates_U+1D11E.json deleted file mode 100644 index 0d5456cc38..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_inverted_surrogates_U+1D11E.json +++ /dev/null @@ -1 +0,0 @@ -["\uDd1e\uD834"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_iso_latin_1.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_iso_latin_1.json deleted file mode 100644 index 9389c98231..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_iso_latin_1.json +++ /dev/null @@ -1 +0,0 @@ -[""] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_lone_second_surrogate.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_lone_second_surrogate.json deleted file mode 100644 index 1dbd397f39..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_lone_second_surrogate.json +++ /dev/null @@ -1 +0,0 @@ -["\uDFAA"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_lone_utf8_continuation_byte.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_lone_utf8_continuation_byte.json deleted file mode 100644 index 729337c0a3..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_lone_utf8_continuation_byte.json +++ /dev/null @@ -1 +0,0 @@ -[""] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_not_in_unicode_range.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_not_in_unicode_range.json deleted file mode 100644 index df90a2916c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_not_in_unicode_range.json +++ /dev/null @@ -1 +0,0 @@ -[""] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_overlong_sequence_2_bytes.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_overlong_sequence_2_bytes.json deleted file mode 100644 index c8cee5e0a1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_overlong_sequence_2_bytes.json +++ /dev/null @@ -1 +0,0 @@ -[""] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_overlong_sequence_6_bytes.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_overlong_sequence_6_bytes.json deleted file mode 100644 index 9a91da791a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_overlong_sequence_6_bytes.json +++ /dev/null @@ -1 +0,0 @@ -[""] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_overlong_sequence_6_bytes_null.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_overlong_sequence_6_bytes_null.json deleted file mode 100644 index d24fffdd98..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_overlong_sequence_6_bytes_null.json +++ /dev/null @@ -1 +0,0 @@ -[""] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_truncated-utf-8.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_truncated-utf-8.json deleted file mode 100644 index 63c7777fb1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_truncated-utf-8.json +++ /dev/null @@ -1 +0,0 @@ -[""] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_utf16BE_no_BOM.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_utf16BE_no_BOM.json deleted file mode 100644 index 57e5392ff6309134268c7e3ec2a9289b99ff7148..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10 PcmZRGW>8{y3B<7g33~zN diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_utf16LE_no_BOM.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_string_utf16LE_no_BOM.json deleted file mode 100644 index c49c1b25d8e5819591993d4d45e4649a46c1b3e0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10 Pcma!MP-1uq#IXzj3t$1} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_structure_500_nested_arrays.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_structure_500_nested_arrays.json deleted file mode 100644 index 7118405898..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_structure_500_nested_arrays.json +++ /dev/null @@ -1 +0,0 @@ -[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_structure_UTF-8_BOM_empty_object.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_structure_UTF-8_BOM_empty_object.json deleted file mode 100644 index 22fdca1b26..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/i_structure_UTF-8_BOM_empty_object.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_1_true_without_comma.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_1_true_without_comma.json deleted file mode 100644 index c14e3f6b1e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_1_true_without_comma.json +++ /dev/null @@ -1 +0,0 @@ -[1 true] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_a_invalid_utf8.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_a_invalid_utf8.json deleted file mode 100644 index 38a86e2e65..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_a_invalid_utf8.json +++ /dev/null @@ -1 +0,0 @@ -[a] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_colon_instead_of_comma.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_colon_instead_of_comma.json deleted file mode 100644 index 0d02ad4483..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_colon_instead_of_comma.json +++ /dev/null @@ -1 +0,0 @@ -["": 1] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_comma_after_close.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_comma_after_close.json deleted file mode 100644 index 2ccba8d950..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_comma_after_close.json +++ /dev/null @@ -1 +0,0 @@ -[""], \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_comma_and_number.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_comma_and_number.json deleted file mode 100644 index d2c84e374a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_comma_and_number.json +++ /dev/null @@ -1 +0,0 @@ -[,1] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_double_comma.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_double_comma.json deleted file mode 100644 index 0431712bc1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_double_comma.json +++ /dev/null @@ -1 +0,0 @@ -[1,,2] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_double_extra_comma.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_double_extra_comma.json deleted file mode 100644 index 3f01d31292..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_double_extra_comma.json +++ /dev/null @@ -1 +0,0 @@ -["x",,] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_extra_close.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_extra_close.json deleted file mode 100644 index c12f9fae1c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_extra_close.json +++ /dev/null @@ -1 +0,0 @@ -["x"]] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_extra_comma.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_extra_comma.json deleted file mode 100644 index 5f8ce18e4b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_extra_comma.json +++ /dev/null @@ -1 +0,0 @@ -["",] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_incomplete.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_incomplete.json deleted file mode 100644 index cc65b0b512..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_incomplete.json +++ /dev/null @@ -1 +0,0 @@ -["x" \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_incomplete_invalid_value.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_incomplete_invalid_value.json deleted file mode 100644 index c21a8f6cff..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_incomplete_invalid_value.json +++ /dev/null @@ -1 +0,0 @@ -[x \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_inner_array_no_comma.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_inner_array_no_comma.json deleted file mode 100644 index c70b716471..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_inner_array_no_comma.json +++ /dev/null @@ -1 +0,0 @@ -[3[4]] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_invalid_utf8.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_invalid_utf8.json deleted file mode 100644 index 6099d3441a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_invalid_utf8.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_items_separated_by_semicolon.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_items_separated_by_semicolon.json deleted file mode 100644 index d4bd7314ca..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_items_separated_by_semicolon.json +++ /dev/null @@ -1 +0,0 @@ -[1:2] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_just_comma.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_just_comma.json deleted file mode 100644 index 9d7077c680..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_just_comma.json +++ /dev/null @@ -1 +0,0 @@ -[,] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_just_minus.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_just_minus.json deleted file mode 100644 index 29501c6ca2..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_just_minus.json +++ /dev/null @@ -1 +0,0 @@ -[-] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_missing_value.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_missing_value.json deleted file mode 100644 index 3a6ba86f3a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_missing_value.json +++ /dev/null @@ -1 +0,0 @@ -[ , ""] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_newlines_unclosed.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_newlines_unclosed.json deleted file mode 100644 index 6466800652..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_newlines_unclosed.json +++ /dev/null @@ -1,3 +0,0 @@ -["a", -4 -,1, \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_number_and_comma.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_number_and_comma.json deleted file mode 100644 index 13f6f1d18a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_number_and_comma.json +++ /dev/null @@ -1 +0,0 @@ -[1,] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_number_and_several_commas.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_number_and_several_commas.json deleted file mode 100644 index 0ac408cb8a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_number_and_several_commas.json +++ /dev/null @@ -1 +0,0 @@ -[1,,] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_spaces_vertical_tab_formfeed.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_spaces_vertical_tab_formfeed.json deleted file mode 100644 index 6cd7cf5855..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_spaces_vertical_tab_formfeed.json +++ /dev/null @@ -1 +0,0 @@ -[" a"\f] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_star_inside.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_star_inside.json deleted file mode 100644 index 5a5194647a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_star_inside.json +++ /dev/null @@ -1 +0,0 @@ -[*] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_unclosed.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_unclosed.json deleted file mode 100644 index 0607330590..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_unclosed.json +++ /dev/null @@ -1 +0,0 @@ -["" \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_unclosed_trailing_comma.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_unclosed_trailing_comma.json deleted file mode 100644 index 6604698ffc..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_unclosed_trailing_comma.json +++ /dev/null @@ -1 +0,0 @@ -[1, \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_unclosed_with_new_lines.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_unclosed_with_new_lines.json deleted file mode 100644 index 4f61de3fb1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_unclosed_with_new_lines.json +++ /dev/null @@ -1,3 +0,0 @@ -[1, -1 -,1 \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_unclosed_with_object_inside.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_unclosed_with_object_inside.json deleted file mode 100644 index 043a87e2db..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_array_unclosed_with_object_inside.json +++ /dev/null @@ -1 +0,0 @@ -[{} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_incomplete_false.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_incomplete_false.json deleted file mode 100644 index eb18c6a143..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_incomplete_false.json +++ /dev/null @@ -1 +0,0 @@ -[fals] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_incomplete_null.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_incomplete_null.json deleted file mode 100644 index c18ef53851..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_incomplete_null.json +++ /dev/null @@ -1 +0,0 @@ -[nul] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_incomplete_true.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_incomplete_true.json deleted file mode 100644 index f451ac6d2e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_incomplete_true.json +++ /dev/null @@ -1 +0,0 @@ -[tru] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_multidigit_number_then_00.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_multidigit_number_then_00.json deleted file mode 100644 index c22507b864f7f089c91c1eb85b1b15cc63b943a6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmXpsGG+h(0mJ~8 diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_++.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_++.json deleted file mode 100644 index bdb62aaf4d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_++.json +++ /dev/null @@ -1 +0,0 @@ -[++1234] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_+1.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_+1.json deleted file mode 100644 index 3cbe58c92b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_+1.json +++ /dev/null @@ -1 +0,0 @@ -[+1] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_+Inf.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_+Inf.json deleted file mode 100644 index 871ae14d53..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_+Inf.json +++ /dev/null @@ -1 +0,0 @@ -[+Inf] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_-01.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_-01.json deleted file mode 100644 index 0df32bac80..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_-01.json +++ /dev/null @@ -1 +0,0 @@ -[-01] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_-1.0..json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_-1.0..json deleted file mode 100644 index 7cf55a85a7..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_-1.0..json +++ /dev/null @@ -1 +0,0 @@ -[-1.0.] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_-2..json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_-2..json deleted file mode 100644 index 9be84365d0..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_-2..json +++ /dev/null @@ -1 +0,0 @@ -[-2.] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_-NaN.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_-NaN.json deleted file mode 100644 index f61615d404..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_-NaN.json +++ /dev/null @@ -1 +0,0 @@ -[-NaN] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_.-1.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_.-1.json deleted file mode 100644 index 1c9f2dd1b7..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_.-1.json +++ /dev/null @@ -1 +0,0 @@ -[.-1] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_.2e-3.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_.2e-3.json deleted file mode 100644 index c6c976f257..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_.2e-3.json +++ /dev/null @@ -1 +0,0 @@ -[.2e-3] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0.1.2.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0.1.2.json deleted file mode 100644 index c83a25621e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0.1.2.json +++ /dev/null @@ -1 +0,0 @@ -[0.1.2] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0.3e+.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0.3e+.json deleted file mode 100644 index a55a1bfefa..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0.3e+.json +++ /dev/null @@ -1 +0,0 @@ -[0.3e+] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0.3e.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0.3e.json deleted file mode 100644 index 3dd5df4b3a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0.3e.json +++ /dev/null @@ -1 +0,0 @@ -[0.3e] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0.e1.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0.e1.json deleted file mode 100644 index c92c71ccb2..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0.e1.json +++ /dev/null @@ -1 +0,0 @@ -[0.e1] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0_capital_E+.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0_capital_E+.json deleted file mode 100644 index 3ba2c7d6d0..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0_capital_E+.json +++ /dev/null @@ -1 +0,0 @@ -[0E+] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0_capital_E.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0_capital_E.json deleted file mode 100644 index 5301840d1c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0_capital_E.json +++ /dev/null @@ -1 +0,0 @@ -[0E] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0e+.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0e+.json deleted file mode 100644 index 8ab0bc4b8b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0e+.json +++ /dev/null @@ -1 +0,0 @@ -[0e+] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0e.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0e.json deleted file mode 100644 index 47ec421bb6..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_0e.json +++ /dev/null @@ -1 +0,0 @@ -[0e] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1.0e+.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1.0e+.json deleted file mode 100644 index cd84b9f69e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1.0e+.json +++ /dev/null @@ -1 +0,0 @@ -[1.0e+] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1.0e-.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1.0e-.json deleted file mode 100644 index 4eb7afa0f9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1.0e-.json +++ /dev/null @@ -1 +0,0 @@ -[1.0e-] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1.0e.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1.0e.json deleted file mode 100644 index 21753f4c74..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1.0e.json +++ /dev/null @@ -1 +0,0 @@ -[1.0e] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1_000.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1_000.json deleted file mode 100644 index 7b18b66b38..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1_000.json +++ /dev/null @@ -1 +0,0 @@ -[1 000.0] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1eE2.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1eE2.json deleted file mode 100644 index 4318a341d7..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_1eE2.json +++ /dev/null @@ -1 +0,0 @@ -[1eE2] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_2.e+3.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_2.e+3.json deleted file mode 100644 index 4442f394dd..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_2.e+3.json +++ /dev/null @@ -1 +0,0 @@ -[2.e+3] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_2.e-3.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_2.e-3.json deleted file mode 100644 index a65060edfc..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_2.e-3.json +++ /dev/null @@ -1 +0,0 @@ -[2.e-3] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_2.e3.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_2.e3.json deleted file mode 100644 index 66f7cf701b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_2.e3.json +++ /dev/null @@ -1 +0,0 @@ -[2.e3] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_9.e+.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_9.e+.json deleted file mode 100644 index 732a7b11ce..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_9.e+.json +++ /dev/null @@ -1 +0,0 @@ -[9.e+] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_Inf.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_Inf.json deleted file mode 100644 index c40c734c3c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_Inf.json +++ /dev/null @@ -1 +0,0 @@ -[Inf] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_NaN.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_NaN.json deleted file mode 100644 index 4992317909..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_NaN.json +++ /dev/null @@ -1 +0,0 @@ -[NaN] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_U+FF11_fullwidth_digit_one.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_U+FF11_fullwidth_digit_one.json deleted file mode 100644 index b14587e5eb..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_U+FF11_fullwidth_digit_one.json +++ /dev/null @@ -1 +0,0 @@ -[1] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_expression.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_expression.json deleted file mode 100644 index 76fdbc8a49..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_expression.json +++ /dev/null @@ -1 +0,0 @@ -[1+2] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_hex_1_digit.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_hex_1_digit.json deleted file mode 100644 index 3b214880c6..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_hex_1_digit.json +++ /dev/null @@ -1 +0,0 @@ -[0x1] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_hex_2_digits.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_hex_2_digits.json deleted file mode 100644 index 83e516ab0e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_hex_2_digits.json +++ /dev/null @@ -1 +0,0 @@ -[0x42] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_infinity.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_infinity.json deleted file mode 100644 index 8c2baf783a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_infinity.json +++ /dev/null @@ -1 +0,0 @@ -[Infinity] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid+-.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid+-.json deleted file mode 100644 index 1cce602b51..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid+-.json +++ /dev/null @@ -1 +0,0 @@ -[0e+-1] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid-negative-real.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid-negative-real.json deleted file mode 100644 index 5fc3c1efb6..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid-negative-real.json +++ /dev/null @@ -1 +0,0 @@ -[-123.123foo] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid-utf-8-in-bigger-int.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid-utf-8-in-bigger-int.json deleted file mode 100644 index 3b97e580e8..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid-utf-8-in-bigger-int.json +++ /dev/null @@ -1 +0,0 @@ -[123] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid-utf-8-in-exponent.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid-utf-8-in-exponent.json deleted file mode 100644 index ea35d723cd..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid-utf-8-in-exponent.json +++ /dev/null @@ -1 +0,0 @@ -[1e1] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid-utf-8-in-int.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid-utf-8-in-int.json deleted file mode 100644 index 371226e4cd..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_invalid-utf-8-in-int.json +++ /dev/null @@ -1 +0,0 @@ -[0] diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_minus_infinity.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_minus_infinity.json deleted file mode 100644 index cf4133d22d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_minus_infinity.json +++ /dev/null @@ -1 +0,0 @@ -[-Infinity] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_minus_sign_with_trailing_garbage.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_minus_sign_with_trailing_garbage.json deleted file mode 100644 index a6d8e78e7c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_minus_sign_with_trailing_garbage.json +++ /dev/null @@ -1 +0,0 @@ -[-foo] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_minus_space_1.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_minus_space_1.json deleted file mode 100644 index 9a5ebedf67..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_minus_space_1.json +++ /dev/null @@ -1 +0,0 @@ -[- 1] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_neg_int_starting_with_zero.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_neg_int_starting_with_zero.json deleted file mode 100644 index 67af0960af..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_neg_int_starting_with_zero.json +++ /dev/null @@ -1 +0,0 @@ -[-012] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_neg_real_without_int_part.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_neg_real_without_int_part.json deleted file mode 100644 index 1f2a43496e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_neg_real_without_int_part.json +++ /dev/null @@ -1 +0,0 @@ -[-.123] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_neg_with_garbage_at_end.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_neg_with_garbage_at_end.json deleted file mode 100644 index 2aa73119fb..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_neg_with_garbage_at_end.json +++ /dev/null @@ -1 +0,0 @@ -[-1x] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_real_garbage_after_e.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_real_garbage_after_e.json deleted file mode 100644 index 9213dfca8d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_real_garbage_after_e.json +++ /dev/null @@ -1 +0,0 @@ -[1ea] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_real_with_invalid_utf8_after_e.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_real_with_invalid_utf8_after_e.json deleted file mode 100644 index 1e52ef964c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_real_with_invalid_utf8_after_e.json +++ /dev/null @@ -1 +0,0 @@ -[1e] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_real_without_fractional_part.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_real_without_fractional_part.json deleted file mode 100644 index 1de287cf89..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_real_without_fractional_part.json +++ /dev/null @@ -1 +0,0 @@ -[1.] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_starting_with_dot.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_starting_with_dot.json deleted file mode 100644 index f682dbdce0..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_starting_with_dot.json +++ /dev/null @@ -1 +0,0 @@ -[.123] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_with_alpha.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_with_alpha.json deleted file mode 100644 index 1e42d81822..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_with_alpha.json +++ /dev/null @@ -1 +0,0 @@ -[1.2a-3] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_with_alpha_char.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_with_alpha_char.json deleted file mode 100644 index b79daccb8a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_with_alpha_char.json +++ /dev/null @@ -1 +0,0 @@ -[1.8011670033376514H-308] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_with_leading_zero.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_with_leading_zero.json deleted file mode 100644 index 7106da1f3b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_number_with_leading_zero.json +++ /dev/null @@ -1 +0,0 @@ -[012] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_bad_value.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_bad_value.json deleted file mode 100644 index a03a8c03b7..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_bad_value.json +++ /dev/null @@ -1 +0,0 @@ -["x", truth] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_bracket_key.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_bracket_key.json deleted file mode 100644 index cc443b4832..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_bracket_key.json +++ /dev/null @@ -1 +0,0 @@ -{[: "x"} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_comma_instead_of_colon.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_comma_instead_of_colon.json deleted file mode 100644 index 8d56377087..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_comma_instead_of_colon.json +++ /dev/null @@ -1 +0,0 @@ -{"x", null} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_double_colon.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_double_colon.json deleted file mode 100644 index 80e8c7b89a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_double_colon.json +++ /dev/null @@ -1 +0,0 @@ -{"x"::"b"} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_emoji.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_emoji.json deleted file mode 100644 index cb4078eaa1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_emoji.json +++ /dev/null @@ -1 +0,0 @@ -{🇨🇭} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_garbage_at_end.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_garbage_at_end.json deleted file mode 100644 index 80c42cbadc..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_garbage_at_end.json +++ /dev/null @@ -1 +0,0 @@ -{"a":"a" 123} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_key_with_single_quotes.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_key_with_single_quotes.json deleted file mode 100644 index 77c3275996..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_key_with_single_quotes.json +++ /dev/null @@ -1 +0,0 @@ -{key: 'value'} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_lone_continuation_byte_in_key_and_trailing_comma.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_lone_continuation_byte_in_key_and_trailing_comma.json deleted file mode 100644 index aa2cb637cd..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_lone_continuation_byte_in_key_and_trailing_comma.json +++ /dev/null @@ -1 +0,0 @@ -{"":"0",} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_missing_colon.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_missing_colon.json deleted file mode 100644 index b98eff62da..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_missing_colon.json +++ /dev/null @@ -1 +0,0 @@ -{"a" b} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_missing_key.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_missing_key.json deleted file mode 100644 index b4fb0f528e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_missing_key.json +++ /dev/null @@ -1 +0,0 @@ -{:"b"} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_missing_semicolon.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_missing_semicolon.json deleted file mode 100644 index e3451384f8..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_missing_semicolon.json +++ /dev/null @@ -1 +0,0 @@ -{"a" "b"} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_missing_value.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_missing_value.json deleted file mode 100644 index 3ef538a60e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_missing_value.json +++ /dev/null @@ -1 +0,0 @@ -{"a": \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_no-colon.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_no-colon.json deleted file mode 100644 index f3797b3576..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_no-colon.json +++ /dev/null @@ -1 +0,0 @@ -{"a" \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_non_string_key.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_non_string_key.json deleted file mode 100644 index b9945b34b4..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_non_string_key.json +++ /dev/null @@ -1 +0,0 @@ -{1:1} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_non_string_key_but_huge_number_instead.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_non_string_key_but_huge_number_instead.json deleted file mode 100644 index b37fa86c0e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_non_string_key_but_huge_number_instead.json +++ /dev/null @@ -1 +0,0 @@ -{9999E9999:1} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_repeated_null_null.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_repeated_null_null.json deleted file mode 100644 index f7d2959d0d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_repeated_null_null.json +++ /dev/null @@ -1 +0,0 @@ -{null:null,null:null} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_several_trailing_commas.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_several_trailing_commas.json deleted file mode 100644 index 3c9afe8dc9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_several_trailing_commas.json +++ /dev/null @@ -1 +0,0 @@ -{"id":0,,,,,} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_single_quote.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_single_quote.json deleted file mode 100644 index e5cdf976ad..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_single_quote.json +++ /dev/null @@ -1 +0,0 @@ -{'a':0} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comma.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comma.json deleted file mode 100644 index a4b0250945..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comma.json +++ /dev/null @@ -1 +0,0 @@ -{"id":0,} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comment.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comment.json deleted file mode 100644 index a372c6553d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comment.json +++ /dev/null @@ -1 +0,0 @@ -{"a":"b"}/**/ \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comment_open.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comment_open.json deleted file mode 100644 index d557f41ca4..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comment_open.json +++ /dev/null @@ -1 +0,0 @@ -{"a":"b"}/**// \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comment_slash_open.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comment_slash_open.json deleted file mode 100644 index e335136c07..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comment_slash_open.json +++ /dev/null @@ -1 +0,0 @@ -{"a":"b"}// \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comment_slash_open_incomplete.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comment_slash_open_incomplete.json deleted file mode 100644 index d892e49f17..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_trailing_comment_slash_open_incomplete.json +++ /dev/null @@ -1 +0,0 @@ -{"a":"b"}/ \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_two_commas_in_a_row.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_two_commas_in_a_row.json deleted file mode 100644 index 7c639ae649..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_two_commas_in_a_row.json +++ /dev/null @@ -1 +0,0 @@ -{"a":"b",,"c":"d"} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_unquoted_key.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_unquoted_key.json deleted file mode 100644 index 8ba137293c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_unquoted_key.json +++ /dev/null @@ -1 +0,0 @@ -{a: "b"} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_unterminated-value.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_unterminated-value.json deleted file mode 100644 index 7fe699a6a3..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_unterminated-value.json +++ /dev/null @@ -1 +0,0 @@ -{"a":"a \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_with_single_string.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_with_single_string.json deleted file mode 100644 index d63f7fbb7e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_with_single_string.json +++ /dev/null @@ -1 +0,0 @@ -{ "foo" : "bar", "a" } \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_with_trailing_garbage.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_with_trailing_garbage.json deleted file mode 100644 index 787c8f0a8c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_object_with_trailing_garbage.json +++ /dev/null @@ -1 +0,0 @@ -{"a":"b"}# \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_single_space.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_single_space.json deleted file mode 100644 index 0519ecba6e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_single_space.json +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_1_surrogate_then_escape.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_1_surrogate_then_escape.json deleted file mode 100644 index acec66d8f4..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_1_surrogate_then_escape.json +++ /dev/null @@ -1 +0,0 @@ -["\uD800\"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_1_surrogate_then_escape_u.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_1_surrogate_then_escape_u.json deleted file mode 100644 index e834b05e96..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_1_surrogate_then_escape_u.json +++ /dev/null @@ -1 +0,0 @@ -["\uD800\u"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_1_surrogate_then_escape_u1.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_1_surrogate_then_escape_u1.json deleted file mode 100644 index a04cd34892..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_1_surrogate_then_escape_u1.json +++ /dev/null @@ -1 +0,0 @@ -["\uD800\u1"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_1_surrogate_then_escape_u1x.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_1_surrogate_then_escape_u1x.json deleted file mode 100644 index bfbd234098..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_1_surrogate_then_escape_u1x.json +++ /dev/null @@ -1 +0,0 @@ -["\uD800\u1x"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_accentuated_char_no_quotes.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_accentuated_char_no_quotes.json deleted file mode 100644 index fd6895693f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_accentuated_char_no_quotes.json +++ /dev/null @@ -1 +0,0 @@ -[é] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_backslash_00.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_backslash_00.json deleted file mode 100644 index b5bf267b5d4ee922d20cec1543b66d1530ff76cf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6 Ncma!6ieXTS1pox&0a*Y5 diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_escape_x.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_escape_x.json deleted file mode 100644 index fae291938d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_escape_x.json +++ /dev/null @@ -1 +0,0 @@ -["\x00"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_escaped_backslash_bad.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_escaped_backslash_bad.json deleted file mode 100644 index 016fcb47ef..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_escaped_backslash_bad.json +++ /dev/null @@ -1 +0,0 @@ -["\\\"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_escaped_ctrl_char_tab.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_escaped_ctrl_char_tab.json deleted file mode 100644 index f35ea382bb..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_escaped_ctrl_char_tab.json +++ /dev/null @@ -1 +0,0 @@ -["\ "] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_escaped_emoji.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_escaped_emoji.json deleted file mode 100644 index a277754213..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_escaped_emoji.json +++ /dev/null @@ -1 +0,0 @@ -["\🌀"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_incomplete_escape.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_incomplete_escape.json deleted file mode 100644 index 3415c33ca8..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_incomplete_escape.json +++ /dev/null @@ -1 +0,0 @@ -["\"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_incomplete_escaped_character.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_incomplete_escaped_character.json deleted file mode 100644 index 0f2197ea29..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_incomplete_escaped_character.json +++ /dev/null @@ -1 +0,0 @@ -["\u00A"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_incomplete_surrogate.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_incomplete_surrogate.json deleted file mode 100644 index 75504a6561..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_incomplete_surrogate.json +++ /dev/null @@ -1 +0,0 @@ -["\uD834\uDd"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_incomplete_surrogate_escape_invalid.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_incomplete_surrogate_escape_invalid.json deleted file mode 100644 index bd9656060d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_incomplete_surrogate_escape_invalid.json +++ /dev/null @@ -1 +0,0 @@ -["\uD800\uD800\x"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_invalid-utf-8-in-escape.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_invalid-utf-8-in-escape.json deleted file mode 100644 index 0c43006430..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_invalid-utf-8-in-escape.json +++ /dev/null @@ -1 +0,0 @@ -["\u"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_invalid_backslash_esc.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_invalid_backslash_esc.json deleted file mode 100644 index d1eb60921a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_invalid_backslash_esc.json +++ /dev/null @@ -1 +0,0 @@ -["\a"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_invalid_unicode_escape.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_invalid_unicode_escape.json deleted file mode 100644 index 7608cb6ba8..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_invalid_unicode_escape.json +++ /dev/null @@ -1 +0,0 @@ -["\uqqqq"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_invalid_utf8_after_escape.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_invalid_utf8_after_escape.json deleted file mode 100644 index 2f757a25b5..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_invalid_utf8_after_escape.json +++ /dev/null @@ -1 +0,0 @@ -["\"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_leading_uescaped_thinspace.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_leading_uescaped_thinspace.json deleted file mode 100644 index 7b297c6365..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_leading_uescaped_thinspace.json +++ /dev/null @@ -1 +0,0 @@ -[\u0020"asd"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_no_quotes_with_bad_escape.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_no_quotes_with_bad_escape.json deleted file mode 100644 index 01bc70abae..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_no_quotes_with_bad_escape.json +++ /dev/null @@ -1 +0,0 @@ -[\n] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_single_doublequote.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_single_doublequote.json deleted file mode 100644 index 9d68933c44..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_single_doublequote.json +++ /dev/null @@ -1 +0,0 @@ -" \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_single_quote.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_single_quote.json deleted file mode 100644 index caff239bfc..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_single_quote.json +++ /dev/null @@ -1 +0,0 @@ -['single quote'] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_single_string_no_double_quotes.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_single_string_no_double_quotes.json deleted file mode 100644 index f2ba8f84ab..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_single_string_no_double_quotes.json +++ /dev/null @@ -1 +0,0 @@ -abc \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_start_escape_unclosed.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_start_escape_unclosed.json deleted file mode 100644 index db62a46fcb..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_start_escape_unclosed.json +++ /dev/null @@ -1 +0,0 @@ -["\ \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_unescaped_crtl_char.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_unescaped_crtl_char.json deleted file mode 100644 index 9f21348071d3d736bdd469f159cea674c09237af..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7 Ocma!6N@Pe>iUj}$`2oKG diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_unescaped_newline.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_unescaped_newline.json deleted file mode 100644 index 700d360869..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_unescaped_newline.json +++ /dev/null @@ -1,2 +0,0 @@ -["new -line"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_unescaped_tab.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_unescaped_tab.json deleted file mode 100644 index 160264a2d9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_unescaped_tab.json +++ /dev/null @@ -1 +0,0 @@ -[" "] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_unicode_CapitalU.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_unicode_CapitalU.json deleted file mode 100644 index 17332bb174..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_unicode_CapitalU.json +++ /dev/null @@ -1 +0,0 @@ -"\UA66D" \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_with_trailing_garbage.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_with_trailing_garbage.json deleted file mode 100644 index efe3bd272c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_string_with_trailing_garbage.json +++ /dev/null @@ -1 +0,0 @@ -""x \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_100000_opening_arrays.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_100000_opening_arrays.json deleted file mode 100644 index a4823eecce..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_100000_opening_arrays.json +++ /dev/null @@ -1 +0,0 @@ -[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[ \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_U+2060_word_joined.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_U+2060_word_joined.json deleted file mode 100644 index 81156a6996..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_U+2060_word_joined.json +++ /dev/null @@ -1 +0,0 @@ -[⁠] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_UTF8_BOM_no_data.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_UTF8_BOM_no_data.json deleted file mode 100644 index 5f282702bb..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_UTF8_BOM_no_data.json +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_angle_bracket_..json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_angle_bracket_..json deleted file mode 100644 index a56fef0b02..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_angle_bracket_..json +++ /dev/null @@ -1 +0,0 @@ -<.> \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_angle_bracket_null.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_angle_bracket_null.json deleted file mode 100644 index 617f262549..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_angle_bracket_null.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_array_trailing_garbage.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_array_trailing_garbage.json deleted file mode 100644 index 5a745e6f3c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_array_trailing_garbage.json +++ /dev/null @@ -1 +0,0 @@ -[1]x \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_array_with_extra_array_close.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_array_with_extra_array_close.json deleted file mode 100644 index 6cfb1398d2..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_array_with_extra_array_close.json +++ /dev/null @@ -1 +0,0 @@ -[1]] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_array_with_unclosed_string.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_array_with_unclosed_string.json deleted file mode 100644 index ba6b1788b6..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_array_with_unclosed_string.json +++ /dev/null @@ -1 +0,0 @@ -["asd] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_ascii-unicode-identifier.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_ascii-unicode-identifier.json deleted file mode 100644 index ef2ab62fe7..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_ascii-unicode-identifier.json +++ /dev/null @@ -1 +0,0 @@ -aå \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_capitalized_True.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_capitalized_True.json deleted file mode 100644 index 7cd88469ab..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_capitalized_True.json +++ /dev/null @@ -1 +0,0 @@ -[True] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_close_unopened_array.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_close_unopened_array.json deleted file mode 100644 index d2af0c646a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_close_unopened_array.json +++ /dev/null @@ -1 +0,0 @@ -1] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_comma_instead_of_closing_brace.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_comma_instead_of_closing_brace.json deleted file mode 100644 index ac61b82006..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_comma_instead_of_closing_brace.json +++ /dev/null @@ -1 +0,0 @@ -{"x": true, \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_double_array.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_double_array.json deleted file mode 100644 index 058d1626e5..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_double_array.json +++ /dev/null @@ -1 +0,0 @@ -[][] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_end_array.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_end_array.json deleted file mode 100644 index 54caf60b13..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_end_array.json +++ /dev/null @@ -1 +0,0 @@ -] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_incomplete_UTF8_BOM.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_incomplete_UTF8_BOM.json deleted file mode 100644 index bfcdd514f3..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_incomplete_UTF8_BOM.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_lone-invalid-utf-8.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_lone-invalid-utf-8.json deleted file mode 100644 index 8b1296cad2..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_lone-invalid-utf-8.json +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_lone-open-bracket.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_lone-open-bracket.json deleted file mode 100644 index 8e2f0bef13..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_lone-open-bracket.json +++ /dev/null @@ -1 +0,0 @@ -[ \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_no_data.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_no_data.json deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_null-byte-outside-string.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_null-byte-outside-string.json deleted file mode 100644 index 326db14422a756e9bd39221edf37b844cb8471c7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3 Kcma!Mhy?%vaR9jh diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_number_with_trailing_garbage.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_number_with_trailing_garbage.json deleted file mode 100644 index 0746539d24..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_number_with_trailing_garbage.json +++ /dev/null @@ -1 +0,0 @@ -2@ \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_object_followed_by_closing_object.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_object_followed_by_closing_object.json deleted file mode 100644 index aa9ebaec57..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_object_followed_by_closing_object.json +++ /dev/null @@ -1 +0,0 @@ -{}} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_object_unclosed_no_value.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_object_unclosed_no_value.json deleted file mode 100644 index 17d045147f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_object_unclosed_no_value.json +++ /dev/null @@ -1 +0,0 @@ -{"": \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_object_with_comment.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_object_with_comment.json deleted file mode 100644 index ed1b569b70..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_object_with_comment.json +++ /dev/null @@ -1 +0,0 @@ -{"a":/*comment*/"b"} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_object_with_trailing_garbage.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_object_with_trailing_garbage.json deleted file mode 100644 index 9ca2336d74..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_object_with_trailing_garbage.json +++ /dev/null @@ -1 +0,0 @@ -{"a": true} "x" \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_apostrophe.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_apostrophe.json deleted file mode 100644 index 8bebe3af09..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_apostrophe.json +++ /dev/null @@ -1 +0,0 @@ -[' \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_comma.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_comma.json deleted file mode 100644 index 6295fdc36d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_comma.json +++ /dev/null @@ -1 +0,0 @@ -[, \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_object.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_object.json deleted file mode 100644 index e870445b2e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_object.json +++ /dev/null @@ -1 +0,0 @@ -[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"":[{"": diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_open_object.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_open_object.json deleted file mode 100644 index 7a63c8c57c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_open_object.json +++ /dev/null @@ -1 +0,0 @@ -[{ \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_open_string.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_open_string.json deleted file mode 100644 index 9822a6baf7..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_open_string.json +++ /dev/null @@ -1 +0,0 @@ -["a \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_string.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_string.json deleted file mode 100644 index 42a6193620..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_array_string.json +++ /dev/null @@ -1 +0,0 @@ -["a" \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object.json deleted file mode 100644 index 81750b96f9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object.json +++ /dev/null @@ -1 +0,0 @@ -{ \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_close_array.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_close_array.json deleted file mode 100644 index eebc700a10..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_close_array.json +++ /dev/null @@ -1 +0,0 @@ -{] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_comma.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_comma.json deleted file mode 100644 index 47bc9106f8..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_comma.json +++ /dev/null @@ -1 +0,0 @@ -{, \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_open_array.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_open_array.json deleted file mode 100644 index 381ede5dea..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_open_array.json +++ /dev/null @@ -1 +0,0 @@ -{[ \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_open_string.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_open_string.json deleted file mode 100644 index 328c30cd78..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_open_string.json +++ /dev/null @@ -1 +0,0 @@ -{"a \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_string_with_apostrophes.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_string_with_apostrophes.json deleted file mode 100644 index 9dba17090c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_object_string_with_apostrophes.json +++ /dev/null @@ -1 +0,0 @@ -{'a' \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_open.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_open.json deleted file mode 100644 index 841fd5f864..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_open_open.json +++ /dev/null @@ -1 +0,0 @@ -["\{["\{["\{["\{ \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_single_eacute.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_single_eacute.json deleted file mode 100644 index 92a39f398b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_single_eacute.json +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_single_star.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_single_star.json deleted file mode 100644 index f59ec20aab..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_single_star.json +++ /dev/null @@ -1 +0,0 @@ -* \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_trailing_#.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_trailing_#.json deleted file mode 100644 index 8986110875..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_trailing_#.json +++ /dev/null @@ -1 +0,0 @@ -{"a":"b"}#{} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_uescaped_LF_before_string.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_uescaped_LF_before_string.json deleted file mode 100644 index df2f0f2422..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_uescaped_LF_before_string.json +++ /dev/null @@ -1 +0,0 @@ -[\u000A""] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_array.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_array.json deleted file mode 100644 index 11209515c8..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_array.json +++ /dev/null @@ -1 +0,0 @@ -[1 \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_array_partial_null.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_array_partial_null.json deleted file mode 100644 index 0d591762c1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_array_partial_null.json +++ /dev/null @@ -1 +0,0 @@ -[ false, nul \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_array_unfinished_false.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_array_unfinished_false.json deleted file mode 100644 index a2ff8504a9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_array_unfinished_false.json +++ /dev/null @@ -1 +0,0 @@ -[ true, fals \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_array_unfinished_true.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_array_unfinished_true.json deleted file mode 100644 index 3149e8f5a7..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_array_unfinished_true.json +++ /dev/null @@ -1 +0,0 @@ -[ false, tru \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_object.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_object.json deleted file mode 100644 index 694d69dbd0..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unclosed_object.json +++ /dev/null @@ -1 +0,0 @@ -{"asd":"asd" \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unicode-identifier.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unicode-identifier.json deleted file mode 100644 index 7284aea33d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_unicode-identifier.json +++ /dev/null @@ -1 +0,0 @@ -å \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_whitespace_U+2060_word_joiner.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_whitespace_U+2060_word_joiner.json deleted file mode 100644 index 81156a6996..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_whitespace_U+2060_word_joiner.json +++ /dev/null @@ -1 +0,0 @@ -[⁠] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_whitespace_formfeed.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_whitespace_formfeed.json deleted file mode 100644 index a9ea535d1b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/n_structure_whitespace_formfeed.json +++ /dev/null @@ -1 +0,0 @@ -[ ] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_arraysWithSpaces.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_arraysWithSpaces.json deleted file mode 100644 index 582290798f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_arraysWithSpaces.json +++ /dev/null @@ -1 +0,0 @@ -[[] ] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_empty-string.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_empty-string.json deleted file mode 100644 index 93b6be2bcc..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_empty-string.json +++ /dev/null @@ -1 +0,0 @@ -[""] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_empty.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_empty.json deleted file mode 100644 index 0637a088a0..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_empty.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_ending_with_newline.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_ending_with_newline.json deleted file mode 100644 index eac5f7b46e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_ending_with_newline.json +++ /dev/null @@ -1 +0,0 @@ -["a"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_false.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_false.json deleted file mode 100644 index 67b2f07601..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_false.json +++ /dev/null @@ -1 +0,0 @@ -[false] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_heterogeneous.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_heterogeneous.json deleted file mode 100644 index d3c1e26484..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_heterogeneous.json +++ /dev/null @@ -1 +0,0 @@ -[null, 1, "1", {}] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_null.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_null.json deleted file mode 100644 index 500db4a86a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_null.json +++ /dev/null @@ -1 +0,0 @@ -[null] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_with_1_and_newline.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_with_1_and_newline.json deleted file mode 100644 index 994825500a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_with_1_and_newline.json +++ /dev/null @@ -1,2 +0,0 @@ -[1 -] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_with_leading_space.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_with_leading_space.json deleted file mode 100644 index 18bfe6422c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_with_leading_space.json +++ /dev/null @@ -1 +0,0 @@ - [1] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_with_several_null.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_with_several_null.json deleted file mode 100644 index 99f6c5d1d8..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_with_several_null.json +++ /dev/null @@ -1 +0,0 @@ -[1,null,null,null,2] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_with_trailing_space.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_with_trailing_space.json deleted file mode 100644 index de9e7a9449..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_array_with_trailing_space.json +++ /dev/null @@ -1 +0,0 @@ -[2] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number.json deleted file mode 100644 index e5f5cc3340..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number.json +++ /dev/null @@ -1 +0,0 @@ -[123e65] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_0e+1.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_0e+1.json deleted file mode 100644 index d1d3967065..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_0e+1.json +++ /dev/null @@ -1 +0,0 @@ -[0e+1] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_0e1.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_0e1.json deleted file mode 100644 index 3283a79365..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_0e1.json +++ /dev/null @@ -1 +0,0 @@ -[0e1] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_after_space.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_after_space.json deleted file mode 100644 index 623570d960..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_after_space.json +++ /dev/null @@ -1 +0,0 @@ -[ 4] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_double_close_to_zero.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_double_close_to_zero.json deleted file mode 100644 index 96555ff782..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_double_close_to_zero.json +++ /dev/null @@ -1 +0,0 @@ -[-0.000000000000000000000000000000000000000000000000000000000000000000000000000001] diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_int_with_exp.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_int_with_exp.json deleted file mode 100644 index a4ca9e754f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_int_with_exp.json +++ /dev/null @@ -1 +0,0 @@ -[20e1] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_minus_zero.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_minus_zero.json deleted file mode 100644 index 37af1312ab..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_minus_zero.json +++ /dev/null @@ -1 +0,0 @@ -[-0] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_negative_int.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_negative_int.json deleted file mode 100644 index 8e30f8bd96..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_negative_int.json +++ /dev/null @@ -1 +0,0 @@ -[-123] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_negative_one.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_negative_one.json deleted file mode 100644 index 99d21a2a0f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_negative_one.json +++ /dev/null @@ -1 +0,0 @@ -[-1] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_negative_zero.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_negative_zero.json deleted file mode 100644 index 37af1312ab..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_negative_zero.json +++ /dev/null @@ -1 +0,0 @@ -[-0] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_capital_e.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_capital_e.json deleted file mode 100644 index 6edbdfcb18..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_capital_e.json +++ /dev/null @@ -1 +0,0 @@ -[1E22] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_capital_e_neg_exp.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_capital_e_neg_exp.json deleted file mode 100644 index 0a01bd3ef4..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_capital_e_neg_exp.json +++ /dev/null @@ -1 +0,0 @@ -[1E-2] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_capital_e_pos_exp.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_capital_e_pos_exp.json deleted file mode 100644 index 5a8fc09724..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_capital_e_pos_exp.json +++ /dev/null @@ -1 +0,0 @@ -[1E+2] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_exponent.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_exponent.json deleted file mode 100644 index da2522d61a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_exponent.json +++ /dev/null @@ -1 +0,0 @@ -[123e45] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_fraction_exponent.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_fraction_exponent.json deleted file mode 100644 index 3944a7a454..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_fraction_exponent.json +++ /dev/null @@ -1 +0,0 @@ -[123.456e78] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_neg_exp.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_neg_exp.json deleted file mode 100644 index ca40d3c255..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_neg_exp.json +++ /dev/null @@ -1 +0,0 @@ -[1e-2] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_pos_exponent.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_pos_exponent.json deleted file mode 100644 index 343601d51f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_real_pos_exponent.json +++ /dev/null @@ -1 +0,0 @@ -[1e+2] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_simple_int.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_simple_int.json deleted file mode 100644 index e47f69afcf..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_simple_int.json +++ /dev/null @@ -1 +0,0 @@ -[123] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_simple_real.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_simple_real.json deleted file mode 100644 index b02878e5fc..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_number_simple_real.json +++ /dev/null @@ -1 +0,0 @@ -[123.456789] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object.json deleted file mode 100644 index 78262eda3f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object.json +++ /dev/null @@ -1 +0,0 @@ -{"asd":"sdf", "dfg":"fgh"} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_basic.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_basic.json deleted file mode 100644 index 646bbe7bb1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_basic.json +++ /dev/null @@ -1 +0,0 @@ -{"asd":"sdf"} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_duplicated_key.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_duplicated_key.json deleted file mode 100644 index bbc2e1ce43..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_duplicated_key.json +++ /dev/null @@ -1 +0,0 @@ -{"a":"b","a":"c"} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_duplicated_key_and_value.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_duplicated_key_and_value.json deleted file mode 100644 index 211581c207..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_duplicated_key_and_value.json +++ /dev/null @@ -1 +0,0 @@ -{"a":"b","a":"b"} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_empty.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_empty.json deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_empty.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_empty_key.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_empty_key.json deleted file mode 100644 index c0013d3b8b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_empty_key.json +++ /dev/null @@ -1 +0,0 @@ -{"":0} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_escaped_null_in_key.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_escaped_null_in_key.json deleted file mode 100644 index 593f0f67f9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_escaped_null_in_key.json +++ /dev/null @@ -1 +0,0 @@ -{"foo\u0000bar": 42} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_extreme_numbers.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_extreme_numbers.json deleted file mode 100644 index a0d3531c32..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_extreme_numbers.json +++ /dev/null @@ -1 +0,0 @@ -{ "min": -1.0e+28, "max": 1.0e+28 } \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_long_strings.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_long_strings.json deleted file mode 100644 index bdc4a08719..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_long_strings.json +++ /dev/null @@ -1 +0,0 @@ -{"x":[{"id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"}], "id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_simple.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_simple.json deleted file mode 100644 index dacac917fb..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_simple.json +++ /dev/null @@ -1 +0,0 @@ -{"a":[]} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_string_unicode.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_string_unicode.json deleted file mode 100644 index 8effdb297c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_string_unicode.json +++ /dev/null @@ -1 +0,0 @@ -{"title":"\u041f\u043e\u043b\u0442\u043e\u0440\u0430 \u0417\u0435\u043c\u043b\u0435\u043a\u043e\u043f\u0430" } \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_with_newlines.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_with_newlines.json deleted file mode 100644 index 246ec6b34d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_object_with_newlines.json +++ /dev/null @@ -1,3 +0,0 @@ -{ -"a": "b" -} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_1_2_3_bytes_UTF-8_sequences.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_1_2_3_bytes_UTF-8_sequences.json deleted file mode 100644 index 9967ddeb8b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_1_2_3_bytes_UTF-8_sequences.json +++ /dev/null @@ -1 +0,0 @@ -["\u0060\u012a\u12AB"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_accepted_surrogate_pair.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_accepted_surrogate_pair.json deleted file mode 100644 index 996875cc8c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_accepted_surrogate_pair.json +++ /dev/null @@ -1 +0,0 @@ -["\uD801\udc37"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_accepted_surrogate_pairs.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_accepted_surrogate_pairs.json deleted file mode 100644 index 3401021ece..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_accepted_surrogate_pairs.json +++ /dev/null @@ -1 +0,0 @@ -["\ud83d\ude39\ud83d\udc8d"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_allowed_escapes.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_allowed_escapes.json deleted file mode 100644 index 7f495532fb..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_allowed_escapes.json +++ /dev/null @@ -1 +0,0 @@ -["\"\\\/\b\f\n\r\t"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_backslash_and_u_escaped_zero.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_backslash_and_u_escaped_zero.json deleted file mode 100644 index d4439eda73..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_backslash_and_u_escaped_zero.json +++ /dev/null @@ -1 +0,0 @@ -["\\u0000"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_backslash_doublequotes.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_backslash_doublequotes.json deleted file mode 100644 index ae03243b67..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_backslash_doublequotes.json +++ /dev/null @@ -1 +0,0 @@ -["\""] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_comments.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_comments.json deleted file mode 100644 index 2260c20c2f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_comments.json +++ /dev/null @@ -1 +0,0 @@ -["a/*b*/c/*d//e"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_double_escape_a.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_double_escape_a.json deleted file mode 100644 index 6715d6f404..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_double_escape_a.json +++ /dev/null @@ -1 +0,0 @@ -["\\a"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_double_escape_n.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_double_escape_n.json deleted file mode 100644 index 44ca56c4d9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_double_escape_n.json +++ /dev/null @@ -1 +0,0 @@ -["\\n"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_escaped_control_character.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_escaped_control_character.json deleted file mode 100644 index 5b014a9c25..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_escaped_control_character.json +++ /dev/null @@ -1 +0,0 @@ -["\u0012"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_escaped_noncharacter.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_escaped_noncharacter.json deleted file mode 100644 index 2ff52e2c50..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_escaped_noncharacter.json +++ /dev/null @@ -1 +0,0 @@ -["\uFFFF"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_in_array.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_in_array.json deleted file mode 100644 index 21d7ae4cd8..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_in_array.json +++ /dev/null @@ -1 +0,0 @@ -["asd"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_in_array_with_leading_space.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_in_array_with_leading_space.json deleted file mode 100644 index 9e1887c1e4..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_in_array_with_leading_space.json +++ /dev/null @@ -1 +0,0 @@ -[ "asd"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_last_surrogates_1_and_2.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_last_surrogates_1_and_2.json deleted file mode 100644 index 3919cef765..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_last_surrogates_1_and_2.json +++ /dev/null @@ -1 +0,0 @@ -["\uDBFF\uDFFF"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_nbsp_uescaped.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_nbsp_uescaped.json deleted file mode 100644 index 2085ab1a1c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_nbsp_uescaped.json +++ /dev/null @@ -1 +0,0 @@ -["new\u00A0line"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_nonCharacterInUTF-8_U+10FFFF.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_nonCharacterInUTF-8_U+10FFFF.json deleted file mode 100644 index 059e4d9dd0..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_nonCharacterInUTF-8_U+10FFFF.json +++ /dev/null @@ -1 +0,0 @@ -["􏿿"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_nonCharacterInUTF-8_U+FFFF.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_nonCharacterInUTF-8_U+FFFF.json deleted file mode 100644 index 4c913bd41a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_nonCharacterInUTF-8_U+FFFF.json +++ /dev/null @@ -1 +0,0 @@ -["￿"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_null_escape.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_null_escape.json deleted file mode 100644 index c1ad844043..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_null_escape.json +++ /dev/null @@ -1 +0,0 @@ -["\u0000"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_one-byte-utf-8.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_one-byte-utf-8.json deleted file mode 100644 index 157185923a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_one-byte-utf-8.json +++ /dev/null @@ -1 +0,0 @@ -["\u002c"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_pi.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_pi.json deleted file mode 100644 index 9df11ae88b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_pi.json +++ /dev/null @@ -1 +0,0 @@ -["π"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_reservedCharacterInUTF-8_U+1BFFF.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_reservedCharacterInUTF-8_U+1BFFF.json deleted file mode 100644 index 10a33a1717..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_reservedCharacterInUTF-8_U+1BFFF.json +++ /dev/null @@ -1 +0,0 @@ -["𛿿"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_simple_ascii.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_simple_ascii.json deleted file mode 100644 index 8cadf7d051..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_simple_ascii.json +++ /dev/null @@ -1 +0,0 @@ -["asd "] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_space.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_space.json deleted file mode 100644 index efd782cc32..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_space.json +++ /dev/null @@ -1 +0,0 @@ -" " \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF.json deleted file mode 100644 index 7620b66559..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF.json +++ /dev/null @@ -1 +0,0 @@ -["\uD834\uDd1e"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_three-byte-utf-8.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_three-byte-utf-8.json deleted file mode 100644 index 108f1d67df..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_three-byte-utf-8.json +++ /dev/null @@ -1 +0,0 @@ -["\u0821"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_two-byte-utf-8.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_two-byte-utf-8.json deleted file mode 100644 index 461503c310..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_two-byte-utf-8.json +++ /dev/null @@ -1 +0,0 @@ -["\u0123"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_u+2028_line_sep.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_u+2028_line_sep.json deleted file mode 100644 index 897b6021af..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_u+2028_line_sep.json +++ /dev/null @@ -1 +0,0 @@ -["
"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_u+2029_par_sep.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_u+2029_par_sep.json deleted file mode 100644 index 8cd998c89e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_u+2029_par_sep.json +++ /dev/null @@ -1 +0,0 @@ -["
"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_uEscape.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_uEscape.json deleted file mode 100644 index f7b41a02fa..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_uEscape.json +++ /dev/null @@ -1 +0,0 @@ -["\u0061\u30af\u30EA\u30b9"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_uescaped_newline.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_uescaped_newline.json deleted file mode 100644 index 3a5a220b69..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_uescaped_newline.json +++ /dev/null @@ -1 +0,0 @@ -["new\u000Aline"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unescaped_char_delete.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unescaped_char_delete.json deleted file mode 100644 index 7d064f4987..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unescaped_char_delete.json +++ /dev/null @@ -1 +0,0 @@ -[""] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode.json deleted file mode 100644 index 3598095b79..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode.json +++ /dev/null @@ -1 +0,0 @@ -["\uA66D"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicodeEscapedBackslash.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicodeEscapedBackslash.json deleted file mode 100644 index 0bb3b51e7e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicodeEscapedBackslash.json +++ /dev/null @@ -1 +0,0 @@ -["\u005C"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_2.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_2.json deleted file mode 100644 index a7dcb97683..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_2.json +++ /dev/null @@ -1 +0,0 @@ -["⍂㈴⍂"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+10FFFE_nonchar.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+10FFFE_nonchar.json deleted file mode 100644 index 9a8370b96a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+10FFFE_nonchar.json +++ /dev/null @@ -1 +0,0 @@ -["\uDBFF\uDFFE"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+1FFFE_nonchar.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+1FFFE_nonchar.json deleted file mode 100644 index c51f8ae45c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+1FFFE_nonchar.json +++ /dev/null @@ -1 +0,0 @@ -["\uD83F\uDFFE"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+200B_ZERO_WIDTH_SPACE.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+200B_ZERO_WIDTH_SPACE.json deleted file mode 100644 index 626d5f8157..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+200B_ZERO_WIDTH_SPACE.json +++ /dev/null @@ -1 +0,0 @@ -["\u200B"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+2064_invisible_plus.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+2064_invisible_plus.json deleted file mode 100644 index 1e23972c65..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+2064_invisible_plus.json +++ /dev/null @@ -1 +0,0 @@ -["\u2064"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+FDD0_nonchar.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+FDD0_nonchar.json deleted file mode 100644 index 18ef151b4f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+FDD0_nonchar.json +++ /dev/null @@ -1 +0,0 @@ -["\uFDD0"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+FFFE_nonchar.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+FFFE_nonchar.json deleted file mode 100644 index 13d261fdad..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_U+FFFE_nonchar.json +++ /dev/null @@ -1 +0,0 @@ -["\uFFFE"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_escaped_double_quote.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_escaped_double_quote.json deleted file mode 100644 index 4e6257856d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_unicode_escaped_double_quote.json +++ /dev/null @@ -1 +0,0 @@ -["\u0022"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_utf8.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_utf8.json deleted file mode 100644 index 40878435f9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_utf8.json +++ /dev/null @@ -1 +0,0 @@ -["€𝄞"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_with_del_character.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_with_del_character.json deleted file mode 100644 index 8bd24907d9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_string_with_del_character.json +++ /dev/null @@ -1 +0,0 @@ -["aa"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_false.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_false.json deleted file mode 100644 index 02e4a84d62..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_false.json +++ /dev/null @@ -1 +0,0 @@ -false \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_int.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_int.json deleted file mode 100644 index f70d7bba4a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_int.json +++ /dev/null @@ -1 +0,0 @@ -42 \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_negative_real.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_negative_real.json deleted file mode 100644 index b5135a207d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_negative_real.json +++ /dev/null @@ -1 +0,0 @@ --0.1 \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_null.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_null.json deleted file mode 100644 index ec747fa47d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_null.json +++ /dev/null @@ -1 +0,0 @@ -null \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_string.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_string.json deleted file mode 100644 index b6e982ca96..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_string.json +++ /dev/null @@ -1 +0,0 @@ -"asd" \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_true.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_true.json deleted file mode 100644 index f32a5804e2..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_lonely_true.json +++ /dev/null @@ -1 +0,0 @@ -true \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_string_empty.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_string_empty.json deleted file mode 100644 index 3cc762b550..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_string_empty.json +++ /dev/null @@ -1 +0,0 @@ -"" \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_trailing_newline.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_trailing_newline.json deleted file mode 100644 index 0c3426d4c2..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_trailing_newline.json +++ /dev/null @@ -1 +0,0 @@ -["a"] diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_true_in_array.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_true_in_array.json deleted file mode 100644 index de601e305f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_true_in_array.json +++ /dev/null @@ -1 +0,0 @@ -[true] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_whitespace_array.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_whitespace_array.json deleted file mode 100644 index 2bedf7f2de..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSONTestSuite/y_structure_whitespace_array.json +++ /dev/null @@ -1 +0,0 @@ - [] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/Acknowledgement.txt b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/Acknowledgement.txt deleted file mode 100644 index 5f19a6715f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/Acknowledgement.txt +++ /dev/null @@ -1,3 +0,0 @@ -These files are from the JSON_checker test suite - -http://www.json.org/JSON_checker/ diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail1.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail1.json deleted file mode 100644 index 6216b865f1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail1.json +++ /dev/null @@ -1 +0,0 @@ -"A JSON payload should be an object or array, not a string." \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail10.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail10.json deleted file mode 100644 index 5d8c0047bd..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail10.json +++ /dev/null @@ -1 +0,0 @@ -{"Extra value after close": true} "misplaced quoted value" \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail11.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail11.json deleted file mode 100644 index 76eb95b458..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail11.json +++ /dev/null @@ -1 +0,0 @@ -{"Illegal expression": 1 + 2} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail12.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail12.json deleted file mode 100644 index 77580a4522..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail12.json +++ /dev/null @@ -1 +0,0 @@ -{"Illegal invocation": alert()} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail13.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail13.json deleted file mode 100644 index 379406b59b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail13.json +++ /dev/null @@ -1 +0,0 @@ -{"Numbers cannot have leading zeroes": 013} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail14.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail14.json deleted file mode 100644 index 0ed366b38a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail14.json +++ /dev/null @@ -1 +0,0 @@ -{"Numbers cannot be hex": 0x14} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail15.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail15.json deleted file mode 100644 index fc8376b605..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail15.json +++ /dev/null @@ -1 +0,0 @@ -["Illegal backslash escape: \x15"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail16.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail16.json deleted file mode 100644 index 3fe21d4b53..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail16.json +++ /dev/null @@ -1 +0,0 @@ -[\naked] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail17.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail17.json deleted file mode 100644 index 62b9214aed..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail17.json +++ /dev/null @@ -1 +0,0 @@ -["Illegal backslash escape: \017"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail18.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail18.json deleted file mode 100644 index edac92716f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail18.json +++ /dev/null @@ -1 +0,0 @@ -[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail19.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail19.json deleted file mode 100644 index 3b9c46fa9a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail19.json +++ /dev/null @@ -1 +0,0 @@ -{"Missing colon" null} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail2.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail2.json deleted file mode 100644 index 6b7c11e5a5..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail2.json +++ /dev/null @@ -1 +0,0 @@ -["Unclosed array" \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail20.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail20.json deleted file mode 100644 index 27c1af3e72..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail20.json +++ /dev/null @@ -1 +0,0 @@ -{"Double colon":: null} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail21.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail21.json deleted file mode 100644 index 62474573b2..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail21.json +++ /dev/null @@ -1 +0,0 @@ -{"Comma instead of colon", null} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail22.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail22.json deleted file mode 100644 index a7752581bc..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail22.json +++ /dev/null @@ -1 +0,0 @@ -["Colon instead of comma": false] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail23.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail23.json deleted file mode 100644 index 494add1ca1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail23.json +++ /dev/null @@ -1 +0,0 @@ -["Bad value", truth] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail24.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail24.json deleted file mode 100644 index caff239bfc..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail24.json +++ /dev/null @@ -1 +0,0 @@ -['single quote'] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail25.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail25.json deleted file mode 100644 index 8b7ad23e01..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail25.json +++ /dev/null @@ -1 +0,0 @@ -[" tab character in string "] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail26.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail26.json deleted file mode 100644 index 845d26a6a5..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail26.json +++ /dev/null @@ -1 +0,0 @@ -["tab\ character\ in\ string\ "] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail27.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail27.json deleted file mode 100644 index 6b01a2ca4a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail27.json +++ /dev/null @@ -1,2 +0,0 @@ -["line -break"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail28.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail28.json deleted file mode 100644 index 621a0101c6..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail28.json +++ /dev/null @@ -1,2 +0,0 @@ -["line\ -break"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail29.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail29.json deleted file mode 100644 index 47ec421bb6..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail29.json +++ /dev/null @@ -1 +0,0 @@ -[0e] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail3.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail3.json deleted file mode 100644 index 168c81eb78..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail3.json +++ /dev/null @@ -1 +0,0 @@ -{unquoted_key: "keys must be quoted"} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail30.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail30.json deleted file mode 100644 index 8ab0bc4b8b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail30.json +++ /dev/null @@ -1 +0,0 @@ -[0e+] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail31.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail31.json deleted file mode 100644 index 1cce602b51..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail31.json +++ /dev/null @@ -1 +0,0 @@ -[0e+-1] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail32.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail32.json deleted file mode 100644 index 45cba7396f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail32.json +++ /dev/null @@ -1 +0,0 @@ -{"Comma instead if closing brace": true, \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail33.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail33.json deleted file mode 100644 index ca5eb19dc9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail33.json +++ /dev/null @@ -1 +0,0 @@ -["mismatch"} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail4.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail4.json deleted file mode 100644 index 9de168bf34..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail4.json +++ /dev/null @@ -1 +0,0 @@ -["extra comma",] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail5.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail5.json deleted file mode 100644 index ddf3ce3d24..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail5.json +++ /dev/null @@ -1 +0,0 @@ -["double extra comma",,] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail6.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail6.json deleted file mode 100644 index ed91580e1b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail6.json +++ /dev/null @@ -1 +0,0 @@ -[ , "<-- missing value"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail7.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail7.json deleted file mode 100644 index 8a96af3e4e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail7.json +++ /dev/null @@ -1 +0,0 @@ -["Comma after the close"], \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail8.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail8.json deleted file mode 100644 index b28479c6ec..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail8.json +++ /dev/null @@ -1 +0,0 @@ -["Extra close"]] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail9.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail9.json deleted file mode 100644 index 5815574f36..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/fail9.json +++ /dev/null @@ -1 +0,0 @@ -{"Extra comma": true,} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/pass1.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/pass1.json deleted file mode 100644 index 70e2685436..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/pass1.json +++ /dev/null @@ -1,58 +0,0 @@ -[ - "JSON Test Pattern pass1", - {"object with 1 member":["array with 1 element"]}, - {}, - [], - -42, - true, - false, - null, - { - "integer": 1234567890, - "real": -9876.543210, - "e": 0.123456789e-12, - "E": 1.234567890E+34, - "": 23456789012E66, - "zero": 0, - "one": 1, - "space": " ", - "quote": "\"", - "backslash": "\\", - "controls": "\b\f\n\r\t", - "slash": "/ & \/", - "alpha": "abcdefghijklmnopqrstuvwyz", - "ALPHA": "ABCDEFGHIJKLMNOPQRSTUVWYZ", - "digit": "0123456789", - "0123456789": "digit", - "special": "`1~!@#$%^&*()_+-={':[,]}|;.?", - "hex": "\u0123\u4567\u89AB\uCDEF\uabcd\uef4A", - "true": true, - "false": false, - "null": null, - "array":[ ], - "object":{ }, - "address": "50 St. James Street", - "url": "http://www.JSON.org/", - "comment": "// /* */": " ", - " s p a c e d " :[1,2 , 3 - -, - -4 , 5 , 6 ,7 ],"compact":[1,2,3,4,5,6,7], - "jsontext": "{\"object with 1 member\":[\"array with 1 element\"]}", - "quotes": "" \u0022 %22 0x22 034 "", - "\/\\\"\uCAFE\uBABE\uAB98\uFCDE\ubcda\uef4A\b\f\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?" -: "A key can be any string" - }, - 0.5 ,98.6 -, -99.44 -, - -1066, -1e1, -0.1e1, -1e-1, -1e00,2e+00,2e-00 -,"rosebud"] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/pass2.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/pass2.json deleted file mode 100644 index d3c63c7ad8..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/pass2.json +++ /dev/null @@ -1 +0,0 @@ -[[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/pass3.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/pass3.json deleted file mode 100644 index 4528d51f1a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/JSON_checker/pass3.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "JSON Test Pattern pass3": { - "The outermost value": "must be an object or array.", - "In this test": "It is an object." - } -} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/address-book.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/address-book.json deleted file mode 100644 index c3d5d98369..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/address-book.json +++ /dev/null @@ -1,13 +0,0 @@ - { - "address-book" : - [ - { - "name":"Jane Roe", - "email":"jane.roe@example.com" - }, - { - "name":"John", - "email" : "john.doe@example.com" - } - ] - } diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/countries.csv b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/countries.csv deleted file mode 100644 index 750b1937d7..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/countries.csv +++ /dev/null @@ -1,5 +0,0 @@ -country_code,name -ABW,ARUBA -ATF,"FRENCH SOUTHERN TERRITORIES, D.R. OF" -VUT,VANUATU -WLF,WALLIS & FUTUNA ISLANDS diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/countries.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/countries.json deleted file mode 100644 index 29629e4253..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/countries.json +++ /dev/null @@ -1,7 +0,0 @@ -[ - ["country_code","name"], - ["ABW","ARUBA"], - ["ATF","FRENCH SOUTHERN TERRITORIES, D.R. OF"], - ["VUT","VANUATU"], - ["WLF","WALLIS & FUTUNA ISLANDS"] -] diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/cyrillic.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/cyrillic.json deleted file mode 100644 index 067839e9d4..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/cyrillic.json +++ /dev/null @@ -1,731 +0,0 @@ -{ - "Xml": { - "SLMS": [ - { - "-idSLMS": "1", - "LMS": [ - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - } - ] - }, - { - "-idSLMS": "2", - "LMS": [ - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - } - ] - }, - { - "-idSLMS": "3", - "LMS": [ - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - } - ] - }, - { - "-idSLMS": "4", - "LMS": [ - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - } - ] - }, - { - "-idSLMS": "5", - "LMS": [ - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - } - ] - }, - { - "-idSLMS": "6", - "LMS": [ - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - } - ] - }, - { - "-idSLMS": "7", - "LMS": [ - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - } - ] - }, - { - "-idSLMS": "8", - "LMS": [ - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - } - ] - }, - { - "-idSLMS": "9", - "LMS": [ - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - } - ] - }, - { - "-idSLMS": "10", - "LMS": [ - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - } - ] - }, - { - "-idSLMS": "11", - "LMS": [ - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - }, - { - "PostTh": "Абвгдеёжзик лмнопрстуфхцчшщъ(Кккккккк ккккккк кккккккк)", - "PostSh": "Абвгде.жзиклм-но(Кккккккк к.к.)", - "KodeMod": "$A", - "PostTabNum": "0000000004" - } - ] - }, - { - "-idSLMS": "12", - "LMS": [ - { - "PostTh": "Департамент ведомственного финансового контроля и аудита МО РФ", - "PostSh": "ДВФКиА МО", - "KodeMod": "$$", - "PostTabNum": "0000000001" - }, - { - "PostTh": "Консультант (Мирошниченко Ирина Владимировна)", - "PostSh": "Консультант(Мирошниченко И.В)", - "KodeMod": "$5", - "PostTabNum": "0000000015" - }, - { - "PostTh": "Главный специалист-эксперт (Власова Екатерина Генадьевнна)", - "PostSh": "Гл.спец-эксперт(Власова Е.Г)", - "KodeMod": "$7", - "PostTabNum": "0000000013" - } - ] - }, - { - "-idSLMS": "13", - "LMS": [ - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - }, - { - "PostTh": "Учёт обращений граждан (Еееееее еееее ееееее)", - "PostSh": "Учет обращений гр.(Ееееееее Е.Е)", - "KodeMod": "$d", - "PostTabNum": "0000000033" - } - ] - } - ] - } -} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/employees.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/employees.json deleted file mode 100644 index 7fff289e9b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/employees.json +++ /dev/null @@ -1,30 +0,0 @@ -[ - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee Dependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - } -] diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/employees.txt b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/employees.txt deleted file mode 100644 index 553ea8af00..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/employees.txt +++ /dev/null @@ -1,6 +0,0 @@ -employee-no employee-name dept salary note -00000001 Smith, Matthew sales 150,000.00 -00000002 Brown, Sarah sales 89,000.00 -00000003 Oberc, Scott finance 110,000.00 -00000004 Scott, Colette sales 75,000.00 """Exemplary"" employee -Dependable, trustworthy" diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/json-exception-1.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/json-exception-1.json deleted file mode 100644 index ccfa766300..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/json-exception-1.json +++ /dev/null @@ -1,20 +0,0 @@ - { - // Members - "members" : [ - { - "first_name":"Jane", - "last_name":"Roe", - "events_attended":10, - "accept_waiver_of_liability" : true - }, - /* Missing - *left* - brace */ - - "first_name":"John", - "last_name":"Doe", - "events_attended":2, - "accept_waiver_of_liability" : true - } - ] - } diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/json-exception-2.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/json-exception-2.json deleted file mode 100644 index 7673749abf..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/json-exception-2.json +++ /dev/null @@ -1,18 +0,0 @@ - { - // Members - "members" : [ - { - "first_name":"Jane", - "last_name":"Roe", - "events_attended":10, - "accept_waiver_of_liability" : true - }, - { - "first_name":"John", - "last_name":"Doe", - "events_attended":2, - "accept_waiver_of_liability" : true - /* Missing *right* - brace */ - ] - } diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/json-multiline-comment.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/json-multiline-comment.json deleted file mode 100644 index 3f481aa256..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/json-multiline-comment.json +++ /dev/null @@ -1,2524 +0,0 @@ -[ -/* - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - }, - { - "dept":"sales", - "employee-name":"Smith, Matthew", - "employee-no":"00000001", - "note":"", - "salary":"150,000.00" - }, - { - "dept":"sales", - "employee-name":"Brown, Sarah", - "employee-no":"00000002", - "note":"", - "salary":"89,000.00" - }, - { - "dept":"finance", - "employee-name":"Oberc, Scott", - "employee-no":"00000003", - "salary":"110,000.00" - }, - { - "dept":"sales", - "employee-name":"Scott, Colette", - "employee-no":"00000004", - "note":"\"Exemplary\" employee\nDependable, trustworthy", - "comment":"Team player", - "salary":"75,000.00" - } -*/ -] diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/locations.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/locations.json deleted file mode 100644 index ec7c8b05fd..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/locations.json +++ /dev/null @@ -1,17 +0,0 @@ - { - // Members - "members" : [ - { - "first_name":"Jane", - "last_name":"Roe", - "events_attended":10, - "accept_waiver_of_liability" : true - }, - /* Error - missing *left* brace */ - "first_name":"John", - "last_name":"Doe", - "events_attended":2, - "accept_waiver_of_liability" : true - } - ] - } diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/members.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/members.json deleted file mode 100644 index 02808f47a9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/members.json +++ /dev/null @@ -1,17 +0,0 @@ - { - // Members - "members" : [ - { - "first_name":"Jane", - "last_name":"Roe", - "events_attended":10, - "accept_waiver_of_liability" : true - }, - { - "first_name":"John", - "last_name":"Doe", - "events_attended":2, - "accept_waiver_of_liability" : true - } - ] - } diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/persons.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/persons.json deleted file mode 100644 index b4685503c1..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/input/persons.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "persons" : ["John","David"] -} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/output/address-book-new.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/output/address-book-new.json deleted file mode 100644 index f0cc264027..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/output/address-book-new.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "address-book": [ - { - "first-name": "Jane", - "last-name": "Roe", - "email2": "jane.roe@example.com" - }, - { - "first-name": "John", - "email2": "john.doe@example.com" - } - ] -} \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/output/store.cbor b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/output/store.cbor deleted file mode 100644 index 03e6321d73..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/output/store.cbor +++ /dev/null @@ -1 +0,0 @@ -kapplicationfhikinghreputonsiassertionlstrong-hikereratediMarilyn CeraterxHikingAsylum.example.comfrating? \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/output/test.json b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/output/test.json deleted file mode 100644 index eb66f851e4..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/output/test.json +++ /dev/null @@ -1,1502 +0,0 @@ -[ - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - }, - { - "person": { - "first_name": "john", - "last_name": "doe", - "birthdate": "1998-05-13", - "sex": "m", - "salary": 70000, - "interests": ["Reading","Mountain biking","Hacking"], - "favorites": { - "color": "blue", - "sport": "soccer", - "food": "spaghetti" - } - } - } -] \ No newline at end of file diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/JSONTestSuite_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/JSONTestSuite_tests.cpp deleted file mode 100644 index 4650aa16e6..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/JSONTestSuite_tests.cpp +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2013-2018 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if defined(_MSC_VER) && _MSC_VER >= 1900 -#include -namespace fs = std::experimental::filesystem; -#else -//#include -//namespace fs = std::filesystem; -#endif - -using namespace jsoncons; - -#if defined(_MSC_VER) && _MSC_VER >= 1900 -TEST_CASE("JSON Parsing Test Suite") -{ - SECTION("Expected success") - { - std::string path = "./input/JSONTestSuite"; - for (auto& p : fs::directory_iterator(path)) - { - if (fs::exists(p) && fs::is_regular_file(p) && p.path().extension() == ".json" && p.path().filename().c_str()[0] == 'y') - { - std::ifstream is(p.path().c_str()); - strict_parse_error_handler err_handler; - json_reader reader(is, err_handler); - std::error_code ec; - reader.read(ec); - if (ec) - { - std::cout << p.path().filename().string() << " failed, expected success\n"; - } - CHECK_FALSE(ec); - } - } - } - SECTION("Expected failure") - { - std::string path = "./input/JSONTestSuite"; - for (auto& p : fs::directory_iterator(path)) - { - if (fs::exists(p) && fs::is_regular_file(p) && p.path().extension() == ".json" && p.path().filename().c_str()[0] == 'n') - { - std::ifstream is(p.path().c_str()); - strict_parse_error_handler err_handler; - json_reader reader(is, err_handler); - std::error_code ec; - reader.read(ec); - if (!ec) - { - std::cout << p.path().filename().string() << " succeeded, expected failure\n"; - } - CHECK(ec); - } - } - } -} -#endif - - - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/bignum_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/bignum_tests.cpp deleted file mode 100644 index 927ccf087a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/bignum_tests.cpp +++ /dev/null @@ -1,329 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" -#endif -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("test_positive_bignum") -{ - std::string expected = "18446744073709551616"; - std::vector v = {1,0,0,0,0,0,0,0,0}; - bignum x(1,v.data(),v.size()); - - std::string sx; - x.dump(sx); - CHECK(sx == expected); - - bignum y(x); - std::string sy; - y.dump(sy); - CHECK(sy == expected); - - bignum z; - z = x; - std::string sz; - y.dump(sz); - CHECK(sz == expected); - - SECTION("dump_hex_string") - { - std::string exp = "10000000000000000"; - std::string s; - x.dump_hex_string(s); - CHECK(s == exp); - } - -} - -TEST_CASE("bignums are equal") -{ - std::string s = "18446744073709551616"; - bignum x(s); - bignum y(s); - - bool test = x == y; - CHECK(test); -} - -TEST_CASE("test_negative_bignum") -{ - std::string expected = "-18446744073709551617"; - std::vector b = {0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; - bignum x(-1,b.data(),b.size()); - - std::string sx; - x.dump(sx); - CHECK(sx == expected); - - bignum y(x); - std::string sy; - y.dump(sy); - CHECK(sy == expected); - - bignum z; - z = x; - std::string sz; - y.dump(sz); - CHECK(sz == expected); - - int signum; - std::vector v; - x.dump(signum, v); - - REQUIRE(v.size() == b.size()); - for (size_t i = 0; i < v.size(); ++i) - { - REQUIRE(v[i] == b[i]); - } - - SECTION("dump_hex_string") - { - std::string exp = "-10000000000000001"; - std::string s; - x.dump_hex_string(s); - //std::cout << "bignum: " << expected << ", s: " << s << "\n"; - CHECK(s == exp); - } -} - -TEST_CASE("test_longlong") -{ - long long n = (std::numeric_limits::max)(); - - bignum val = n; - - //std::cout << "long long " << n << " == " << val << std::endl; - //std::cout << val.to_string(16) << std::endl; -} - -TEST_CASE("test_bignum2") -{ - std::string v = "10000000000000000"; - bignum val(v.data()); - - //std::cout << val << std::endl; -} - -TEST_CASE("test_logical_operations") -{ - bignum x( "888888888888888888" ); - bignum y( "888888888888888888" ); - - bignum z = x & y; - - bool test = z == x; - CHECK(test); -} - -TEST_CASE("test_addition") -{ - bignum x( "4444444444444444444444444444444" ); - bignum y( "4444444444444444444444444444444" ); - bignum a( "8888888888888888888888888888888" ); - - bignum z = x + y; - bool test = z == a; - CHECK(test); -} - -TEST_CASE("test_multiplication") -{ - bignum x( "4444444444444444444444444444444" ); - bignum a( "8888888888888888888888888888888" ); - - bignum z = 2*x; - bool test = z == a; - CHECK(test); - - z = x*2; - - test = z == a; - CHECK(test); -} - -TEST_CASE("test_conversion_0") -{ - bignum x(1, {}); - - json j(x); - - bignum y = j.as(); - CHECK(bool(x == y)); - - std::string s; - y.dump(s); - - CHECK(s == "0"); -} - -TEST_CASE("test_traits1") -{ - bignum x(1, {0x01,0x00}); - - json j(x); - - bignum y = j.as(); - CHECK(bool(x == y)); - - std::string s; - y.dump(s); - - CHECK(s == "256"); -} - -TEST_CASE("test_traits2") -{ - bignum x(1, {0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}); - - json j(x); - - bignum y = j.as(); - CHECK(bool(x == y)); - - std::string s; - y.dump(s); - - CHECK(s == "18446744073709551616"); -} - -TEST_CASE("test_traits3") -{ - bignum x(-1, {0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}); - - int signum; - std::vector v; - x.dump(signum,v); - - REQUIRE(signum == -1); - //for (auto c : v) - //{ - // //std::cout << std::hex << (int)c; - //} - //std::cout << std::endl; - - json j(x); - - bignum y = j.as(); - CHECK(bool(x == y)); - - std::string s; - y.dump(s); - - CHECK(s == "-18446744073709551617"); -} - -TEST_CASE("test shift left") -{ - SECTION("n << 1") - { - bignum n("1"); - bignum x = n << 1; - std::string s; - x.dump(s); - CHECK(s == "2"); - } - SECTION("n << 100") - { - bignum n(1); - bignum x = n << 100; - std::string s; - x.dump(s); - CHECK(s == "1267650600228229401496703205376"); - } - SECTION("n << 100, += 1") - { - bignum n(1); - bignum x = n << 100; - x += 1; - std::string s; - x.dump(s); - CHECK(s == "1267650600228229401496703205377"); - } -} - -TEST_CASE("times 10") -{ - SECTION("1") - { - bignum n("1234"); - bignum m = n * 10; - std::string s; - m.dump(s); - CHECK(s == "12340"); - } - SECTION("31") - { - std::string expected("1234"); - bignum n(expected); - - for (size_t i = 0; i < 31; ++i) - { - n *= (uint64_t)10; - expected.push_back('0'); - } - std::string s; - n.dump(s); - CHECK(s == expected); - //std::cout << "x31: " << s << "\n"; - } - SECTION("32") - { - std::string expected("1234"); - bignum n(expected); - for (size_t i = 0; i < 32; ++i) - { - n *= (uint64_t)10; - expected.push_back('0'); - } - std::string s; - n.dump(s); - CHECK(s == expected); - //std::cout << "x31: " << s << "\n"; - } -} - - -TEST_CASE("bignum div") -{ -#if defined(_MSC_VER) && _MSC_VER >= 1910 - SECTION("bignum") - { - bignum big_pos = bignum("18364494661702398480"); - bignum small_pos = bignum("65535"); - bignum res_pos = bignum("280224226164681"); - bignum big_neg = -big_pos; - bignum small_neg = -small_pos; - bignum res_neg = -res_pos; - - CHECK((big_neg / big_neg) == bignum(1)); - CHECK((big_neg / small_neg) == res_pos); - CHECK((big_neg / small_pos) == res_neg); - CHECK((big_neg / big_pos) == bignum(-1)); - - CHECK((small_neg / big_neg) == bignum(0)); - CHECK((small_neg / small_neg) == bignum(1)); - CHECK((small_neg / small_pos) == bignum(-1)); - CHECK((small_neg / big_pos) == bignum(0)); - - CHECK((small_pos / big_neg) == bignum(0)); - CHECK((small_pos / small_neg) == bignum(-1)); - CHECK((small_pos / small_pos) == bignum(1)); - CHECK((small_pos / big_pos) == bignum(0)); - - CHECK((big_pos / big_neg) == bignum(-1)); - CHECK((big_pos / small_neg) == res_neg); - CHECK((big_pos / small_pos) == res_pos); - CHECK((big_pos / big_pos) == bignum(1)); - } -#endif -} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/bson/bson_encoder_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/bson/bson_encoder_tests.cpp deleted file mode 100644 index f8dd2f9f0e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/bson/bson_encoder_tests.cpp +++ /dev/null @@ -1,247 +0,0 @@ -// Copyright 2016 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("serialize object to bson") -{ - std::vector v; - bson::bson_bytes_encoder encoder(v); - - encoder.begin_object(); - encoder.name("null"); - encoder.null_value(); - encoder.end_object(); - encoder.flush(); - - try - { - json result = bson::decode_bson(v); - std::cout << result << std::endl; - } - catch (const std::exception& e) - { - std::cout << e.what() << std::endl; - } -} - -namespace jsoncons { namespace bson { - - void test_equal(const std::vector& v, const std::vector& expected) - { - REQUIRE(v.size() == expected.size()); - - for (size_t i = 0; i < v.size(); ++i) - { - CHECK(v[i] == expected[i]); - } - } - - void check_equal(const std::vector& v, const std::vector& expected) - { - test_equal(v, expected); - try - { - json j = bson::decode_bson(v); - std::vector u; - bson::encode_bson(j, u); - test_equal(v,u); - } - catch (const std::exception& e) - { - std::cout << e.what() << std::endl; - } - } -}} - -TEST_CASE("serialize to bson") -{ - SECTION("array") - { - std::vector v; - bson::bson_bytes_encoder encoder(v); - - encoder.begin_array(); - encoder.int64_value((std::numeric_limits::max)()); - encoder.uint64_value((uint64_t)(std::numeric_limits::max)()); - encoder.double_value((std::numeric_limits::max)()); - encoder.bool_value(true); - encoder.bool_value(false); - encoder.null_value(); - encoder.string_value("Pussy cat"); - encoder.byte_string_value(byte_string({'h','i','s','s'})); - encoder.end_array(); - encoder.flush(); - - std::vector bson = {0x4d,0x00,0x00,0x00, - 0x12, // int64 - 0x30, // '0' - 0x00, // terminator - 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, - 0x12, // int64 - 0x31, // '1' - 0x00, // terminator - 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, - 0x01, // double - 0x32, // '2' - 0x00, // terminator - 0xff,0xff,0xff,0xff,0xff,0xff,0xef,0x7f, - 0x08, // bool - 0x33, // '3' - 0x00, // terminator - 0x01, - 0x08, // bool - 0x34, // '4' - 0x00, // terminator - 0x00, - 0x0a, // null - 0x35, // '5' - 0x00, // terminator - 0x02, // string - 0x36, // '6' - 0x00, // terminator - 0x0a,0x00,0x00,0x00, // string length - 'P','u','s','s','y',' ','c','a','t', - 0x00, // terminator - 0x05, // binary - 0x37, // '7' - 0x00, // terminator - 0x04,0x00,0x00,0x00, // byte string length - 'h','i','s','s', - 0x00 // terminator - }; - jsoncons::bson::check_equal(v,bson); - - } - SECTION("object") - { - std::vector v; - bson::bson_bytes_encoder encoder(v); - - encoder.begin_object(); - encoder.name("0"); - encoder.int64_value((std::numeric_limits::max)()); - encoder.name("1"); - encoder.uint64_value((uint64_t)(std::numeric_limits::max)()); - encoder.name("2"); - encoder.double_value((std::numeric_limits::max)()); - encoder.name("3"); - encoder.bool_value(true); - encoder.name("4"); - encoder.bool_value(false); - encoder.name("5"); - encoder.null_value(); - encoder.name("6"); - encoder.string_value("Pussy cat"); - encoder.name("7"); - encoder.byte_string_value(byte_string({'h','i','s','s'})); - encoder.end_object(); - encoder.flush(); - - std::vector bson = {0x4d,0x00,0x00,0x00, - 0x12, // int64 - 0x30, // '0' - 0x00, // terminator - 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, - 0x12, // int64 - 0x31, // '1' - 0x00, // terminator - 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, - 0x01, // double - 0x32, // '2' - 0x00, // terminator - 0xff,0xff,0xff,0xff,0xff,0xff,0xef,0x7f, - 0x08, // bool - 0x33, // '3' - 0x00, // terminator - 0x01, - 0x08, // bool - 0x34, // '4' - 0x00, // terminator - 0x00, - 0x0a, // null - 0x35, // '5' - 0x00, // terminator - 0x02, // string - 0x36, // '6' - 0x00, // terminator - 0x0a,0x00,0x00,0x00, // string length - 'P','u','s','s','y',' ','c','a','t', - 0x00, // terminator - 0x05, // binary - 0x37, // '7' - 0x00, // terminator - 0x04,0x00,0x00,0x00, // byte string length - 'h','i','s','s', - 0x00 // terminator - }; - jsoncons::bson::check_equal(v,bson); - } - - SECTION("outer object") - { - std::vector v; - bson::bson_bytes_encoder encoder(v); - - encoder.begin_object(); - encoder.name("a"); - encoder.begin_object(); - encoder.name("0"); - encoder.int64_value((std::numeric_limits::max)()); - encoder.end_object(); - encoder.end_object(); - encoder.flush(); - - std::vector bson = {0x18,0x00,0x00,0x00, - 0x03, // object - 'a', - 0x00, - 0x10,0x00,0x00,0x00, - 0x12, // int64 - 0x30, // '0' - 0x00, // terminator - 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, - 0x00, // terminator - 0x00 // terminator - }; - jsoncons::bson::check_equal(v,bson); - } - - SECTION("outer array") - { - std::vector v; - bson::bson_bytes_encoder encoder(v); - - encoder.begin_array(); - encoder.begin_object(); - encoder.name("0"); - encoder.int64_value((std::numeric_limits::max)()); - encoder.end_object(); - encoder.end_array(); - encoder.flush(); - - std::vector bson = {0x18,0x00,0x00,0x00, - 0x03, // object - '0', - 0x00, - 0x10,0x00,0x00,0x00, - 0x12, // int64 - 0x30, // '0' - 0x00, // terminator - 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, - 0x00, // terminator - 0x00 // terminator - }; - jsoncons::bson::check_equal(v,bson); - } -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/bson/bson_reader_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/bson/bson_reader_tests.cpp deleted file mode 100644 index a903d8a210..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/bson/bson_reader_tests.cpp +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2016 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -void check_decode_bson(const std::vector& v, const json& expected) -{ - json result = bson::decode_bson(v); - REQUIRE(result == expected); - - std::string s; - for (auto c : v) - { - s.push_back(c); - } - std::istringstream is(s); - json j2 = bson::decode_bson(is); - REQUIRE(j2 == expected); -} - -TEST_CASE("bson hello world") -{ - check_decode_bson({0x16,0x00,0x00,0x00, // total document size - 0x02, // string - 'h','e','l','l','o', 0x00, // field name - 0x06,0x00,0x00,0x00, // size of value - 'w','o','r','l','d',0x00, // field value and null terminator - 0x00 // end of document - },json::parse("{\"hello\":\"world\"}")); -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/byte_string_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/byte_string_tests.cpp deleted file mode 100644 index 297c111f10..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/byte_string_tests.cpp +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -// https://tools.ietf.org/html/rfc4648#section-4 test vectors - -void check_encode_base64(const std::vector& input, const std::string& expected) -{ - std::string result; - encode_base64(input.begin(),input.end(),result); - REQUIRE(result.size() == expected.size()); - for (size_t i = 0; i < result.size(); ++i) - { - CHECK(result[i] == expected[i]); - } - - std::vector output; - decode_base64(result.begin(), result.end(), output); - REQUIRE(output.size() == input.size()); - for (size_t i = 0; i < output.size(); ++i) - { - CHECK(output[i] == input[i]); - } -} - -void check_encode_base64url(const std::vector& input, const std::string& expected) -{ - std::string result; - encode_base64url(input.begin(),input.end(),result); - REQUIRE(result.size() == expected.size()); - for (size_t i = 0; i < result.size(); ++i) - { - CHECK(result[i] == expected[i]); - } - - std::vector output; - decode_base64url(result.begin(), result.end(), output); - REQUIRE(output.size() == input.size()); - for (size_t i = 0; i < output.size(); ++i) - { - CHECK(output[i] == input[i]); - } -} - -void check_encode_base16(const std::vector& input, const std::string& expected) -{ - std::string result; - encode_base16(input.begin(),input.end(),result); - REQUIRE(result.size() == expected.size()); - for (size_t i = 0; i < result.size(); ++i) - { - CHECK(result[i] == expected[i]); - } - - std::vector output; - decode_base16(result.begin(), result.end(), output); - REQUIRE(output.size() == input.size()); - for (size_t i = 0; i < output.size(); ++i) - { - CHECK(output[i] == input[i]); - } -} -TEST_CASE("test_base64_conversion") -{ - check_encode_base64({}, ""); - check_encode_base64({'f'}, "Zg=="); - check_encode_base64({'f','o'}, "Zm8="); - check_encode_base64({'f','o','o'}, "Zm9v"); - check_encode_base64({'f','o','o','b'}, "Zm9vYg=="); - check_encode_base64({'f','o','o','b','a'}, "Zm9vYmE="); - check_encode_base64({'f','o','o','b','a','r'}, "Zm9vYmFy"); -} - -TEST_CASE("test_base64url_conversion") -{ - check_encode_base64url({}, ""); - check_encode_base64url({'f'}, "Zg"); - check_encode_base64url({'f','o'}, "Zm8"); - check_encode_base64url({'f','o','o'}, "Zm9v"); - check_encode_base64url({'f','o','o','b'}, "Zm9vYg"); - check_encode_base64url({'f','o','o','b','a'}, "Zm9vYmE"); - check_encode_base64url({'f','o','o','b','a','r'}, "Zm9vYmFy"); -} - -TEST_CASE("test_base16_conversion") -{ - check_encode_base16({}, ""); - check_encode_base16({'f'}, "66"); - check_encode_base16({'f','o'}, "666F"); - check_encode_base16({'f','o','o'}, "666F6F"); - check_encode_base16({'f','o','o','b'}, "666F6F62"); - check_encode_base16({'f','o','o','b','a'}, "666F6F6261"); - check_encode_base16({'f','o','o','b','a','r'}, "666F6F626172"); -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/cbor_encoder_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/cbor_encoder_tests.cpp deleted file mode 100644 index 44eef14b57..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/cbor_encoder_tests.cpp +++ /dev/null @@ -1,393 +0,0 @@ -// Copyright 2016 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("test_serialize_to_stream") -{ -json j = json::parse(R"( -{ - "application": "hiking", - "reputons": [ - { - "rater": "HikingAsylum.example.com", - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rating": 0.90 - } - ] -} -)"); - - std::ofstream os; - os.open("./output/store.cbor", std::ios::binary | std::ios::out); - cbor::encode_cbor(j,os); - - std::vector v; - std::ifstream is; - is.open("./output/store.cbor", std::ios::binary | std::ios::in); - - json j2 = cbor::decode_cbor(is); - - //std::cout << pretty_print(j2) << std::endl; - - CHECK(j == j2); -} - -TEST_CASE("serialize array to cbor") -{ - std::vector v; - cbor::cbor_bytes_encoder encoder(v); - //encoder.begin_object(1); - encoder.begin_array(3); - encoder.bool_value(true); - encoder.bool_value(false); - encoder.null_value(); - encoder.end_array(); - //encoder.end_object(); - encoder.flush(); - - try - { - json result = cbor::decode_cbor(v); - //std::cout << result << std::endl; - } - catch (const std::exception& e) - { - std::cout << e.what() << std::endl; - } -} - -TEST_CASE("test_serialize_indefinite_length_array") -{ - std::vector v; - cbor::cbor_bytes_encoder encoder(v); - encoder.begin_array(); - encoder.begin_array(4); - encoder.bool_value(true); - encoder.bool_value(false); - encoder.null_value(); - encoder.string_value("Hello"); - encoder.end_array(); - encoder.end_array(); - encoder.flush(); - - try - { - json result = cbor::decode_cbor(v); - //std::cout << result << std::endl; - } - catch (const std::exception& e) - { - std::cout << e.what() << std::endl; - } -} -TEST_CASE("test_serialize_bignum") -{ - std::vector v; - cbor::cbor_bytes_encoder encoder(v); - encoder.begin_array(); - - std::vector bytes = {0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; - bignum n(1, bytes.data(), bytes.size()); - std::string s; - n.dump(s); - encoder.string_value(s, semantic_tag::bigint); - encoder.end_array(); - encoder.flush(); - - try - { - json result = cbor::decode_cbor(v); - CHECK(result[0].as() == std::string("18446744073709551616")); - } - catch (const std::exception& e) - { - std::cout << e.what() << std::endl; - } -} - -TEST_CASE("test_serialize_negative_bignum1") -{ - std::vector v; - cbor::cbor_bytes_encoder encoder(v); - encoder.begin_array(); - - std::vector bytes = {0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; - bignum n(-1, bytes.data(), bytes.size()); - std::string s; - n.dump(s); - encoder.string_value(s, semantic_tag::bigint); - encoder.end_array(); - encoder.flush(); - - try - { - json result = cbor::decode_cbor(v); - CHECK(result[0].as() == std::string("-18446744073709551617")); - } - catch (const std::exception& e) - { - std::cout << e.what() << std::endl; - } -} - -TEST_CASE("test_serialize_negative_bignum2") -{ - std::vector v; - cbor::cbor_bytes_encoder encoder(v); - encoder.begin_array(); - - std::vector bytes = {0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; - bignum n(-1, bytes.data(), bytes.size()); - std::string s; - n.dump(s); - encoder.string_value(s, semantic_tag::bigint); - encoder.end_array(); - encoder.flush(); - - try - { - json result = cbor::decode_cbor(v); - json_options options; - options.bigint_format(bigint_chars_format::number); - std::string text; - result.dump(text,options); - CHECK(text == std::string("[-18446744073709551617]")); - } - catch (const std::exception& e) - { - std::cout << e.what() << std::endl; - } -} - -TEST_CASE("test_serialize_negative_bignum3") -{ - std::vector v; - cbor::cbor_bytes_encoder encoder(v); - encoder.begin_array(); - - std::vector bytes = {0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; - - bignum n(-1, bytes.data(), bytes.size()); - std::string s; - n.dump(s); - encoder.string_value(s, semantic_tag::bigint); - encoder.end_array(); - encoder.flush(); - - try - { - json result = cbor::decode_cbor(v); - json_options options; - options.bigint_format(bigint_chars_format::base64url); - std::string text; - result.dump(text,options); - CHECK(text == std::string("[\"~AQAAAAAAAAAA\"]")); - } - catch (const std::exception& e) - { - std::cout << e.what() << std::endl; - } -} - -TEST_CASE("serialize bigdec to cbor") -{ - SECTION("-1 184467440737095516160") - { - std::vector v; - cbor::cbor_bytes_encoder encoder(v); - encoder.string_value("18446744073709551616.0", semantic_tag::bigdec); - encoder.flush(); - try - { - json result = cbor::decode_cbor(v); - CHECK(result.as() == std::string("1.84467440737095516160e+19")); - } - catch (const std::exception& e) - { - std::cout << e.what() << std::endl; - } - } - SECTION("18446744073709551616e-5") - { - std::vector v; - cbor::cbor_bytes_encoder encoder(v); - encoder.string_value("18446744073709551616e-5", semantic_tag::bigdec); - encoder.flush(); - try - { - json result = cbor::decode_cbor(v); - CHECK(result.as() == std::string("184467440737095.51616")); - } - catch (const std::exception& e) - { - std::cout << e.what() << std::endl; - } - } - SECTION("-18446744073709551616e-5") - { - std::vector v; - cbor::cbor_bytes_encoder encoder(v); - encoder.string_value("-18446744073709551616e-5", semantic_tag::bigdec); - encoder.flush(); - try - { - json result = cbor::decode_cbor(v); - CHECK(result.as() == std::string("-184467440737095.51616")); - } - catch (const std::exception& e) - { - std::cout << e.what() << std::endl; - } - } - SECTION("-18446744073709551616e5") - { - std::vector v; - cbor::cbor_bytes_encoder encoder(v); - encoder.string_value("-18446744073709551616e5", semantic_tag::bigdec); - encoder.flush(); - try - { - json result = cbor::decode_cbor(v); - CHECK(result.as() == std::string("-1.8446744073709551616e+24")); - } - catch (const std::exception& e) - { - std::cout << e.what() << std::endl; - } - } -} - -TEST_CASE("Too many and too few items in CBOR map or array") -{ - std::error_code ec{}; - std::vector v; - cbor::cbor_bytes_encoder encoder(v); - - SECTION("Too many items in array") - { - CHECK(encoder.begin_array(3)); - CHECK(encoder.bool_value(true)); - CHECK(encoder.bool_value(false)); - CHECK(encoder.null_value()); - CHECK(encoder.begin_array(2)); - CHECK(encoder.string_value("cat")); - CHECK(encoder.string_value("feline")); - CHECK(encoder.end_array()); - REQUIRE_THROWS_WITH(encoder.end_array(), cbor::cbor_error_category_impl().message((int)cbor::cbor_errc::too_many_items).c_str()); - encoder.flush(); - } - SECTION("Too few items in array") - { - CHECK(encoder.begin_array(5)); - CHECK(encoder.bool_value(true)); - CHECK(encoder.bool_value(false)); - CHECK(encoder.null_value()); - CHECK(encoder.begin_array(2)); - CHECK(encoder.string_value("cat")); - CHECK(encoder.string_value("feline")); - CHECK(encoder.end_array()); - REQUIRE_THROWS_WITH(encoder.end_array(), cbor::cbor_error_category_impl().message((int)cbor::cbor_errc::too_few_items).c_str()); - encoder.flush(); - } - SECTION("Too many items in map") - { - CHECK(encoder.begin_object(3)); - CHECK(encoder.name("a")); - CHECK(encoder.bool_value(true)); - CHECK(encoder.name("b")); - CHECK(encoder.bool_value(false)); - CHECK(encoder.name("c")); - CHECK(encoder.null_value()); - CHECK(encoder.name("d")); - CHECK(encoder.begin_array(2)); - CHECK(encoder.string_value("cat")); - CHECK(encoder.string_value("feline")); - CHECK(encoder.end_array()); - REQUIRE_THROWS_WITH(encoder.end_object(), cbor::cbor_error_category_impl().message((int)cbor::cbor_errc::too_many_items).c_str()); - encoder.flush(); - } - SECTION("Too few items in map") - { - CHECK(encoder.begin_object(5)); - CHECK(encoder.name("a")); - CHECK(encoder.bool_value(true)); - CHECK(encoder.name("b")); - CHECK(encoder.bool_value(false)); - CHECK(encoder.name("c")); - CHECK(encoder.null_value()); - CHECK(encoder.name("d")); - CHECK(encoder.begin_array(2)); - CHECK(encoder.string_value("cat")); - CHECK(encoder.string_value("feline")); - CHECK(encoder.end_array()); - REQUIRE_THROWS_WITH(encoder.end_object(), cbor::cbor_error_category_impl().message((int)cbor::cbor_errc::too_few_items).c_str()); - encoder.flush(); - } - SECTION("Just enough items") - { - CHECK(encoder.begin_array(4)); // a fixed length array - CHECK(encoder.string_value("foo")); - CHECK(encoder.byte_string_value(byte_string{'P','u','s','s'})); // no suggested conversion - CHECK(encoder.string_value("-18446744073709551617", semantic_tag::bigint)); - CHECK(encoder.string_value("273.15", semantic_tag::bigdec)); - CHECK(encoder.end_array()); - CHECK_FALSE(ec); - encoder.flush(); - } -} - -TEST_CASE("encode stringref") -{ - ojson j = ojson::parse(R"( -[ - { - "name" : "Cocktail", - "count" : 417, - "rank" : 4 - }, - { - "rank" : 4, - "count" : 312, - "name" : "Bath" - }, - { - "count" : 691, - "name" : "Food", - "rank" : 4 - } - ] -)"); - - cbor::cbor_options options; - options.pack_strings(true); - std::vector buf; - - cbor::encode_cbor(j, buf, options); - - //for (auto c : buf) - //{ - // std::cout << std::hex << std::setprecision(2) << std::setw(2) - // << std::noshowbase << std::setfill('0') << static_cast(c); - //} - //std::cout << "\n"; - - ojson j2 = cbor::decode_cbor(buf); - CHECK(j2 == j); -} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/cbor_reader_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/cbor_reader_tests.cpp deleted file mode 100644 index 3591056cba..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/cbor_reader_tests.cpp +++ /dev/null @@ -1,733 +0,0 @@ -// Copyright 2016 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; -using namespace jsoncons::cbor; - -void check_parse_cbor(const std::vector& v, const json& expected) -{ - try - { - std::error_code ec; - - jsoncons::json_decoder decoder; - cbor_bytes_reader parser(v, decoder); - parser.read(ec); - - json result = decoder.get_result(); - - if (!(result == expected)) - { - std::cout << "v: "; - for (auto b : v) - { - std::cout << "0x" << std::hex << (int)b; - } - std::cout << "\n"; - std::cout << "result: " << result << "\n"; - std::cout << "expected: " << expected << "\n"; - } - - REQUIRE(result == expected); - CHECK(result.get_semantic_tag() == expected.get_semantic_tag()); - - std::string s; - for (auto c : v) - { - s.push_back(c); - } - std::istringstream is(s); - json j2 = decode_cbor(is); - REQUIRE(j2 == expected); - } - catch (const std::exception& e) - { - std::cout << e.what() << std::endl; - std::cout << expected.to_string() << std::endl; - } -} -TEST_CASE("test_cbor_parsing") -{ - // unsigned integer - check_parse_cbor({0x00},json(0U)); - check_parse_cbor({0x01},json(1U)); - check_parse_cbor({0x0a},json(10U)); - check_parse_cbor({0x17},json(23U)); - check_parse_cbor({0x18,0x18},json(24U)); - check_parse_cbor({0x18,0xff},json(255U)); - check_parse_cbor({0x19,0x01,0x00},json(256U)); - check_parse_cbor({0x19,0xff,0xff},json(65535U)); - check_parse_cbor({0x1a,0,1,0x00,0x00},json(65536U)); - check_parse_cbor({0x1a,0xff,0xff,0xff,0xff},json(4294967295U)); - check_parse_cbor({0x1b,0,0,0,1,0,0,0,0},json(4294967296U)); - check_parse_cbor({0x1b,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff},json((std::numeric_limits::max)())); - - // positive signed integer - check_parse_cbor({0x00},json(0)); - check_parse_cbor({0x01},json(1)); - check_parse_cbor({0x0a},json(10)); - check_parse_cbor({0x17},json(23)); - check_parse_cbor({0x18,0x18},json(24)); - check_parse_cbor({0x18,0xff},json(255)); - check_parse_cbor({0x19,0x01,0x00},json(256)); - check_parse_cbor({0x19,0xff,0xff},json(65535)); - check_parse_cbor({0x1a,0,1,0x00,0x00},json(65536)); - check_parse_cbor({0x1a,0xff,0xff,0xff,0xff},json(4294967295)); - check_parse_cbor({0x1b,0,0,0,1,0,0,0,0},json(4294967296)); - check_parse_cbor({0x1b,0x7f,0xff,0xff,0xff,0xff,0xff,0xff,0xff},json((std::numeric_limits::max)())); - // negative integers - check_parse_cbor({0x20},json(-1)); - check_parse_cbor({0x21},json(-2)); - check_parse_cbor({0x37},json(-24)); - check_parse_cbor({0x38,0x18},json(-25)); - check_parse_cbor({0x38,0xff},json(-256)); - check_parse_cbor({0x39,0x01,0x00},json(-257)); - check_parse_cbor({0x39,0xff,0xff},json(-65536)); - check_parse_cbor({0x3a,0,1,0x00,0x00},json(-65537)); - - check_parse_cbor({0x3a,0xff,0xff,0xff,0xff},json(-4294967296)); - check_parse_cbor({0x3b,0,0,0,1,0,0,0,0},json(-4294967297)); - - // null, undefined, true, false - check_parse_cbor({0xf6},json::null()); - check_parse_cbor({0xf7},json{null_type(),semantic_tag::undefined}); - check_parse_cbor({0xf5},json(true)); - check_parse_cbor({0xf4},json(false)); - - // floating point - check_parse_cbor({0xfb,0,0,0,0,0,0,0,0},json(0.0)); - check_parse_cbor({0xfb,0xbf,0xf0,0,0,0,0,0,0},json(-1.0)); - check_parse_cbor({0xfb,0xc1,0x6f,0xff,0xff,0xe0,0,0,0},json(-16777215.0)); - check_parse_cbor({0xfa,0xcb,0x7f,0xff,0xff},json(-16777215.0)); - - // byte string - std::vector v; - check_parse_cbor({0x40},json(byte_string_view(v.data(),v.size()))); - v = {' '}; - check_parse_cbor({0x41,' '},json(byte_string_view(v.data(),v.size()))); - v = {0}; - check_parse_cbor({0x41,0},json(byte_string_view(v.data(),v.size()))); - v = {'H','e','l','l','o'}; - check_parse_cbor({0x45,'H','e','l','l','o'},json(byte_string_view(v.data(),v.size()))); - v = {'1','2','3','4','5','6','7','8','9','0','1','2','3','4','5','6','7','8','9','0','1','2','3','4'}; - check_parse_cbor({0x58,0x18,'1','2','3','4','5','6','7','8','9','0','1','2','3','4','5','6','7','8','9','0','1','2','3','4'}, - json(byte_string_view(v.data(),v.size()))); - - // string - check_parse_cbor({0x60},json("")); - check_parse_cbor({0x61,' '},json(" ")); - check_parse_cbor({0x78,0x18,'1','2','3','4','5','6','7','8','9','0','1','2','3','4','5','6','7','8','9','0','1','2','3','4'}, - json("123456789012345678901234")); - - // byte strings with undefined length - check_parse_cbor({0x5f,0xff}, json(byte_string())); - check_parse_cbor({0x5f,0x40,0xff}, json(byte_string())); - check_parse_cbor({0x5f,0x40,0x40,0xff}, json(byte_string())); - - check_parse_cbor({0x5f,0x43,'H','e','l',0x42,'l','o',0xff}, json(byte_string("Hello"))); - check_parse_cbor({0x5f,0x41,'H',0x41,'e',0x41,'l',0x41,'l',0x41,'o',0xff}, json(byte_string("Hello"))); - check_parse_cbor({0x5f,0x41,'H',0x41,'e',0x40,0x41,'l',0x41,'l',0x41,'o',0xff}, json(byte_string("Hello"))); - - // text strings with undefined length - - check_parse_cbor({0x7f,0xff}, json("")); - - check_parse_cbor({0x7f,0x60,0xff}, json("")); - check_parse_cbor({0x7f,0x60,0x60,0xff}, json("")); - check_parse_cbor({0x7f,0x63,'H','e','l',0x62,'l','o',0xff}, json("Hello")); - check_parse_cbor({0x7f,0x61,'H',0x61,'e',0x61,'l',0x61,'l',0x61,'o',0xff}, json("Hello")); - check_parse_cbor({0x7f,0x61,'H',0x61,'e',0x61,'l',0x60,0x61,'l',0x61,'o',0xff}, json("Hello")); - - SECTION ("arrays with definite length") - { - check_parse_cbor({0x80},json::array()); - check_parse_cbor({0x81,'\0'},json::parse("[0]")); - check_parse_cbor({0x82,'\0','\0'},json::array({0,0})); - check_parse_cbor({0x82,0x81,'\0','\0'}, json::parse("[[0],0]")); - check_parse_cbor({0x81,0x65,'H','e','l','l','o'},json::parse("[\"Hello\"]")); - - check_parse_cbor({0x83,0x01,0x82,0x02,0x03,0x82,0x04,0x05},json::parse("[1, [2, 3], [4, 5]]")); - check_parse_cbor({0x82, - 0x7f,0xff, - 0x7f,0xff}, - json::parse("[\"\",\"\"]")); - - check_parse_cbor({0x82, - 0x5f,0xff, - 0x5f,0xff}, - json::array{json(byte_string()),json(byte_string())}); - } - - SECTION("arrays with indefinite length") - { - //check_parse_cbor({0x9f,0xff},json::array()); - check_parse_cbor({0x9f,0x9f,0xff,0xff},json::parse("[[]]")); - - check_parse_cbor({0x9f,0x01,0x82,0x02,0x03,0x9f,0x04,0x05,0xff,0xff},json::parse("[1, [2, 3], [4, 5]]")); - check_parse_cbor({0x9f,0x01,0x82,0x02,0x03,0x82,0x04,0x05,0xff},json::parse("[1, [2, 3], [4, 5]]")); - - check_parse_cbor({0x83,0x01,0x82,0x02,0x03,0x9f,0x04,0x05,0xff},json::parse("[1, [2, 3], [4, 5]]")); - check_parse_cbor({0x83, // Array of length 3 - 0x01, // 1 - 0x9f, // Start indefinite-length array - 0x02, // 2 - 0x03, // 3 - 0xff, // "break" - 0x82, // Array of length 2 - 0x04, // 4 - 0x05}, // 5 - json::parse("[1, [2, 3], [4, 5]]")); - - } - - // big float - - check_parse_cbor({0xc5, // Tag 5 - 0x82, // Array of length 2 - 0x21, // -2 - 0x19, 0x6a, 0xb3 // 27315 - },json("0x6AB3p-2",semantic_tag::bigfloat)); - - SECTION("maps with definite length") - { - //check_parse_cbor({0xa0},json::object()); - check_parse_cbor({0xa1,0x62,'o','c',0x81,'\0'}, json::parse("{\"oc\": [0]}")); - //check_parse_cbor({0xa1,0x62,'o','c',0x84,'\0','\1','\2','\3'}, json::parse("{\"oc\": [0, 1, 2, 3]}")); - } - SECTION("maps with indefinite length") - { - check_parse_cbor({0xbf,0xff},json::object()); - check_parse_cbor({0xbf,0x64,'N','a','m','e',0xbf,0xff,0xff},json::parse("{\"Name\":{}}")); - - check_parse_cbor({0xbf, // Start indefinite-length map - 0x63, // First key, UTF-8 string length 3 - 0x46,0x75,0x6e, // "Fun" - 0xf5, // First value, true - 0x63, // Second key, UTF-8 string length 3 - 0x41,0x6d,0x74, // "Amt" - 0x21, // -2 - 0xff}, // "break" - json::parse("{\"Fun\": true, \"Amt\": -2}")); - check_parse_cbor({0xbf, // Start indefinite-length map - 0x21, // First key, -2 - 0xf5, // First value, true - 0xf5, // Second key, UTF-8 string length 3 - 0x21, // -2 - 0xff}, // "break" - json::parse("{\"-2\": true, \"true\": -2}")); - } - - SECTION("maps with non-string keys") - { - check_parse_cbor({0xbf, // Start indefinite-length map - 0x21, // First key, -2 - 0xf5, // First value, true - 0xf5, // Second key, UTF-8 string length 3 - 0x21, // -2 - 0xff}, // "break" - json::parse("{\"-2\": true, \"true\": -2}")); - } - - // bignum - check_parse_cbor({0xc2,0x49,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, - json(bignum(1,{0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}))); - - // datetime - check_parse_cbor({0xc0,0x78,0x19,'2','0','1','5','-','0','5','-','0','7',' ','1','2',':','4','1',':','0','7','-','0','7',':','0','0'}, - json("2015-05-07 12:41:07-07:00", semantic_tag::datetime)); - - // epoch_time - check_parse_cbor({0xc1,0x1a,0x55,0x4b,0xbf,0xd3}, - json(1431027667, semantic_tag::timestamp)); -} - -TEST_CASE("cbor decimal fraction") -{ - check_parse_cbor({0xc4, // Tag 4 - 0x82, // Array of length 2 - 0x21, // -2 - 0x19,0x6a,0xb3 // 27315 - }, - json("273.15", semantic_tag::bigdec)); - check_parse_cbor({0xc4, // Tag 4 - 0x82, // Array of length 2 - 0x22, // -3 - 0x19,0x6a,0xb3 // 27315 - }, - json("27.315", semantic_tag::bigdec)); - check_parse_cbor({0xc4, // Tag 4 - 0x82, // Array of length 2 - 0x23, // -4 - 0x19,0x6a,0xb3 // 27315 - }, - json("2.7315", semantic_tag::bigdec)); - check_parse_cbor({0xc4, // Tag 4 - 0x82, // Array of length 2 - 0x24, // -5 - 0x19,0x6a,0xb3 // 27315 - }, - json("0.27315", semantic_tag::bigdec)); - check_parse_cbor({0xc4, // Tag 4 - 0x82, // Array of length 2 - 0x25, // -6 - 0x19,0x6a,0xb3 // 27315 - }, - json("0.027315", semantic_tag::bigdec)); - - check_parse_cbor({0xc4, // Tag 4 - 0x82, // Array of length 2 - 0x04, // 4 - 0x19,0x6a,0xb3 // 27315 - }, - json("273150000.0", semantic_tag::bigdec)); -} - -TEST_CASE("test_decimal_as_string") -{ - SECTION("-2 27315") - { - std::vector v = {0xc4, // Tag 4, - 0x82, // Array of length 2 - 0x21, // -2 - 0x19,0x6a,0xb3 // 27315 - }; - - json j = decode_cbor(v); - CHECK(j.as() == std::string("273.15")); - } - SECTION("-6 27315") - { - std::vector v = {0xc4, // Tag 4, - 0x82, // Array of length 2 - 0x25, // -6 - 0x19,0x6a,0xb3 // 27315 - }; - - json j = decode_cbor(v); - CHECK(j.as() == std::string("0.027315")); - } - SECTION("-5 27315") - { - std::vector v = {0xc4, // Tag 4, - 0x82, // Array of length 2 - 0x24, // -5 - 0x19,0x6a,0xb3 // 27315 - }; - - json j = decode_cbor(v); - CHECK(j.as() == std::string("0.27315")); - } - SECTION("0 27315") - { - std::vector v = {0xc4, // Tag 4, - 0x82, // Array of length 2 - 0x00, // 0 - 0x19,0x6a,0xb3 // 27315 - }; - - json j = decode_cbor(v); - CHECK(j.as() == std::string("27315.0")); - } - SECTION("2 27315") - { - std::vector v = {0xc4, // Tag 4, - 0x82, // Array of length 2 - 0x02, // 2 - 0x19,0x6a,0xb3 // 27315 - }; - - json j = decode_cbor(v); - CHECK(j.as() == std::string("2731500.0")); - } - SECTION("-2 18446744073709551616") - { - std::vector v = {0xc4, // Tag 4, - 0x82, // Array of length 2 - 0x21, // -2 - 0xc2,0x49,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 // 18446744073709551616 - }; - - json j = decode_cbor(v); - CHECK(j.as() == std::string("1.8446744073709551616e+17")); - } - SECTION("-2 -65537") - { - std::vector v = {0xc4, // Tag 4, - 0x82, // Array of length 2 - 0x21, // -2 - 0x3a,0,1,0x00,0x00 // -65537 - }; - - json j = decode_cbor(v); - CHECK(j.as() == std::string("-655.37")); - } - SECTION("-5 -65537") - { - std::vector v = {0xc4, // Tag 4, - 0x82, // Array of length 2 - 0x24, // -5 - 0x3a,0,1,0x00,0x00 // -65537 - }; - - json j = decode_cbor(v); - CHECK(j.as() == std::string("-0.65537")); - } - SECTION("-6 -65537") - { - std::vector v = {0xc4, // Tag 4, - 0x82, // Array of length 2 - 0x25, // -6 - 0x3a,0,1,0x00,0x00 // -65537 - }; - - json j = decode_cbor(v); - CHECK(j.as() == std::string("-0.065537")); - } -} - -TEST_CASE("Compare CBOR packed item and jsoncons item") -{ - std::vector bytes; - cbor::cbor_bytes_encoder encoder(bytes); - encoder.begin_array(); // indefinite length outer array - encoder.string_value("foo"); - encoder.byte_string_value(byte_string{'b','a','r'}); - encoder.string_value("-18446744073709551617", semantic_tag::bigint); - encoder.string_value("-273.15", semantic_tag::bigdec); - encoder.string_value("273.15", semantic_tag::bigdec); - encoder.string_value("18446744073709551616.15", semantic_tag::bigdec); - encoder.string_value("-18446744073709551617.15", semantic_tag::bigdec); - encoder.string_value("2018-10-19 12:41:07-07:00", semantic_tag::datetime) ; - encoder.int64_value(1431027667, semantic_tag::timestamp); - encoder.int64_value(-1431027667, semantic_tag::timestamp); - encoder.double_value(1431027667.5, semantic_tag::timestamp); - encoder.end_array(); - encoder.flush(); - - json expected = json::array(); - expected.emplace_back("foo"); - expected.emplace_back(byte_string{ 'b','a','r' }); - expected.emplace_back("-18446744073709551617", semantic_tag::bigint); - expected.emplace_back("-273.15", semantic_tag::bigdec); - expected.emplace_back("273.15", semantic_tag::bigdec); - expected.emplace_back("1.844674407370955161615e+19", semantic_tag::bigdec); - expected.emplace_back("-1.844674407370955161715e+19", semantic_tag::bigdec); - expected.emplace_back("2018-10-19 12:41:07-07:00", semantic_tag::datetime); - expected.emplace_back(1431027667, semantic_tag::timestamp); - expected.emplace_back(-1431027667, semantic_tag::timestamp); - expected.emplace_back(1431027667.5, semantic_tag::timestamp); - - json j = cbor::decode_cbor(bytes); - - REQUIRE(j == expected); - for (size_t i = 0; i < j.size(); ++i) - { - CHECK(j[i].get_semantic_tag() == expected[i].get_semantic_tag()); - } -} - -TEST_CASE("CBOR stringref tag 1") -{ - std::vector v = {0xd9,0x01,0x00, // tag(256) - 0x83, // array(3) - 0xa3, // map(3) - 0x44, // bytes(4) - 0x72,0x61,0x6e,0x6b, // "rank" - 0x04, // unsigned(4) - 0x45, // bytes(5) - 0x63,0x6f,0x75,0x6e,0x74, // "count" - 0x19,0x01,0xa1, // unsigned(417) - 0x44, // bytes(4) - 0x6e,0x61,0x6d,0x65, // "name" - 0x48, // bytes(8) - 0x43,0x6f,0x63,0x6b,0x74,0x61,0x69,0x6c, // "Cocktail" - 0xa3, // map(3) - 0xd8,0x19, // tag(25) - 0x02, // unsigned(2) - 0x44, // bytes(4) - 0x42,0x61,0x74,0x68, // "Bath" - 0xd8,0x19, // tag(25) - 0x01, // unsigned(1) - 0x19,0x01,0x38, // unsigned(312) - 0xd8,0x19, // tag(25) - 0x00, // unsigned(0) - 0x04, // unsigned(4) - 0xa3, // map(3) - 0xd8,0x19, // tag(25) - 0x02, // unsigned(2) - 0x44, // bytes(4) - 0x46,0x6f,0x6f,0x64, // "Food" - 0xd8,0x19, // tag(25) - 0x01, // unsigned(1) - 0x19,0x02,0xb3, // unsigned(691) - 0xd8,0x19, // tag(25) - 0x00, // unsigned(0) - 0x04 // unsigned(4) - }; - - SECTION("decode") - { - ojson j = decode_cbor(v); - //std::cout << pretty_print(j) << "\n"; - - { - auto it = j[0].object_range().begin(); - std::string key1; - decode_base64url(it->key().begin(),it->key().end(),key1); - CHECK(key1 == std::string("rank")); - ++it; - std::string key2; - decode_base64url(it->key().begin(),it->key().end(),key2); - CHECK(key2 == std::string("count")); - ++it; - std::string key3; - decode_base64url(it->key().begin(),it->key().end(),key3); - CHECK(key3 == std::string("name")); - } - { - auto it = j[1].object_range().begin(); - std::string key3; - decode_base64url(it->key().begin(),it->key().end(),key3); - CHECK(key3 == std::string("name")); - ++it; - std::string key2; - decode_base64url(it->key().begin(),it->key().end(),key2); - CHECK(key2 == std::string("count")); - ++it; - std::string key1; - decode_base64url(it->key().begin(),it->key().end(),key1); - CHECK(key1 == std::string("rank")); - } - { - auto it = j[2].object_range().begin(); - std::string key3; - decode_base64url(it->key().begin(),it->key().end(),key3); - CHECK(key3 == std::string("name")); - ++it; - std::string key2; - decode_base64url(it->key().begin(),it->key().end(),key2); - CHECK(key2 == std::string("count")); - ++it; - std::string key1; - decode_base64url(it->key().begin(),it->key().end(),key1); - CHECK(key1 == std::string("rank")); - } - } -} - -TEST_CASE("CBOR stringref tag 2") -{ - std::vector v = {0xd9, 0x01, 0x00, // tag(256) - 0x98, 0x20, // array(32) - 0x41, // bytes(1) - 0x31, // "1" - 0x43, // bytes(3) - 0x32,0x32,0x32, // "222" - 0x43, // bytes(3) - 0x33,0x33,0x33, // "333" - 0x41, // bytes(1) - 0x34, // "4" - 0x43, // bytes(3) - 0x35,0x35,0x35, // "555" - 0x43, // bytes(3) - 0x36,0x36,0x36, // "666" - 0x43, // bytes(3) - 0x37,0x37,0x37, // "777" - 0x43, // bytes(3) - 0x38,0x38,0x38, // "888" - 0x43, // bytes(3) - 0x39,0x39,0x39, // "999" - 0x43, // bytes(3) - 0x61,0x61,0x61, // "aaa" - 0x43, // bytes(3) - 0x62,0x62,0x62, // "bbb" - 0x43, // bytes(3) - 0x63,0x63,0x63, // "ccc" - 0x43, // bytes(3) - 0x64,0x64,0x64, // "ddd" - 0x43, // bytes(3) - 0x65,0x65,0x65, // "eee" - 0x43, // bytes(3) - 0x66,0x66,0x66, // "fff" - 0x43, // bytes(3) - 0x67,0x67,0x67, // "ggg" - 0x43, // bytes(3) - 0x68,0x68,0x68, // "hhh" - 0x43, // bytes(3) - 0x69,0x69,0x69, // "iii" - 0x43, // bytes(3) - 0x6a,0x6a,0x6a, // "jjj" - 0x43, // bytes(3) - 0x6b,0x6b,0x6b, // "kkk" - 0x43, // bytes(3) - 0x6c,0x6c,0x6c, // "lll" - 0x43, // bytes(3) - 0x6d,0x6d,0x6d, // "mmm" - 0x43, // bytes(3) - 0x6e,0x6e,0x6e, // "nnn" - 0x43, // bytes(3) - 0x6f,0x6f,0x6f, // "ooo" - 0x43, // bytes(3) - 0x70,0x70,0x70, // "ppp" - 0x43, // bytes(3) - 0x71,0x71,0x71, // "qqq" - 0x43, // bytes(3) - 0x72,0x72,0x72, // "rrr" - 0xd8, 0x19, // tag(25) - 0x01, // unsigned(1) - 0x44, // bytes(4) - 0x73,0x73,0x73,0x73, // "ssss" - 0xd8, 0x19, // tag(25) - 0x17, // unsigned(23) - 0x43, // bytes(3) - 0x72,0x72,0x72, // "rrr" - 0xd8, 0x19, // tag(25) - 0x18, 0x18 // unsigned(24) - }; - - ojson j = decode_cbor(v); - - byte_string bs = j[0].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("1")); - - bs = j[1].as(); // 0 - CHECK(std::string(bs.begin(),bs.end()) == std::string("222")); - - bs = j[2].as(); // 1 - CHECK(std::string(bs.begin(),bs.end()) == std::string("333")); - - bs = j[3].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("4")); - - bs = j[4].as(); // 2 - CHECK(std::string(bs.begin(),bs.end()) == std::string("555")); - - bs = j[5].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("666")); - - bs = j[6].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("777")); - - bs = j[7].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("888")); - - bs = j[8].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("999")); - - bs = j[9].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("aaa")); - - bs = j[10].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("bbb")); - - bs = j[11].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("ccc")); - - bs = j[12].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("ddd")); - - bs = j[13].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("eee")); - - bs = j[14].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("fff")); - - bs = j[15].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("ggg")); - - bs = j[16].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("hhh")); - - bs = j[17].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("iii")); - - bs = j[18].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("jjj")); - - bs = j[19].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("kkk")); - - bs = j[20].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("lll")); - - bs = j[21].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("mmm")); - - bs = j[22].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("nnn")); - - bs = j[23].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("ooo")); - - bs = j[24].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("ppp")); - - bs = j[25].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("qqq")); - - bs = j[26].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("rrr")); - - bs = j[27].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("333")); - - bs = j[28].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("ssss")); - - bs = j[29].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("qqq")); - - bs = j[30].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("rrr")); - - bs = j[31].as(); - CHECK(std::string(bs.begin(),bs.end()) == std::string("ssss")); -} - -TEST_CASE("CBOR stringref tag 3") -{ - std::vector v = {0xd9,0x01,0x00, // tag(256) - 0x85, // array(5) - 0x63, // text(3) - 0x61,0x61,0x61, // "aaa" - 0xd8, 0x19, // tag(25) - 0x00, // unsigned(0) - 0xd9, 0x01,0x00, // tag(256) - 0x83, // array(3) - 0x63, // text(3) - 0x62,0x62,0x62, // "bbb" - 0x63, // text(3) - 0x61,0x61,0x61, // "aaa" - 0xd8, 0x19, // tag(25) - 0x01, // unsigned(1) - 0xd9, 0x01,0x00, // tag(256) - 0x82, // array(2) - 0x63, // text(3) - 0x63,0x63,0x63, // "ccc" - 0xd8, 0x19, // tag(25) - 0x00, // unsigned(0) - 0xd8, 0x19, // tag(25) - 0x00 // unsigned(0) - }; - - json j = cbor::decode_cbor(v); - - json expected = json::parse(R"( - ["aaa","aaa",["bbb","aaa","aaa"],["ccc","ccc"],"aaa"] - )"); - - CHECK(j == expected); -} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/cbor_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/cbor_tests.cpp deleted file mode 100644 index 0e4717b977..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/cbor_tests.cpp +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright 2016 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("cbor_test_floating_point") -{ - json j1; - j1["max double"] = (std::numeric_limits::max)(); - j1["max float"] = (std::numeric_limits::max)(); - - std::vector v; - cbor::encode_cbor(j1, v); - - json j2 = cbor::decode_cbor(v); - - CHECK(j1 == j2); -} - -TEST_CASE("cbor_test") -{ - json j1; - j1["zero"] = 0; - j1["one"] = 1; - j1["two"] = 2; - j1["null"] = null_type(); - j1["true"] = true; - j1["false"] = false; - j1["max int64_t"] = (std::numeric_limits::max)(); - j1["max uint64_t"] = (std::numeric_limits::max)(); - j1["min int64_t"] = (std::numeric_limits::lowest)(); - j1["max int32_t"] = (std::numeric_limits::max)(); - j1["max uint32_t"] = (std::numeric_limits::max)(); - j1["min int32_t"] = (std::numeric_limits::lowest)(); - j1["max int16_t"] = (std::numeric_limits::max)(); - j1["max uint16_t"] = (std::numeric_limits::max)(); - j1["min int16_t"] = (std::numeric_limits::lowest)(); - j1["max int8_t"] = (std::numeric_limits::max)(); - j1["max uint8_t"] = (std::numeric_limits::max)(); - j1["min int8_t"] = (std::numeric_limits::lowest)(); - j1["max double"] = (std::numeric_limits::max)(); - j1["min double"] = (std::numeric_limits::lowest)(); - j1["max float"] = (std::numeric_limits::max)(); - j1["zero float"] = 0.0; - j1["min float"] = (std::numeric_limits::lowest)(); - j1["String too long for small string optimization"] = "String too long for small string optimization"; - - json ja = json::array(); - ja.push_back(0); - ja.push_back(1); - ja.push_back(2); - ja.push_back(null_type()); - ja.push_back(true); - ja.push_back(false); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back(0.0); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back("String too long for small string optimization"); - - j1["An array"] = ja; - - std::vector v; - cbor::encode_cbor(j1, v); - - json j2 = cbor::decode_cbor(v); - - CHECK(j1 == j2); -} - -TEST_CASE("cbor_test2") -{ - wjson j1; - j1[L"zero"] = 0; - j1[L"one"] = 1; - j1[L"two"] = 2; - j1[L"null"] = null_type(); - j1[L"true"] = true; - j1[L"false"] = false; - j1[L"max int64_t"] = (std::numeric_limits::max)(); - j1[L"max uint64_t"] = (std::numeric_limits::max)(); - j1[L"min int64_t"] = (std::numeric_limits::lowest)(); - j1[L"max int32_t"] = (std::numeric_limits::max)(); - j1[L"max uint32_t"] = (std::numeric_limits::max)(); - j1[L"min int32_t"] = (std::numeric_limits::lowest)(); - j1[L"max int16_t"] = (std::numeric_limits::max)(); - j1[L"max uint16_t"] = (std::numeric_limits::max)(); - j1[L"min int16_t"] = (std::numeric_limits::lowest)(); - j1[L"max int8_t"] = (std::numeric_limits::max)(); - j1[L"max uint8_t"] = (std::numeric_limits::max)(); - j1[L"min int8_t"] = (std::numeric_limits::lowest)(); - j1[L"max double"] = (std::numeric_limits::max)(); - j1[L"min double"] = (std::numeric_limits::lowest)(); - j1[L"max float"] = (std::numeric_limits::max)(); - j1[L"zero float"] = 0.0; - j1[L"min float"] = (std::numeric_limits::lowest)(); - j1[L"S"] = L"S"; - j1[L"String too long for small string optimization"] = L"String too long for small string optimization"; - - wjson ja = wjson::array(); - ja.push_back(0); - ja.push_back(1); - ja.push_back(2); - ja.push_back(null_type()); - ja.push_back(true); - ja.push_back(false); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back(0.0); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back(L"S"); - ja.push_back(L"String too long for small string optimization"); - - j1[L"An array"] = ja; - - std::vector v; - cbor::encode_cbor(j1, v); - - wjson j2 = cbor::decode_cbor(v); - - CHECK(j1 == j2); -} - -TEST_CASE("cbor_reputon_test") -{ -ojson j1 = ojson::parse(R"( -{ - "application": "hiking", - "reputons": [ - { - "rater": "HikingAsylum.example.com", - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rating": 0.90 - } - ] -} -)"); - - std::vector v; - cbor::encode_cbor(j1, v); - - ojson j2 = cbor::decode_cbor(v); - CHECK(j1 == j2); - //std::cout << pretty_print(j2) << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/decode_cbor_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/decode_cbor_tests.cpp deleted file mode 100644 index 09bfa00894..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/decode_cbor_tests.cpp +++ /dev/null @@ -1,663 +0,0 @@ -// Copyright 2016 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("cbor_view_test") -{ - ojson j1 = ojson::parse(R"( - { - "application": "hiking", - "reputons": [ - { - "rater": "HikingAsylum.example.com", - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rating": 0.90 - } - ] - } - )"); - - std::vector c; - cbor::encode_cbor(j1, c); - - json v = cbor::decode_cbor(c); - CHECK(v.is_object()); - CHECK_FALSE(v.is_array()); - - const json& reputons = v.at("reputons"); - CHECK(reputons.is_array()); - - const json& reputons_0 = reputons.at(0); - - const json& reputons_0_rated = reputons_0.at("rated"); - (void)reputons_0_rated; - - const json& rating = reputons_0.at("rating"); - CHECK(rating.as_double() == 0.90); - - for (const auto& member : v.object_range()) - { - const auto& key = member.key(); - const json& jval = member.value(); - - (void)key; - (void)jval; - - //std::cout << key << ": " << jval << std::endl; - } - //std::cout << std::endl; - - for (auto element : reputons.array_range()) - { - json j = element; - //std::cout << j << std::endl; - } - //std::cout << std::endl; -} - -TEST_CASE("jsonpointer_test") -{ - json j = json::parse(R"( - { - "application": "hiking", - "reputons": [ - { - "rater": "HikingAsylum.example.com", - "assertion": "strong-hiker", - "rated": "Marilyn C", - "rating": 0.90 - } - ] - } - )"); - - std::vector v; - cbor::encode_cbor(j, v); - - json jdoc = cbor::decode_cbor(v); - std::string s; - jdoc.dump(s); - json j1 = json::parse(s); - CHECK(j1 == j); - - std::error_code ec; - const json& application = jsonpointer::get(jdoc, "/application", ec); - CHECK_FALSE(ec); - - CHECK(application == j["application"]); - - const json& reputons_0_rated = jsonpointer::get(jdoc, "/reputons", ec); - CHECK_FALSE(ec); - - json j4 = j["reputons"]; - CHECK(reputons_0_rated == j4); - - //std::cout << pretty_print(j3) << std::endl; -} - -TEST_CASE("as_string_test") -{ - std::vector v; - cbor::cbor_bytes_encoder encoder(v); - encoder.begin_array(10); - encoder.bool_value(true); - encoder.bool_value(false); - encoder.null_value(); - encoder.string_value("Toronto"); - encoder.byte_string_value(byte_string{'H','e','l','l','o'}); - encoder.int64_value(-100); - encoder.uint64_value(100); - encoder.string_value("18446744073709551616", semantic_tag::bigint); - encoder.double_value(10.5); - encoder.string_value("-18446744073709551617", semantic_tag::bigint); - encoder.end_array(); - encoder.flush(); - - json j = cbor::decode_cbor(v); - - std::string s0; - j[0].dump(s0); - CHECK(std::string("true") == s0); - CHECK(std::string("true") == j[0].as_string()); - CHECK(true == j[0].as()); - CHECK(j[0].is()); - - std::string s1; - j[1].dump(s1); - CHECK(std::string("false") == s1); - CHECK(std::string("false") == j[1].as_string()); - CHECK(false == j[1].as()); - CHECK(j[1].is()); - - std::string s2; - j[2].dump(s2); - CHECK(std::string("null") == s2); - CHECK(std::string("null") == j[2].as_string()); - - std::string s3; - j[3].dump(s3); - CHECK(std::string("\"Toronto\"") == s3); - CHECK(std::string("Toronto") == j[3].as_string()); - CHECK(std::string("Toronto") ==j[3].as()); - - std::string s4; - j[4].dump(s4); - CHECK(std::string("\"SGVsbG8\"") == s4); - CHECK(std::string("SGVsbG8") == j[4].as_string()); - CHECK(byte_string({'H','e','l','l','o'}) == j[4].as()); - - std::string s5; - j[5].dump(s5); - CHECK(std::string("-100") ==s5); - CHECK(std::string("-100") == j[5].as_string()); - CHECK(-100 ==j[5].as()); - - std::string s6; - j[6].dump(s6); - CHECK(std::string("100") == s6); - CHECK(std::string("100") == j[6].as_string()); - - std::string s7; - j[7].dump(s7); - CHECK(std::string("\"18446744073709551616\"") == s7); - CHECK(std::string("18446744073709551616") == j[7].as_string()); - - std::string s8; - j[8].dump(s8); - CHECK(std::string("10.5") == s8); - CHECK(std::string("10.5") == j[8].as_string()); - - std::string s9; - j[9].dump(s9); - CHECK(std::string("\"-18446744073709551617\"") == s9); - CHECK(std::string("-18446744073709551617") == j[9].as_string()); - -} - -TEST_CASE("dump cbor to string test") -{ - std::vector v; - cbor::cbor_bytes_encoder encoder(v); - encoder.begin_array(); - std::vector bytes = {0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; - bignum n(-1, bytes.data(), bytes.size()); - std::string s; - n.dump(s); - encoder.string_value(s, semantic_tag::bigint); - encoder.end_array(); - encoder.flush(); - - json j = cbor::decode_cbor(v); - - std::string s0; - j.dump(s0); - CHECK("[\"-18446744073709551617\"]" == s0); - //std::cout << s0 << std::endl; - - std::string s1; - json_options options1; - options1.bigint_format(bigint_chars_format::number); - j.dump(s1,options1); - CHECK("[-18446744073709551617]" == s1); - //std::cout << s1 << std::endl; - - std::string s2; - json_options options2; - options2.bigint_format(bigint_chars_format::base10); - j.dump(s2,options2); - CHECK("[\"-18446744073709551617\"]" == s2); - //std::cout << s2 << std::endl; - - std::string s3; - json_options options3; - options3.bigint_format(bigint_chars_format::base64url); - j.dump(s3,options3); - CHECK("[\"~AQAAAAAAAAAA\"]" == s3); - //std::cout << s3 << std::endl; -} - -TEST_CASE("test_dump_to_stream") -{ - std::vector v; - cbor::cbor_bytes_encoder encoder(v); - encoder.begin_array(); - std::vector bytes = {0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; - bignum n(-1, bytes.data(), bytes.size()); - std::string s; - n.dump(s); - encoder.string_value(s, semantic_tag::bigint); - encoder.end_array(); - encoder.flush(); - - json j = cbor::decode_cbor(v); - - std::ostringstream os0; - j.dump(os0); - CHECK("[\"-18446744073709551617\"]" == os0.str()); - //std::cout << os0.str() << std::endl; - - std::ostringstream os1; - json_options options1; - options1.bigint_format(bigint_chars_format::number); - j.dump(os1,options1); - CHECK("[-18446744073709551617]" == os1.str()); - //std::cout << os1.str() << std::endl; - - std::ostringstream os2; - json_options options2; - options2.bigint_format(bigint_chars_format::base10); - j.dump(os2,options2); - CHECK("[\"-18446744073709551617\"]" == os2.str()); - //std::cout << os2.str() << std::endl; - - std::ostringstream os3; - json_options options3; - options3.bigint_format(bigint_chars_format::base64url); - j.dump(os3,options3); - CHECK("[\"~AQAAAAAAAAAA\"]" == os3.str()); - //std::cout << os3.str() << std::endl; -} - -TEST_CASE("test_indefinite_length_object_iterator") -{ - std::vector v; - cbor::cbor_bytes_encoder encoder(v); - encoder.begin_object(); // indefinite length object - encoder.name("City"); - encoder.string_value("Toronto"); - encoder.name("Province"); - encoder.string_value("Ontario"); - encoder.end_object(); - encoder.flush(); - json bv2 = cbor::decode_cbor(v); - - auto it2 = bv2.object_range().begin(); - CHECK_FALSE((it2 == bv2.object_range().end())); - CHECK_FALSE((++it2 == bv2.object_range().end())); - CHECK((++it2 == bv2.object_range().end())); -} - -TEST_CASE("test_indefinite_length_array_iterator") -{ - std::vector v; - cbor::cbor_bytes_encoder encoder(v); - encoder.begin_array(); // indefinite length array - encoder.string_value("Toronto"); - encoder.string_value("Ontario"); - encoder.end_array(); - encoder.flush(); - json j = cbor::decode_cbor(v); - - CHECK(j.size() == 2); - - auto it2 = j.array_range().begin(); - CHECK_FALSE((it2 == j.array_range().end())); - CHECK_FALSE((++it2 == j.array_range().end())); - CHECK((++it2 == j.array_range().end())); - -} - -TEST_CASE("cbor array comparison test") -{ - std::vector v1; - cbor::cbor_bytes_encoder encoder1(v1); - encoder1.begin_array(); // indefinite length array - encoder1.string_value("Toronto"); - encoder1.string_value("Vancouver"); - encoder1.end_array(); - encoder1.flush(); - json j1 = cbor::decode_cbor(v1); - - std::vector v2; - cbor::cbor_bytes_encoder serializer2(v2); - serializer2.begin_array(); // indefinite length array - serializer2.string_value("Toronto"); - serializer2.string_value("Vancouver"); - serializer2.end_array(); - serializer2.flush(); - json j2 = cbor::decode_cbor(v2); - - std::vector v3; - cbor::cbor_bytes_encoder serializer3(v3); - serializer3.begin_array(); // indefinite length array - serializer3.string_value("Toronto"); - serializer3.string_value("Montreal"); - serializer3.end_array(); - serializer3.flush(); - json j3 = cbor::decode_cbor(v3); - - SECTION("operator== test") - { - CHECK(j1 == j2); - REQUIRE(j1.size() == 2); - REQUIRE(j2.size() == 2); - CHECK(j1[0] == j2[0]); - CHECK(j1[1] == j2[1]); - } - - SECTION("element operator== test") - { - CHECK_FALSE(j1 == j3); - REQUIRE(j1.size() == 2); - REQUIRE(j1.size() == j3.size()); - CHECK(j1[0] == j3[0]); - CHECK_FALSE(j1[1] == j3[1]); - } -} - -TEST_CASE("cbor object comparison") -{ - std::vector v; - cbor::cbor_bytes_encoder encoder1(v); - encoder1.begin_object(); // indefinite length array - encoder1.name("City"); - encoder1.string_value("Montreal"); - encoder1.name("Amount"); - encoder1.string_value("273.15", semantic_tag::bigdec); - encoder1.name("Date"); - encoder1.string_value("2018-05-07 12:41:07-07:00", semantic_tag::datetime) ; - encoder1.end_object(); - encoder1.flush(); - json j1 = cbor::decode_cbor(v); - - //std::cout << pretty_print(j1) << "\n"; - - REQUIRE(j1.size() == 3); - - std::vector buf2; - cbor::cbor_bytes_encoder serializer2(buf2); - serializer2.begin_object(); // indefinite length array - serializer2.name("City"); - serializer2.string_value("Toronto"); - serializer2.name("Amount"); - serializer2.string_value("273.15", semantic_tag::bigdec); - serializer2.name("Date"); - serializer2.string_value("2018-10-18 12:41:07-07:00", semantic_tag::datetime) ; - serializer2.end_object(); - serializer2.flush(); - json j2 = cbor::decode_cbor(buf2); - REQUIRE(j2.size() == j1.size()); - - std::vector buf3; - cbor::cbor_bytes_encoder serializer3(buf3); - serializer3.begin_object(); // indefinite length array - serializer3.name("empty-object"); - serializer3.begin_object(0); - serializer3.end_object(); - serializer3.name("empty-array"); - serializer3.begin_array(0); - serializer3.end_array(); - serializer3.name("empty-string"); - serializer3.string_value(""); - serializer3.name("empty-byte_string"); - serializer3.byte_string_value(jsoncons::byte_string{}); - serializer3.end_object(); - serializer3.flush(); - json j3 = cbor::decode_cbor(buf3); - - SECTION("contains") - { - CHECK(j1.contains("City")); - CHECK(j1.contains("Amount")); - CHECK(j1.contains("Date")); - CHECK_FALSE(j1.contains("Country")); - } - - SECTION("empty") - { - CHECK_FALSE(j3.empty()); - CHECK(j3["empty-object"].empty()); - CHECK(j3["empty-array"].empty()); - CHECK(j3["empty-string"].empty()); - CHECK(j3["empty-byte_string"].empty()); - } - - SECTION("size") - { - CHECK(j1.size() == 3); - } - - SECTION("operator==") - { - CHECK_FALSE(j1 == j2); - CHECK_FALSE(j1["City"] == j2["City"]); - CHECK(j1["Amount"] == j2["Amount"]); - CHECK_FALSE(j1["Date"] == j2["Date"]); - } -} - -TEST_CASE("cbor member tests") -{ - std::vector v; - cbor::cbor_bytes_encoder encoder(v); - encoder.begin_object(); // indefinite length object - encoder.name("empty-object"); - encoder.begin_object(0); - encoder.end_object(); - encoder.name("empty-array"); - encoder.begin_array(0); - encoder.end_array(); - encoder.name("empty-string"); - encoder.string_value(""); - encoder.name("empty-byte_string"); - encoder.byte_string_value(jsoncons::byte_string{}); - - encoder.name("City"); - encoder.string_value("Montreal"); - encoder.name("Amount"); - encoder.string_value("273.15", semantic_tag::bigdec); - encoder.name("Date"); - encoder.string_value("2018-05-07 12:41:07-07:00", semantic_tag::datetime) ; - - encoder.end_object(); - encoder.flush(); - json j = cbor::decode_cbor(v); - - SECTION("contains") - { - CHECK(j.contains("City")); - CHECK(j.contains("Amount")); - CHECK(j.contains("Date")); - CHECK_FALSE(j.contains("Country")); - } - - SECTION("empty") - { - CHECK_FALSE(j.empty()); - CHECK(j["empty-object"].empty()); - CHECK(j["empty-array"].empty()); - CHECK(j["empty-string"].empty()); - CHECK(j["empty-byte_string"].empty()); - } - - SECTION("size") - { - CHECK(j.size() == 7); - } -} - -TEST_CASE("cbor conversion tests") -{ - std::vector v; - cbor::cbor_bytes_encoder encoder(v); - encoder.begin_array(); // indefinite length outer array - encoder.begin_array(4); // a fixed length array - encoder.string_value("foo"); - encoder.byte_string_value(byte_string{'P','u','s','s'}); // no suggested conversion - encoder.string_value("-18446744073709551617", semantic_tag::bigint); - encoder.string_value("273.15", semantic_tag::bigdec); - encoder.end_array(); - encoder.end_array(); - encoder.flush(); - - json j = cbor::decode_cbor(v); - REQUIRE(j.size() == 1); - - auto range1 = j.array_range(); - auto it = range1.begin(); - const json& inner_array = *it++; - REQUIRE(inner_array.size() == 4); - REQUIRE((it == range1.end())); - - auto range2 = inner_array.array_range(); - auto it2 = range2.begin(); - CHECK(it2->as_string() == "foo"); - it2++; - CHECK(it2->as_byte_string() == byte_string{'P','u','s','s'}); - it2++; - CHECK(bool(it2->as_bignum() == bignum{"-18446744073709551617"})); - it2++; - CHECK(bool(it2->as_string() == std::string{"273.15"})); - it2++; - CHECK((it2 == range2.end())); -} - -TEST_CASE("cbor array as<> test") -{ - std::vector v; - cbor::cbor_bytes_encoder encoder(v); - encoder.begin_array(); // indefinite length outer array - encoder.string_value("foo"); - encoder.byte_string_value(byte_string({'b','a','r'})); - encoder.string_value("-18446744073709551617", semantic_tag::bigint); - encoder.string_value("273.15", semantic_tag::bigdec); - encoder.string_value("2015-05-07 12:41:07-07:00", semantic_tag::datetime) ; - encoder.int64_value(1431027667, semantic_tag::timestamp); - encoder.int64_value(-1431027667, semantic_tag::timestamp); - encoder.double_value(1431027667.5, semantic_tag::timestamp); - encoder.end_array(); - encoder.flush(); - -/* -9f -- Start indefinite length array - 63 -- String value of length 3 - 666f6f -- "foo" - 43 -- Byte string value of length 3 - 626172 -- 'b''a''r' - c3 -- Tag 3 (negative bignum) - 49 Byte string value of length 9 - 010000000000000000 -- Bytes content - c4 - Tag 4 (decimal fraction) - 82 -- Array of length 2 - 21 -- -2 - 19 6ab3 -- 27315 - c0 -- Tag 0 (date-time) - 78 19 -- Length (25) - 323031352d30352d30372031323a34313a30372d30373a3030 -- "2015-05-07 12:41:07-07:00" - c1 -- Tag 1 (epoch time) - 1a -- uint32_t - 554bbfd3 -- 1431027667 - c1 - 3a - 554bbfd2 - c1 - fb - 41d552eff4e00000 - ff -- "break" -*/ - - //std::cout << "v: \n"; - //for (auto c : v) - //{ - // std::cout << std::hex << std::setprecision(2) << std::setw(2) - // << std::setfill('0') << static_cast(c); - //} - //std::cout << "\n\n"; - - json j = cbor::decode_cbor(v); // a non-owning view of the CBOR v - - CHECK(j.size() == 8); - - SECTION("j[0].is()") - { - CHECK(j[0].is()); - CHECK(j[1].is()); - CHECK(j[1].is()); - CHECK(j[2].is()); - CHECK(j[3].is_string()); - CHECK(j[3].get_semantic_tag() == semantic_tag::bigdec); - CHECK(j[4].is()); - CHECK(j[5].is()); - CHECK(j[5].is()); - CHECK(j[6].is()); - CHECK_FALSE(j[6].is()); - CHECK(j[7].is()); - } - - SECTION("j[0].as()") - { - CHECK(j[0].as() == std::string("foo")); - CHECK(j[1].as() == jsoncons::byte_string({'b','a','r'})); - CHECK(j[2].as() == std::string("-18446744073709551617")); - CHECK(bool(j[2].as() == jsoncons::bignum("-18446744073709551617"))); - CHECK(j[3].as() == std::string("273.15")); - CHECK(j[4].as() == std::string("2015-05-07 12:41:07-07:00")); - CHECK(j[5].as() == 1431027667); - CHECK(j[5].as() == 1431027667U); - CHECK(j[6].as() == -1431027667); - CHECK(j[7].as() == 1431027667.5); - } - - SECTION("array_iterator is test") - { - auto it = j.array_range().begin(); - CHECK(it++->is()); - CHECK(it++->is()); - CHECK(it++->is()); - CHECK(it++->is()); - CHECK(it++->is()); - CHECK(it++->is()); - CHECK(it++->is()); - CHECK(it++->is()); - } -} - -TEST_CASE("cbor bigfloat tests") -{ - SECTION("1.5") - { - std::vector v = {0xc5, // Tag 5 - 0x82, // Array of length 2 - 0x20, // -1 - 0x03 // 3 - }; - - json j = cbor::decode_cbor(v); - - //std::string s = j.as(); - - double val = j.as(); - CHECK(val == Approx(1.5).epsilon(0.0000000001)); - } - SECTION("-1.5") - { - std::vector v = {0xc5, // Tag 5 - 0x82, // Array of length 2 - 0x20, // -1 - 0x22 // -3 - }; - - json j = cbor::decode_cbor(v); - //std::string s = j.as(); - //CHECK(s == std::string("-1.5")); - - double val = j.as(); - CHECK(val == Approx(-1.5).epsilon(0.0000000001)); - } -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/encode_cbor_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/encode_cbor_tests.cpp deleted file mode 100644 index ce5507f1f9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/cbor/encode_cbor_tests.cpp +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2016 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -// test vectors from tinycbor https://github.com/01org/tinycbor tst_encoder.cpp -// MIT license - -void check_encode_cbor(const std::vector& expected, const json& j) -{ - std::vector result; - cbor::encode_cbor(j,result); - REQUIRE(result.size() == expected.size()); - for (size_t i = 0; i < expected.size(); ++i) - { - REQUIRE(result[i] == expected[i]); - } -} - -TEST_CASE("cbor_encoder_test") -{ - // unsigned integer - check_encode_cbor({0x00},json(0U)); - check_encode_cbor({0x01},json(1U)); - check_encode_cbor({0x0a},json(10U)); - check_encode_cbor({0x17},json(23U)); - check_encode_cbor({0x18,0x18},json(24U)); - check_encode_cbor({0x18,0xff},json(255U)); - check_encode_cbor({0x19,0x01,0x00},json(256U)); - check_encode_cbor({0x19,0xff,0xff},json(65535U)); - check_encode_cbor({0x1a,0,1,0x00,0x00},json(65536U)); - check_encode_cbor({0x1a,0xff,0xff,0xff,0xff},json(4294967295U)); - check_encode_cbor({0x1b,0,0,0,1,0,0,0,0},json(4294967296U)); - check_encode_cbor({0x1b,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff},json((std::numeric_limits::max)())); - - // positive signed integer - check_encode_cbor({0x00},json(0)); - check_encode_cbor({0x01},json(1)); - check_encode_cbor({0x0a},json(10)); - check_encode_cbor({0x17},json(23)); - check_encode_cbor({0x18,0x18},json(24)); - check_encode_cbor({0x18,0xff},json(255)); - check_encode_cbor({0x19,0x01,0x00},json(256)); - check_encode_cbor({0x19,0xff,0xff},json(65535)); - check_encode_cbor({0x1a,0,1,0x00,0x00},json(65536)); - check_encode_cbor({0x1a,0xff,0xff,0xff,0xff},json(4294967295)); - check_encode_cbor({0x1b,0,0,0,1,0,0,0,0},json(4294967296)); - check_encode_cbor({0x1b,0x7f,0xff,0xff,0xff,0xff,0xff,0xff,0xff},json((std::numeric_limits::max)())); - - // negative integers - check_encode_cbor({0x20},json(-1)); - check_encode_cbor({0x21},json(-2)); - check_encode_cbor({0x37},json(-24)); - check_encode_cbor({0x38,0x18},json(-25)); - check_encode_cbor({0x38,0xff},json(-256)); - check_encode_cbor({0x39,0x01,0x00},json(-257)); - check_encode_cbor({0x39,0xff,0xff},json(-65536)); - check_encode_cbor({0x3a,0,1,0x00,0x00},json(-65537)); - check_encode_cbor({0x3a,0xff,0xff,0xff,0xff},json(-4294967296)); - check_encode_cbor({0x3b,0,0,0,1,0,0,0,0},json(-4294967297)); - - // null, true, false - check_encode_cbor({0xf6},json::null()); - check_encode_cbor({0xf5},json(true)); - check_encode_cbor({0xf4},json(false)); - - // floating point - check_encode_cbor({0xfa,0,0,0,0},json(0.0)); - check_encode_cbor({0xfa,0xbf,0x80,0,0},json(-1.0)); - - SECTION("-16777215.0") - { - double val = -16777215.0; - float valf = (float)val; - CHECK((double)valf == val); - check_encode_cbor({0xfa,0xcb,0x7f,0xff,0xff},json(val)); - } - // From https://en.wikipedia.org/wiki/Double-precision_floating-point_format - SECTION("0.333333333333333314829616256247390992939472198486328125") - { - double val = 0.333333333333333314829616256247390992939472198486328125; - float valf = (float)val; - CHECK((double)valf != val); - check_encode_cbor({0xfb,0x3F,0xD5,0x55,0x55,0x55,0x55,0x55,0x55},json(val)); - } - - // byte string - check_encode_cbor({0x40},json(byte_string())); - check_encode_cbor({0x41,' '},json(byte_string({' '}))); - check_encode_cbor({0x41,0},json(byte_string({0}))); - check_encode_cbor({0x45,'H','e','l','l','o'},json(byte_string("Hello"))); - check_encode_cbor({0x58,0x18,'1','2','3','4','5','6','7','8','9','0','1','2','3','4','5','6','7','8','9','0','1','2','3','4'}, - json(byte_string("123456789012345678901234"))); - - // text string - check_encode_cbor({0x60},json("")); - check_encode_cbor({0x61,' '},json(" ")); - check_encode_cbor({0x78,0x18,'1','2','3','4','5','6','7','8','9','0','1','2','3','4','5','6','7','8','9','0','1','2','3','4'}, - json("123456789012345678901234")); - -} - -TEST_CASE("cbor_arrays_and_maps") -{ - check_encode_cbor({ 0x80 }, json::array()); - check_encode_cbor({ 0xa0 }, json::object()); - - check_encode_cbor({ 0x81,'\0' }, json::parse("[0]")); - check_encode_cbor({ 0x82,'\0','\0' }, json::array({ 0,0 })); - check_encode_cbor({ 0x82,0x81,'\0','\0' }, json::parse("[[0],0]")); - check_encode_cbor({ 0x81,0x65,'H','e','l','l','o' }, json::parse("[\"Hello\"]")); - - // big float - json j("0x6AB3p-2", semantic_tag::bigfloat); - CHECK(j.get_semantic_tag() == semantic_tag::bigfloat); - json j2 = j; - CHECK(j2.get_semantic_tag() == semantic_tag::bigfloat); - json j3; - j3 = j; - CHECK(j3.get_semantic_tag() == semantic_tag::bigfloat); - - check_encode_cbor({ 0xc5, // Tag 5 - 0x82, // Array of length 2 - 0x21, // -2 - 0x19, 0x6a, 0xb3 // 27315 - }, json("0x6AB3p-2", semantic_tag::bigfloat)); - - check_encode_cbor({ 0xa1,0x62,'o','c',0x81,'\0' }, json::parse("{\"oc\": [0]}")); - check_encode_cbor({ 0xa1,0x62,'o','c',0x84,'\0','\1','\2','\3' }, json::parse("{\"oc\": [0, 1, 2, 3]}")); -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/csv/csv_subfield_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/csv/csv_subfield_tests.cpp deleted file mode 100644 index 9657c5a705..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/csv/csv_subfield_tests.cpp +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -//#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; -using namespace jsoncons::literals; - -TEST_CASE("test_n_objects") -{ - const std::string s = R"(calculationPeriodCenters,paymentCenters,resetCenters -NY;LON,TOR,LON -NY,LON,TOR;LON -"NY";"LON","TOR","LON" -"NY","LON","TOR";"LON" -)"; - csv::csv_options options; - options.assume_header(true) - .subfield_delimiter(';'); - - json expected = R"( -[ - { - "calculationPeriodCenters": ["NY","LON"], - "paymentCenters": "TOR", - "resetCenters": "LON" - }, - { - "calculationPeriodCenters": "NY", - "paymentCenters": "LON", - "resetCenters": ["TOR","LON"] - }, - { - "calculationPeriodCenters": ["NY","LON"], - "paymentCenters": "TOR", - "resetCenters": "LON" - }, - { - "calculationPeriodCenters": "NY", - "paymentCenters": "LON", - "resetCenters": ["TOR","LON"] - } -] - )"_json; - - try - { - json j = csv::decode_csv(s,options); - CHECK(j == expected); - //std::cout << pretty_print(j) << std::endl; - } - catch (const std::exception& e) - { - std::cerr << e.what() << std::endl; - } -} - -TEST_CASE("test_n_rows") -{ - const std::string s = R"(calculationPeriodCenters,paymentCenters,resetCenters -NY;LON,TOR,LON -NY,LON,TOR;LON -"NY";"LON","TOR","LON" -"NY","LON","TOR";"LON" -)"; - csv::csv_options options; - options.mapping(csv::mapping_type::n_rows) - .subfield_delimiter(';'); - - json expected = R"( -[ - ["calculationPeriodCenters","paymentCenters","resetCenters"], - [ - ["NY","LON"],"TOR","LON" - ], - ["NY","LON", - ["TOR","LON"] - ], - [ - ["NY","LON"],"TOR","LON" - ], - ["NY","LON", - ["TOR","LON"] - ] -] - )"_json; - - try - { - json j = csv::decode_csv(s,options); - CHECK(j == expected); - //std::cout << pretty_print(j) << std::endl; - } - catch (const std::exception& e) - { - std::cerr << e.what() << std::endl; - } -} - -TEST_CASE("test_m_columns") -{ - const std::string s = R"(calculationPeriodCenters,paymentCenters,resetCenters -NY;LON,TOR,LON -NY,LON,TOR;LON -"NY";"LON","TOR","LON" -"NY","LON","TOR";"LON" -)"; - csv::csv_options options; - options.assume_header(true) - .mapping(csv::mapping_type::m_columns) - .subfield_delimiter(';'); - - json expected = R"( -{ - "calculationPeriodCenters": [ - ["NY","LON"],"NY", - ["NY","LON"],"NY" - ], - "paymentCenters": ["TOR","LON","TOR","LON"], - "resetCenters": ["LON", - ["TOR","LON"],"LON", - ["TOR","LON"] - ] -} - )"_json; - - try - { - json j = csv::decode_csv(s,options); - CHECK(j == expected); - //std::cout << pretty_print(j) << std::endl; - } - catch (const std::exception& e) - { - std::cerr << e.what() << std::endl; - } -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/csv/csv_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/csv/csv_tests.cpp deleted file mode 100644 index 3ef4e8d656..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/csv/csv_tests.cpp +++ /dev/null @@ -1,1227 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -//#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("n_objects_test") -{ - const std::string bond_yields = R"(Date,1Y,2Y,3Y,5Y -2017-01-09,0.0062,0.0075,0.0083,0.011 -2017-01-08,0.0063,0.0076,0.0084,0.0112 -2017-01-08,0.0063,0.0076,0.0084,0.0112 -)"; - - json_decoder decoder; - csv::csv_options options; - options.assume_header(true) - .subfield_delimiter(0); - //.column_types("string,float,float,float,float"); - - options.mapping(csv::mapping_type::n_rows); - csv::csv_reader reader1(bond_yields,decoder,options); - reader1.read(); - ojson val1 = decoder.get_result(); - //std::cout << "\n(1)\n"<< pretty_print(val1) << "\n"; - CHECK(val1.size() == 4); - - options.mapping(csv::mapping_type::n_objects); - csv::csv_reader reader2(bond_yields,decoder,options); - reader2.read(); - ojson val2 = decoder.get_result(); - //std::cout << "\n(2)\n"<< pretty_print(val2) << "\n"; - REQUIRE(val2.size() == 3); - CHECK("2017-01-09" == val2[0]["Date"].as()); -} - -TEST_CASE("m_columns_test") -{ - const std::string bond_yields = R"(Date,ProductType,1Y,2Y,3Y,5Y -2017-01-09,"Bond",0.0062,0.0075,0.0083,0.011 -2017-01-08,"Bond",0.0063,0.0076,0.0084,0.0112 -2017-01-08,"Bond",0.0063,0.0076,0.0084,0.0112 -)"; - - json_decoder decoder; - csv::csv_options options; - options.assume_header(true) - .mapping(csv::mapping_type::m_columns); - - std::istringstream is(bond_yields); - csv::csv_reader reader(is, decoder, options); - reader.read(); - ojson j = decoder.get_result(); - CHECK(6 == j.size()); - CHECK(3 == j["Date"].size()); - CHECK(3 == j["1Y"].size()); - CHECK(3 == j["2Y"].size()); - CHECK(3 == j["3Y"].size()); - CHECK(3 == j["5Y"].size()); -} - -TEST_CASE("m_columns with ignore_empty_value") -{ - const std::string bond_yields = R"(Date,ProductType,1Y,2Y,3Y,5Y -2017-01-09,"Bond",0.0062,0.0075,0.0083,0.011 -2017-01-08,"Bond",0.0063,0.0076,,0.0112 -2017-01-08,"Bond",0.0063,0.0076,0.0084, -)"; - //std::cout << bond_yields << std::endl < decoder; - csv::csv_options options; - options.assume_header(true) - .ignore_empty_values(true) - .mapping(csv::mapping_type::m_columns); - - std::istringstream is(bond_yields); - csv::csv_reader reader(is, decoder, options); - reader.read(); - ojson j = decoder.get_result(); - //std::cout << "\n(1)\n"<< pretty_print(j) << "\n"; - CHECK(6 == j.size()); - CHECK(3 == j["Date"].size()); - CHECK(3 == j["1Y"].size()); - CHECK(3 == j["2Y"].size()); - CHECK(2 == j["3Y"].size()); - CHECK(2 == j["5Y"].size()); -} - -TEST_CASE("csv_test_empty_values") -{ - std::string input = "bool-f,int-f,float-f,string-f" -"\n,,,," -"\ntrue,12,24.7,\"test string\"," -"\n,,,,"; - - std::istringstream is(input); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(true) - .column_types("boolean,integer,float,string"); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val[0]["bool-f"].is_null()); - CHECK(val[0]["bool-f"].is()); - CHECK(val[0]["int-f"].is_null()); - CHECK(val[0]["int-f"].is()); - CHECK(val[0]["float-f"].is_null()); - CHECK(val[0]["float-f"].is()); - CHECK(val[0]["string-f"].as() == ""); - CHECK(val[0]["string-f"].is()); - - CHECK(val[1]["bool-f"] .as()== true); - CHECK(val[1]["bool-f"].is()); - CHECK(val[1]["int-f"] .as()== 12); - CHECK(val[1]["int-f"].is()); - CHECK(val[1]["float-f"] .as()== 24.7); - CHECK(val[1]["float-f"].is()); - CHECK(val[1]["string-f"].as() == "test string"); - CHECK(val[1]["string-f"].is()); - - CHECK(val[0]["bool-f"].is_null()); - CHECK(val[0]["bool-f"].is()); - CHECK(val[0]["int-f"].is_null()); - CHECK(val[0]["int-f"].is()); - CHECK(val[0]["float-f"].is_null()); - CHECK(val[0]["float-f"].is()); - CHECK(val[0]["string-f"] .as() == ""); - CHECK(val[0]["string-f"].is()); -} - -TEST_CASE("csv_test_empty_values_with_defaults") -{ - std::string input = "bool-f,int-f,float-f,string-f" -"\n,,,," -"\ntrue,12,24.7,\"test string\"," -"\n,,,,"; - - std::istringstream is(input); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(true) - .column_types("boolean,integer,float,string") - .column_defaults("false,0,0.0,\"\""); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - std::cout << pretty_print(val) << std::endl; - - CHECK(val[0]["bool-f"].as() == false); - CHECK(val[0]["bool-f"].is()); - CHECK(val[0]["int-f"] .as()== 0); - CHECK(val[0]["int-f"].is()); - CHECK(val[0]["float-f"].as() == 0.0); - CHECK(val[0]["float-f"].is()); - CHECK(val[0]["string-f"] .as() == ""); - CHECK(val[0]["string-f"].is()); - - CHECK(val[1]["bool-f"] .as()== true); - CHECK(val[1]["bool-f"].is()); - CHECK(val[1]["int-f"] .as()== 12); - CHECK(val[1]["int-f"].is()); - CHECK(val[1]["float-f"] .as()== 24.7); - CHECK(val[1]["float-f"].is()); - CHECK(val[1]["string-f"].as() == "test string"); - CHECK(val[1]["string-f"].is()); - - CHECK(val[2]["bool-f"].as() == false); - CHECK(val[2]["bool-f"].is()); - CHECK(val[2]["int-f"] .as()== 0); - CHECK(val[2]["int-f"].is()); - CHECK(val[2]["float-f"].as() == 0.0); - CHECK(val[2]["float-f"].is()); - CHECK(val[2]["string-f"].as() == ""); - CHECK(val[2]["string-f"].is()); -} - -TEST_CASE("csv_test_empty_values_with_empty_defaults") -{ - std::string input = "bool-f,int-f,float-f,string-f" -"\n,,,," -"\ntrue,12,24.7,\"test string\"," -"\n,,,,"; - - std::istringstream is(input); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(true) - .column_types("boolean,integer,float,string") - .column_defaults(",,,"); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val[0]["bool-f"].is_null()); - CHECK(val[0]["bool-f"].is()); - CHECK(val[0]["int-f"].is_null()); - CHECK(val[0]["int-f"].is()); - CHECK(val[0]["float-f"].is_null()); - CHECK(val[0]["float-f"].is()); - CHECK(val[0]["string-f"] .as() == ""); - CHECK(val[0]["string-f"].is()); - - CHECK(val[1]["bool-f"] .as() == true); - CHECK(val[1]["bool-f"].is()); - CHECK(val[1]["int-f"] .as()== 12); - CHECK(val[1]["int-f"].is()); - CHECK(val[1]["float-f"] .as()== 24.7); - CHECK(val[1]["float-f"].is()); - CHECK(val[1]["string-f"].as() == "test string"); - CHECK(val[1]["string-f"].is()); - - CHECK(val[0]["bool-f"].is_null()); - CHECK(val[0]["bool-f"].is()); - CHECK(val[0]["int-f"].is_null()); - CHECK(val[0]["int-f"].is()); - CHECK(val[0]["float-f"].is_null()); - CHECK(val[0]["float-f"].is()); - CHECK(val[0]["string-f"] .as() == ""); - CHECK(val[0]["string-f"].is()); -} - -TEST_CASE("csv_test1_array_1col_skip1_a") -{ - std::string text = "a\n1\n4"; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.header_lines(1); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val.size()==2); - CHECK(val[0].size()==1); - CHECK(val[1].size()==1); - CHECK(val[0][0]==json(1)); - CHECK(val[1][0]==json(4)); -} - -TEST_CASE("csv_test1_array_1col_skip1_b") -{ - std::string text = "a\n1\n4"; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.header_lines(1); - options.infer_types(false); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val.size()==2); - CHECK(val[0].size()==1); - CHECK(val[1].size()==1); - CHECK(val[0][0]==json("1")); - CHECK(val[1][0]==json("4")); -} - -TEST_CASE("csv_test1_array_1col_a") -{ - std::string text = "1\n4"; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(false); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val.size()==2); - CHECK(val[0].size()==1); - CHECK(val[1].size()==1); - CHECK(val[0][0]==json(1)); - CHECK(val[1][0]==json(4)); -} - -TEST_CASE("csv_test1_array_1col_b") -{ - std::string text = "1\n4"; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(false) - .infer_types(false); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val.size()==2); - CHECK(val[0].size()==1); - CHECK(val[1].size()==1); - CHECK(val[0][0]==json("1")); - CHECK(val[1][0]==json("4")); -} - -TEST_CASE("csv_test1_array_3cols") -{ - std::string text = "a,b,c\n1,2,3\n4,5,6"; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(false); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val.size()==3); - CHECK(val[0].size()==3); - CHECK(val[1].size()==3); - CHECK(val[2].size()==3); - CHECK(val[0][0]==json("a")); - CHECK(val[0][1]==json("b")); - CHECK(val[0][2]==json("c")); - CHECK(val[1][0]==json(1)); - CHECK(val[1][1]==json(2)); - CHECK(val[1][2]==json(3)); - CHECK(val[2][0]==json(4)); - CHECK(val[2][1]==json(5)); - CHECK(val[2][2]==json(6)); -} -TEST_CASE("csv_test1_array_3cols_trim_leading") -{ - std::string text = "a ,b ,c \n 1, 2, 3\n 4 , 5 , 6 "; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(false) - .trim_leading(true); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val.size()==3); - CHECK(val[0].size()==3); - CHECK(val[1].size()==3); - CHECK(val[2].size()==3); - CHECK(val[0][0]==json("a ")); - CHECK(val[0][1]==json("b ")); - CHECK(val[0][2]==json("c ")); - CHECK(val[1][0]==json(1)); - CHECK(val[1][1]==json(2)); - CHECK(val[1][2]==json(3)); - CHECK(val[2][0]==json("4 ")); - CHECK(val[2][1]==json("5 ")); - CHECK(val[2][2]==json("6 ")); -} - -TEST_CASE("csv_test1_array_3cols_trim_trailing") -{ - std::string text = "a ,b ,c \n 1, 2, 3\n 4 , 5 , 6 "; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(false) - .trim_trailing(true); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val.size()==3); - CHECK(val[0].size()==3); - CHECK(val[1].size()==3); - CHECK(val[2].size()==3); - CHECK(val[0][0]==json("a")); - CHECK(val[0][1]==json("b")); - CHECK(val[0][2]==json("c")); - CHECK(json(" 1") == val[1][0]); - CHECK(val[1][1]==json(" 2")); - CHECK(val[1][2]==json(" 3")); - CHECK(val[2][0]==json(" 4")); - CHECK(val[2][1]==json(" 5")); - CHECK(val[2][2]==json(" 6")); -} - -TEST_CASE("csv_test1_array_3cols_trim") -{ - std::string text = "a ,, \n 1, 2, 3\n 4 , 5 , 6 "; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(false) - .trim(true) - .unquoted_empty_value_is_null(true); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val.size()==3); - CHECK(val[0].size()==3); - CHECK(val[1].size()==3); - CHECK(val[2].size()==3); - CHECK(val[0][0]==json("a")); - CHECK(val[0][1]==json::null()); - CHECK(val[0][2]==json::null()); - CHECK(val[1][0]==json(1)); - CHECK(val[1][1]==json(2)); - CHECK(val[1][2]==json(3)); - CHECK(val[2][0]==json(4)); - CHECK(val[2][1]==json(5)); - CHECK(val[2][2]==json(6)); -} - -TEST_CASE("csv_test1_array_3cols_comment") -{ - std::string text = "a,b,c\n#1,2,3\n4,5,6"; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.comment_starter('#'); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val.size()==2); - CHECK(val[0].size()==3); - CHECK(val[1].size()==3); - CHECK(val[0][0]==json("a")); - CHECK(val[0][1]==json("b")); - CHECK(val[0][2]==json("c")); - CHECK(val[1][0]==json(4)); - CHECK(val[1][1]==json(5)); - CHECK(val[1][2]==json(6)); -} - -TEST_CASE("csv_test1_object_1col") -{ - std::string text = "a\n1\n4"; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(true); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val.size()==2); - CHECK(val[0].size()==1); - CHECK(val[1].size()==1); - CHECK(val[0]["a"]==json(1)); - CHECK(val[1]["a"]==json(4)); -} - -TEST_CASE("csv_test1_object_3cols") -{ - std::string text = "a,b,c\n1,2,3\n4,5,6"; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(true); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val.size()==2); - CHECK(val[0].size()==3); - CHECK(val[1].size()==3); - CHECK(val[0]["a"]==json(1)); - CHECK(val[0]["b"]==json(2)); - CHECK(val[0]["c"]==json(3)); - CHECK(val[1]["a"]==json(4)); - CHECK(val[1]["b"]==json(5)); - CHECK(val[1]["c"]==json(6)); -} - -TEST_CASE("csv_test1_object_3cols_header") -{ - std::string text = "a,b,c\n1,2,3\n4,5,6"; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.column_names("x,y,z") - .header_lines(1); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val.size()==2); - CHECK(val[0].size()==3); - CHECK(val[1].size()==3); - CHECK(val[0]["x"]==json(1)); - CHECK(val[0]["y"]==json(2)); - CHECK(val[0]["z"]==json(3)); - CHECK(val[1]["x"]==json(4)); - CHECK(val[1]["y"]==json(5)); - CHECK(val[1]["z"]==json(6)); -} - -TEST_CASE("csv_test1_object_3cols_bool") -{ - std::string text = "a,b,c\n1,0,1\ntrue,FalSe,TrUe"; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.column_names("x,y,z") - .column_types("boolean,boolean,boolean") - .header_lines(1); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val.size()==2); - CHECK(val[0].size()==3); - CHECK(val[1].size()==3); - CHECK(val[0]["x"]==json(true)); - CHECK(val[0]["y"]==json(false)); - CHECK(val[0]["z"]==json(true)); - CHECK(val[1]["x"]==json(true)); - CHECK(val[1]["y"]==json(false)); - CHECK(val[1]["z"]==json(true)); -} - -TEST_CASE("csv_test1_object_1col_quoted") -{ - std::string text = "a\n\"1\"\n\"4\""; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(true); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val.size()==2); - CHECK(val[0].size()==1); - CHECK(val[1].size()==1); - CHECK(val[0]["a"]==json("1")); - CHECK(val[1]["a"]==json("4")); -} - -TEST_CASE("csv_test1_object_3cols_quoted") -{ - std::string text = "a,b,c\n\"1\",\"2\",\"3\"\n4,5,\"6\""; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(true); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val.size()==2); - CHECK(val[0].size()==3); - CHECK(val[1].size()==3); - CHECK(val[0]["a"]==json("1")); - CHECK(val[0]["b"]==json("2")); - CHECK(val[0]["c"]==json("3")); - CHECK(val[1]["a"]==json(4)); - CHECK(val[1]["b"]==json(5)); - CHECK(val[1]["c"]==json("6")); -} - -TEST_CASE("csv_test1_array_1col_crlf") -{ - std::string text = "1\r\n4"; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(false); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val.size()==2); - CHECK(val[0].size()==1); - CHECK(val[1].size()==1); - CHECK(val[0][0]==json(1)); - CHECK(val[1][0]==json(4)); -} - -TEST_CASE("csv_test1_array_3cols_crlf") -{ - std::string text = "a,b,c\r\n1,2,3\r\n4,5,6"; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(false); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val.size()==3); - CHECK(val[0].size()==3); - CHECK(val[1].size()==3); - CHECK(val[2].size()==3); - CHECK(val[0][0]==json("a")); - CHECK(val[0][1]==json("b")); - CHECK(val[0][2]==json("c")); - CHECK(val[1][0]==json(1)); - CHECK(val[1][1]==json(2)); - CHECK(val[1][2]==json(3)); - CHECK(val[2][0]==json(4)); - CHECK(val[2][1]==json(5)); - CHECK(val[2][2]==json(6)); -} - -TEST_CASE("csv_test1_object_1col_crlf") -{ - std::string text = "a\r\n1\r\n4"; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(true); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val.size()==2); - CHECK(val[0].size()==1); - CHECK(val[1].size()==1); - CHECK(val[0]["a"]==json(1)); - CHECK(val[1]["a"]==json(4)); -} - -TEST_CASE("csv_test1_object_3cols_crlf") -{ - std::string text = "a,b,c\r\n1,2,3\r\n4,5,6"; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(true); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - CHECK(val.size()==2); - CHECK(val[0].size()==3); - CHECK(val[1].size()==3); - CHECK(val[0]["a"]==json(1)); - CHECK(val[0]["b"]==json(2)); - CHECK(val[0]["c"]==json(3)); - CHECK(val[1]["a"]==json(4)); - CHECK(val[1]["b"]==json(5)); - CHECK(val[1]["c"]==json(6)); -} - -TEST_CASE("read_comma_delimited_file") -{ - std::string in_file = "./input/countries.csv"; - std::ifstream is(in_file); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(true); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json countries = decoder.get_result(); - - CHECK(4 == countries.size()); - CHECK(json("ABW") == countries[0]["country_code"]); - CHECK(json("ARUBA") ==countries[0]["name"]); - CHECK(json("ATF") == countries[1]["country_code"]); - CHECK(json("FRENCH SOUTHERN TERRITORIES, D.R. OF") == countries[1]["name"]); - CHECK(json("VUT") == countries[2]["country_code"]); - CHECK(json("VANUATU") ==countries[2]["name"]); - CHECK(json("WLF") == countries[3]["country_code"]); - CHECK(json("WALLIS & FUTUNA ISLANDS") == countries[3]["name"]); -} - -TEST_CASE("read_comma_delimited_file_header") -{ - std::string in_file = "./input/countries.csv"; - std::ifstream is(in_file); - - json_decoder decoder; - - csv::csv_options options; - options.column_names("Country Code,Name") - .header_lines(1); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json countries = decoder.get_result(); - CHECK(4 == countries.size()); - CHECK(json("ABW") == countries[0]["Country Code"]); - CHECK(json("ARUBA") ==countries[0]["Name"]); - CHECK(json("ATF") == countries[1]["Country Code"]); - CHECK(json("FRENCH SOUTHERN TERRITORIES, D.R. OF") == countries[1]["Name"]); - CHECK(json("VUT") == countries[2]["Country Code"]); - CHECK(json("VANUATU") == countries[2]["Name"]); - CHECK(json("WLF") == countries[3]["Country Code"]); - CHECK(json("WALLIS & FUTUNA ISLANDS") == countries[3]["Name"]); -} - -TEST_CASE("serialize_comma_delimited_file") -{ - std::string in_file = "./input/countries.json"; - std::ifstream is(in_file); - - csv::csv_options options; - options.assume_header(false); - - json_decoder encoder1; - json_reader reader1(is,encoder1); - reader1.read(); - ojson countries1 = encoder1.get_result(); - - std::stringstream ss; - csv::csv_encoder encoder(ss,options); - countries1.dump(encoder); - - json_decoder encoder2; - csv::csv_reader reader2(ss,encoder2,options); - reader2.read(); - ojson countries2 = encoder2.get_result(); - - CHECK(countries1 == countries2); -} - -TEST_CASE("test_tab_delimited_file") -{ - std::string in_file = "./input/employees.txt"; - std::ifstream is(in_file); - - json_decoder decoder; - csv::csv_options options; - options.field_delimiter('\t') - .assume_header(true); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json employees = decoder.get_result(); - CHECK(4 == employees.size()); - CHECK(std::string("00000001") ==employees[0]["employee-no"].as()); - CHECK(std::string("00000002") ==employees[1]["employee-no"].as()); - CHECK(std::string("00000003") ==employees[2]["employee-no"].as()); - CHECK(std::string("00000004") ==employees[3]["employee-no"].as()); -} - -TEST_CASE("serialize_tab_delimited_file") -{ - std::string in_file = "./input/employees.json"; - std::ifstream is(in_file); - - json_decoder decoder; - csv::csv_options options; - options.assume_header(false) - .header_lines(1) - .column_names("dept,employee-name,employee-no,note,comment,salary") - .field_delimiter('\t'); - - json_reader reader(is,decoder); - reader.read_next(); - ojson employees1 = decoder.get_result(); - - std::stringstream ss; - csv::csv_encoder encoder(ss,options); - //std::cout << pretty_print(employees1) << std::endl; - employees1.dump(encoder); - //std::cout << ss.str() << std::endl; - - json_decoder encoder2; - csv::csv_reader reader2(ss,encoder2,options); - reader2.read(); - ojson employees2 = encoder2.get_result(); - //std::cout << pretty_print(employees2) << std::endl; - - CHECK(employees1.size() == employees2.size()); - - for (size_t i = 0; i < employees1.size(); ++i) - { - CHECK(employees1[i]["dept"] == employees2[i]["dept"]); - CHECK(employees1[i]["employee-name"] ==employees2[i]["employee-name"]); - CHECK(employees1[i]["employee-no"] ==employees2[i]["employee-no"]); - CHECK(employees1[i]["salary"] == employees2[i]["salary"]); - CHECK(employees1[i].get_with_default("note","") == employees2[i].get_with_default("note","")); - } -} - -TEST_CASE("csv_test1_array_3cols_grouped1") -{ - std::string text = "1,2,3\n4,5,6\n7,8,9"; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(false) - .column_types("integer,[integer]*"); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - //std::cout << val << std::endl; - - /*CHECK(val.size()==3); - CHECK(val[0].size()==3); - CHECK(val[1].size()==3); - CHECK(val[2].size()==3); - CHECK(val[0][0]==json(1)); - CHECK(val[0][1]==json(2)); - CHECK(val[0][2]==json(3)); - CHECK(val[1][0]==json(4)); - CHECK(val[1][1]==json(5)); - CHECK(val[1][2]==json(6)); - CHECK(val[2][0]==json(7)); - CHECK(val[2][1]==json(8)); - CHECK(val[2][2]==json(9));*/ -} - -TEST_CASE("csv_test1_array_3cols_grouped2") -{ - std::string text = "1,2,3,4,5\n4,5,6,7,8\n7,8,9,10,11"; - std::istringstream is(text); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(false) - .column_types("integer,[integer,integer]*"); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json val = decoder.get_result(); - - //std::cout << val << std::endl; -/* - REQUIRE(options.column_types().size() == 4); - CHECK(options.column_types()[0].first == csv::csv_column_type::integer_t); - CHECK(options.column_types()[0].second == 0); - CHECK(options.column_types()[1].first == csv::csv_column_type::integer_t); - CHECK(options.column_types()[1].second == 1); - CHECK(options.column_types()[2].first == csv::csv_column_type::integer_t); - CHECK(options.column_types()[2].second == 1); - CHECK(options.column_types()[3].first == csv::csv_column_type::repeat_t); - CHECK(options.column_types()[3].second == 2); -*/ -} - -TEST_CASE("csv_test1_repeat") -{ - const std::string text = R"(Date,1Y,2Y,3Y,5Y - 2017-01-09,0.0062,0.0075,0.0083,0.011,0.012 - 2017-01-08,0.0063,0.0076,0.0084,0.0112,0.013 - 2017-01-08,0.0063,0.0076,0.0084,0.0112,0.014 - )"; - - auto result = csv::csv_options::parse_column_types("string,float*"); - REQUIRE(result.size() == 3); - CHECK(result[0].col_type == csv::csv_column_type::string_t); - CHECK(result[0].level == 0); - CHECK(0 == result[0].rep_count); - CHECK(result[1].col_type == csv::csv_column_type::float_t); - CHECK(result[1].level == 0); - CHECK(0 == result[1].rep_count); - CHECK(result[2].col_type == csv::csv_column_type::repeat_t); - CHECK(result[2].level == 0); - CHECK(1 == result[2].rep_count); - - auto result2 = csv::csv_options::parse_column_types("string,[float*]"); - REQUIRE(result2.size() == 3); - CHECK(result2[0].col_type == csv::csv_column_type::string_t); - CHECK(result2[0].level == 0); - CHECK(0 == result2[0].rep_count); - CHECK(result2[1].col_type == csv::csv_column_type::float_t); - CHECK(result2[1].level == 1); - CHECK(0 == result2[1].rep_count); - CHECK(result2[2].col_type == csv::csv_column_type::repeat_t); - CHECK(result2[2].level == 1); - CHECK(1 == result2[2].rep_count); - - auto result3 = csv::csv_options::parse_column_types("string,[float]*"); - REQUIRE(result3.size() == 3); - CHECK(result3[0].col_type == csv::csv_column_type::string_t); - CHECK(result3[0].level == 0); - CHECK(0 == result3[0].rep_count); - CHECK(result3[1].col_type == csv::csv_column_type::float_t); - CHECK(result3[1].level == 1); - CHECK(0 == result3[1].rep_count); - CHECK(result3[2].col_type == csv::csv_column_type::repeat_t); - CHECK(result3[2].level == 0); - CHECK(1 == result3[2].rep_count); -} - -TEST_CASE("csv_test1_repeat2") -{ - csv::csv_options options1; - options1.column_types("[integer,string]*"); - for (auto x : options1.column_types()) - { - std::cout << (int)x.col_type << " " << x.level << " " << x.rep_count << std::endl; - } -} - -TEST_CASE("empty_line_test_1") -{ - std::string input = R"(country_code,name -ABW,ARUBA - -ATF,"FRENCH SOUTHERN TERRITORIES, D.R. OF" -VUT,VANUATU -WLF,WALLIS & FUTUNA ISLANDS -)"; - - std::istringstream is(input); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(true); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json j = decoder.get_result(); - CHECK(j.size() == 4); - - std::cout << pretty_print(j) << std::endl; -} - -TEST_CASE("empty_line_test_2") -{ - std::string input = R"(country_code,name -ABW,ARUBA - -ATF,"FRENCH SOUTHERN TERRITORIES, D.R. OF" -VUT,VANUATU -WLF,WALLIS & FUTUNA ISLANDS -)"; - - std::istringstream is(input); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(true) - .ignore_empty_lines(false); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json j = decoder.get_result(); - CHECK(j.size() == 5); - - std::cout << pretty_print(j) << std::endl; -} - -TEST_CASE("line_with_one_space") -{ - std::string input = R"(country_code,name -ABW,ARUBA -\t -ATF,"FRENCH SOUTHERN TERRITORIES, D.R. OF" -VUT,VANUATU -WLF,WALLIS & FUTUNA ISLANDS -)"; - - std::istringstream is(input); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(true); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json j = decoder.get_result(); - CHECK(j.size() == 5); - - std::cout << pretty_print(j) << std::endl; -} - -TEST_CASE("line_with_one_space_and_trim") -{ - std::string input = R"(country_code,name -ABW,ARUBA - -ATF,"FRENCH SOUTHERN TERRITORIES, D.R. OF" -VUT,VANUATU -WLF,WALLIS & FUTUNA ISLANDS -)"; - - std::istringstream is(input); - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(true) - .trim(true); - - csv::csv_reader reader(is,decoder,options); - reader.read(); - json j = decoder.get_result(); - CHECK(j.size() == 4); - - std::cout << pretty_print(j) << std::endl; -} - -TEST_CASE("test_decode_csv_from_string") -{ - std::string s = "some label\nsome value"; - csv::csv_options options; - options.assume_header(true); - std::cout << csv::decode_csv(s,options) << std::endl; -} - -TEST_CASE("test_decode_csv_from_stream") -{ - std::string s = "some label\nsome value"; - std::stringstream is(s); - csv::csv_options options; - options.assume_header(true); - std::cout << csv::decode_csv(is,options) << std::endl; -} - -TEST_CASE("test_encode_csv_to_stream") -{ - json j = json::array(); - j.push_back(json::object({ {"a",1},{"b",2} })); - std::cout << j << std::endl; - csv::csv_options options; - options.assume_header(true); - std::ostringstream os; - csv::encode_csv(j, os, options); - std::cout << os.str() << std::endl; -} - -TEST_CASE("test_type_inference") -{ - const std::string input = R"(customer_name,has_coupon,phone_number,zip_code,sales_tax_rate,total_amount -"John Roe",true,0272561313,01001,0.05,431.65 -"Jane Doe",false,416-272-2561,55416,0.15,480.70 -"Joe Bloggs",false,"4162722561","55416",0.15,300.70 -"John Smith",FALSE,NULL,22313-1450,0.15,300.70 -)"; - - std::cout << input << std::endl; - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(true) - .mapping(csv::mapping_type::n_rows); - - ojson j1 = csv::decode_csv(input,options); - std::cout << "\n(1)\n"<< pretty_print(j1) << "\n"; - //CHECK(j1.size() == 4); - - options.mapping(csv::mapping_type::n_objects); - ojson j2 = csv::decode_csv(input,options); - std::cout << "\n(2)\n"<< pretty_print(j2) << "\n"; - - options.mapping(csv::mapping_type::m_columns); - ojson j3 = csv::decode_csv(input,options); - std::cout << "\n(3)\n"<< pretty_print(j3) << "\n"; - //REQUIRE(j2.size() == 3); - //CHECK("2017-01-09" == j2[0]["Date"].as()); -} - -TEST_CASE("csv_options lossless_number") -{ - const std::string input = R"(index_id,observation_date,rate -EUR_LIBOR_06M,2015-10-23,0.0000214 -EUR_LIBOR_06M,2015-10-26,0.0000143 -EUR_LIBOR_06M,2015-10-27,0.0000001 -)"; - - std::cout << input << std::endl; - - json_decoder decoder; - - csv::csv_options options; - options.assume_header(true) - .mapping(csv::mapping_type::n_objects) - .trim(true) - .lossless_number(true); - - ojson j1 = csv::decode_csv(input,options); - std::cout << pretty_print(j1) << "\n"; - REQUIRE(j1.size() == 3); - CHECK((j1[0]["rate"].as() == "0.0000214")); -} - -// Test case contributed by karimhm -TEST_CASE("csv detect bom") -{ - const std::string input = "\xEF\xBB\xBFstop_id,stop_name,stop_lat,stop_lon,location_type,parent_station\n" -"\"8220B007612\",\"Hotel Merrion Street\",\"53\",\"-6\",\"\",\"\""; - - csv::csv_options options; - options.assume_header(true); - - std::istringstream is(input); - ojson j = csv::decode_csv(is, options); - REQUIRE(j.size() == 1); - ojson station = j[0]; - - try { - auto it = station.find("stop_id"); - REQUIRE((it != station.object_range().end())); - std::string stop_id = it->value().as(); - CHECK((stop_id == "8220B007612")); - } catch (const std::exception& e) { - std::cout << e.what() << std::endl; - } -} - -TEST_CASE("test_encode_decode csv string") -{ - typedef std::vector> cpp_type; - std::string s1 = "\"a\",1\n\"b\",2"; - csv::csv_options options; - options.mapping(csv::mapping_type::n_rows) - .assume_header(false); - - SECTION("string") - { - cpp_type v = csv::decode_csv(s1, options); - REQUIRE(v.size() == 2); - CHECK(std::get<0>(v[0]) == "a"); - CHECK(std::get<1>(v[0]) == 1); - CHECK(std::get<0>(v[1]) == "b"); - CHECK(std::get<1>(v[1]) == 2); - - std::string s2; - csv::encode_csv(v, s2, options); - - json j1 = csv::decode_csv(s1); - json j2 = csv::decode_csv(s2); - - CHECK(j1 == j2); - } - - SECTION("stream") - { - std::stringstream ss1(s1); - cpp_type v = csv::decode_csv(ss1, options); - REQUIRE(v.size() == 2); - CHECK(std::get<0>(v[0]) == "a"); - CHECK(std::get<1>(v[0]) == 1); - CHECK(std::get<0>(v[1]) == "b"); - CHECK(std::get<1>(v[1]) == 2); - - std::stringstream ss2; - csv::encode_csv(v, ss2, options); - - json j1 = csv::decode_csv(s1); - json j2 = csv::decode_csv(ss2); - - CHECK(j1 == j2); - } -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/double_round_trip_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/double_round_trip_tests.cpp deleted file mode 100644 index 53b8b44f02..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/double_round_trip_tests.cpp +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("test_round_trip") -{ - { - std::ostringstream os; - double d = 42.229999999999997; - json j = d; - os << j; - CHECK(json::parse(os.str()).as() == d); - } - { - std::ostringstream os; - double d = 9.0099999999999998; - json j = d; - os << j; - CHECK(json::parse(os.str()).as() == d); - } - { - std::ostringstream os; - double d = 13.449999999999999; - json j = d; - os << j; - CHECK(json::parse(os.str()).as() == d); - } - { - std::ostringstream os; - double d = 0.000071; - json j = d; - os << j; - CHECK(json::parse(os.str()).as() == d); - } -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/double_to_string_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/double_to_string_tests.cpp deleted file mode 100644 index 9abec2bb37..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/double_to_string_tests.cpp +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -template -std::basic_string float_to_string(double val, int precision) -{ - jsoncons::detail::print_double print(floating_point_options(chars_format::general,precision,0)); - - std::basic_string s; - jsoncons::string_result> writer(s); - print(val, writer); - writer.flush(); - return s; -} - -TEST_CASE("test_double_to_string") -{ - double x = 1.0e100; - std::string s = float_to_string(x, std::numeric_limits::digits10); - //std::cout << x << ": " << s << std::endl; - CHECK(s == std::string("1e+100")); - - x = 1.0e-100; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::string("1e-100")); - - x = 0.123456789e-100; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::string("1.23456789e-101")); - - x = 0.123456789e100; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::string("1.23456789e+99")); - - x = 1234563; - s = float_to_string(x, 6); - CHECK(s == std::string("1.23456e+06")); - - x = 0.0000001234563; - s = float_to_string(x, 6); - CHECK(s == std::string("1.23456e-07")); - - x = -1.0e+100; - s = float_to_string(x, std::numeric_limits::digits10); - std::cout << s << "\n"; - CHECK(s == std::string("-1e+100")); - - x = -1.0e-100; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::string("-1e-100")); - - x = 0; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::string("0.0")); - - x = -0; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::string("0.0")); - - x = 1; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::string("1.0")); - - x = 0.1; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::string("0.1")); - - x = 1.1; - s = float_to_string(x, 17); - CHECK(s == std::string("1.1000000000000001")); - - x = -1; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::string("-1.0")); - - x = 10; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::string("10.0")); - - x = -10; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::string("-10.0")); - - x = 11; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::string("11.0")); - - x = -11; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::string("-11.0")); -} - -#if defined(_MSC_VER) -TEST_CASE("test_locale") -{ - wchar_t * loc = _wsetlocale(LC_ALL, L"de-DE"); - CHECK_FALSE(loc == nullptr); - - double x = 123456789.0123; - std::wstring s = float_to_string(x, 13); - //std::wcout << std::setprecision(13) << x << L": " << s << std::endl; - CHECK(std::wstring(L"123456789.0123") == s); - _wsetlocale(LC_ALL, L"C"); -} -#endif - -TEST_CASE("test_double_to_wstring") -{ - double x = 1.0e100; - std::wstring s = float_to_string(x, std::numeric_limits::digits10); - //std::wcout << x << L":" << s << std::endl; - CHECK(s == std::wstring(L"1e+100")); - - x = 1.0e-100; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::wstring(L"1e-100")); - - x = -1.0e+100; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::wstring(L"-1e+100")); - - x = -1.0e-100; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::wstring(L"-1e-100")); - - x = 0; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::wstring(L"0.0")); - - x = -0; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::wstring(L"0.0")); - - x = 1; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::wstring(L"1.0")); - - x = -1; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::wstring(L"-1.0")); - - x = 10; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::wstring(L"10.0")); - - x = -10; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::wstring(L"-10.0")); - - x = 11; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::wstring(L"11.0")); - - x = -11; - s = float_to_string(x, std::numeric_limits::digits10); - CHECK(s == std::wstring(L"-11.0")); -} - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/dtoa_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/dtoa_tests.cpp deleted file mode 100644 index b8e3a4cfd5..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/dtoa_tests.cpp +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include - -using namespace jsoncons; - -static void check_safe_dtoa(double x, const std::vector& expected) -{ - std::string s; - bool result = jsoncons::detail::dtoa(x, '.', s, std::false_type()); - if (!result) - { - std::cout << "safe dtoa failed " << s << "\n"; - } - REQUIRE(result); - - bool accept = false; - for (size_t i = 0; !accept && i < expected.size(); ++i) - { - accept = s == expected[i]; - } - if (!accept) - { - std::cout << "safe dtoa does not match expected " << x << " " << s << "\n"; - } - - CHECK(accept); -} - -static void check_dtoa(double x, const std::vector& expected) -{ - std::string s; - bool result = jsoncons::detail::dtoa(x, '.', s); - if (!result) - { - std::cout << "dtoa failed " << s << "\n"; - } - REQUIRE(result); - - bool accept = false; - for (size_t i = 0; !accept && i < expected.size(); ++i) - { - accept = s == expected[i]; - } - if (!accept) - { - std::cout << "dtoa does not match expected " << x << " " << s << "\n"; - } - CHECK(accept); - - check_safe_dtoa(x,expected); -} - -TEST_CASE("test grisu3") -{ - check_dtoa(1.0e100, {"1e+100"}); - check_dtoa(1.0e-100, {"1e-100"}); - check_dtoa(0.123456789e-100, {"1.23456789e-101"}); - check_dtoa(0.123456789e100, {"1.23456789e+99"}); - - check_dtoa(1234563, {"1234563.0"}); - - check_dtoa(0.0000001234563, {"1.234563e-07"}); - - check_dtoa(-1.0e+100, {"-1e+100"}); - - check_dtoa(-1.0e-100, {"-1e-100"}); - - check_dtoa(0, {"0.0"}); - check_dtoa(-0, {"0.0"}); - check_dtoa(1, {"1.0"}); - check_dtoa(0.1, {"0.1"}); - - check_dtoa(1.1, {"1.1"}); - - check_dtoa(-1, {"-1.0"}); - check_dtoa(10, {"10.0"}); - check_dtoa(-10, {"-10.0"}); - check_dtoa(-11, {"-11.0"}); - - check_dtoa(12.272727012634277, {"12.272727012634277"}); - - check_dtoa(4094.1111111111113, {"4094.1111111111113"}); - - check_dtoa(0.119942, {"0.119942"}); - - check_dtoa(-36.973846435546875, {"-36.973846435546875"}); - - check_dtoa(42.229999999999997, {"42.23"}); - check_dtoa(9.0099999999999998, {"9.01"}); - check_dtoa(13.449999999999999, {"13.45"}); - - check_dtoa(0.000071, {"7.1e-05"}); -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/error_recovery_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/error_recovery_tests.cpp deleted file mode 100644 index d005d57bcf..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/error_recovery_tests.cpp +++ /dev/null @@ -1,79 +0,0 @@ - -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -class relaxed_error_handler : public parse_error_handler -{ -private: - - bool do_error(std::error_code ec, - const ser_context&) noexcept override - { - if (ec == jsoncons::json_errc::extra_comma) - { - return true; - } - return false; - } -}; - -TEST_CASE("test_array_extra_comma") -{ - relaxed_error_handler err_handler; - - json expected = json::parse("[1,2,3]"); - json val = json::parse("[1,2,3,]", err_handler); - - CHECK(val == expected); -} - -TEST_CASE("test_object_extra_comma") -{ - relaxed_error_handler err_handler; - - json expected = json::parse(R"( - { - "first" : 1, - "second" : 2 - } - )", - err_handler); - - json val = json::parse(R"( - { - "first" : 1, - "second" : 2, - } - )", - err_handler); - - CHECK(val == expected); -} - -TEST_CASE("test_name_without_quotes") -{ - relaxed_error_handler err_handler; - - /*json val = json::parse(R"( - { - first : 1, - second : 2 - } - )", - err_handler); - - std::cout << val << std::endl;*/ -} - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/heap_only_string_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/heap_only_string_tests.cpp deleted file mode 100644 index cce6ed84e6..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/heap_only_string_tests.cpp +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("test_heap_only_string") -{ - std::string input = "Hello World"; - auto s = jsoncons::detail::heap_only_string_factory>::create(input.data(), input.size()); - - //std::cout << s->c_str() << std::endl; - CHECK(input == std::string(s->c_str())); - - jsoncons::detail::heap_only_string_factory>::destroy(s); -} - -TEST_CASE("test_heap_only_string_wchar_t") -{ - std::wstring input = L"Hello World"; - auto s = jsoncons::detail::heap_only_string_factory>::create(input.data(), input.size()); - - //std::wcout << s->c_str() << std::endl; - - CHECK(input == std::wstring(s->c_str())); - - jsoncons::detail::heap_only_string_factory>::destroy(s); -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_array_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_array_tests.cpp deleted file mode 100644 index 36bbecfb08..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_array_tests.cpp +++ /dev/null @@ -1,462 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("test_initializer_list_of_integers") -{ - json arr = json::array{0,1,2,3}; - CHECK(arr.is_array()); - CHECK(arr.size() == 4); - for (size_t i = 0; i < arr.size(); ++i) - { - CHECK(i == arr[i].as()); - } -} - -TEST_CASE("test_assignment_to_initializer_list") -{ - json arr = json::array({0,1,2,3}); - - arr = json::array{0,1,2,3}; - CHECK(arr.is_array()); - CHECK(arr.size() == 4); - for (size_t i = 0; i < arr.size(); ++i) - { - CHECK(i == arr[i].as()); - } -} - -TEST_CASE("test_assignment_to_initializer_list2") -{ - json val; - val["data"]["id"] = json::array{0,1,2,3,4,5,6,7}; - val["data"]["item"] = json::array{json::array{2}, - json::array{4,5,2,3}, - json::array{4}, - json::array{4,5,2,3}, - json::array{2}, - json::array{4,5,3}, - json::array{2}, - json::array{4,3}}; - - CHECK(val["data"]["item"][0][0] == json(2)); - CHECK(val["data"]["item"][1][0] == json(4)); - CHECK(val["data"]["item"][2][0] == json(4)); - CHECK(val["data"]["item"][3][0] == json(4)); - CHECK(val["data"]["item"][4][0] == json(2)); - CHECK(val["data"]["item"][5][0] == json(4)); - CHECK(val["data"]["item"][6][0] == json(2)); - CHECK(val["data"]["item"][7][0] == json(4)); - CHECK(val["data"]["item"][7][1] == json(3)); -} - -TEST_CASE("test_assignment_to_initializer_list3") -{ - json val; - val["data"]["id"] = json::array{0,1,2,3,4,5,6,7}; - val["data"]["item"] = json::array{json::object{{"first",1},{"second",2}}}; - - json expected_id = json::parse(R"( -[0,1,2,3,4,5,6,7] - )"); - - json expected_item = json::parse(R"( - [{"first":1,"second":2}] - )"); - - CHECK(expected_id == val["data"]["id"]); - CHECK(expected_item == val["data"]["item"]); -} - -TEST_CASE("test_assign_initializer_list_of_object") -{ - json arr = json::array(); - - json transaction; - transaction["Debit"] = 10000; - - arr = json::array{transaction}; - CHECK(arr.is_array()); - CHECK(arr.size() == 1); - CHECK(arr[0] == transaction); -} - -TEST_CASE("test_initializer_list_of_objects") -{ - json book1; - book1["author"] = "Smith"; - book1["title"] = "Old Bones"; - - json book2; - book2["author"] = "Jones"; - book2["title"] = "New Things"; - - json arr = json::array{book1, book2}; - CHECK(arr.is_array()); - CHECK(arr.size() == 2); - - CHECK(book1 == arr[0]); - CHECK(book2 == arr[1]); -} - -TEST_CASE("test_array_constructor") -{ - json arr = json::array(); - arr.resize(10,10.0); - CHECK(arr.is_array()); - CHECK(arr.size() == 10); - CHECK(arr[0].as() == Approx(10.0).epsilon(0.0000001)); -} - -TEST_CASE("test_make_array") -{ - json arr = json::array(); - CHECK(arr.size() == 0); - arr.resize(10,10.0); - CHECK(arr.is_array()); - CHECK(arr.size() == 10); - CHECK(arr[0].as() == Approx(10.0).epsilon(0.0000001)); - -} - -TEST_CASE("test_add_element_to_array") -{ - json arr = json::array(); - CHECK(arr.is_array()); - CHECK(arr.is()); - arr.push_back("Toronto"); - arr.push_back("Vancouver"); - arr.insert(arr.array_range().begin(),"Montreal"); - - CHECK(arr.size() == 3); - - CHECK(arr[0].as() == std::string("Montreal")); - CHECK(arr[1].as() == std::string("Toronto")); - CHECK(arr[2].as() == std::string("Vancouver")); -} - -TEST_CASE("test_emplace_element_to_array") -{ - json a = json::array(); - CHECK(a.is_array()); - CHECK(a.is()); - a.emplace_back("Toronto"); - a.emplace_back("Vancouver"); - a.emplace(a.array_range().begin(),"Montreal"); - - CHECK(a.size() == 3); - - CHECK(a[0].as() == std::string("Montreal")); - CHECK(a[1].as() == std::string("Toronto")); - CHECK(a[2].as() == std::string("Vancouver")); -} - -TEST_CASE("test_array_add_pos") -{ - json arr = json::array(); - CHECK(arr.is_array()); - CHECK(arr.is()); - arr.push_back("Toronto"); - arr.push_back("Vancouver"); - arr.insert(arr.array_range().begin(),"Montreal"); - - CHECK(arr.size() == 3); - - CHECK(arr[0].as() == std::string("Montreal")); - CHECK(arr[1].as() == std::string("Toronto")); - CHECK(arr[2].as() == std::string("Vancouver")); -} - -TEST_CASE("test_array_erase_range") -{ - json arr = json::array(); - CHECK(arr.is_array()); - CHECK(arr.is()); - arr.push_back("Toronto"); - arr.push_back("Vancouver"); - arr.insert(arr.array_range().begin(),"Montreal"); - - CHECK(arr.size() == 3); - - arr.erase(arr.array_range().begin()+1,arr.array_range().end()); - - CHECK(arr.size() == 1); - CHECK(arr[0].as() == std::string("Montreal")); -} - -TEST_CASE("test_reserve_array_capacity") -{ - json cities = json::array(); - CHECK(cities.is_array()); - CHECK(cities.is()); - cities.reserve(10); // storage is allocated - CHECK(cities.capacity() == 10); - CHECK(cities.size() == 0); - - cities.push_back("Toronto"); - CHECK(cities.is_array()); - CHECK(cities.is()); - CHECK(cities.capacity() == 10); - CHECK(cities.size() == 1); - cities.push_back("Vancouver"); - cities.insert(cities.array_range().begin(),"Montreal"); - CHECK(cities.capacity() == 10); - CHECK(cities.size() == 3); -} - -TEST_CASE("test make_array()") -{ - json j = json::make_array(); - CHECK(j.is_array()); - CHECK(j.size() == 0); - j.emplace_back("Toronto"); - j.emplace_back("Vancouver"); - j.emplace(j.array_range().begin(),"Montreal"); - CHECK(j[0].as() == std::string("Montreal")); - CHECK(j[1].as() == std::string("Toronto")); - CHECK(j[2].as() == std::string("Vancouver")); -} - -TEST_CASE("test_one_dim_array") -{ - basic_json> a = basic_json>::make_array<1>(10,0); - CHECK(a.size() == 10); - CHECK(a[0].as() == 0); - a[1] = 1; - a[2] = 2; - CHECK(a[1].as() == 1); - CHECK(a[2].as() == 2); - CHECK(a[9].as() == 0); - - CHECK(a[1].as() == 1); - CHECK(a[2].as() == 2); - CHECK(a[9].as() == 0); -} - -TEST_CASE("test_two_dim_array") -{ - json a = json::make_array<2>(3,4,0); - CHECK(a.size() == 3); - a[0][0] = "Tenor"; - a[0][1] = "ATM vol"; - a[0][2] = "25-d-MS"; - a[0][3] = "25-d-RR"; - a[1][0] = "1Y"; - a[1][1] = 0.20; - a[1][2] = 0.009; - a[1][3] = -0.006; - a[2][0] = "2Y"; - a[2][1] = 0.18; - a[2][2] = 0.009; - a[2][3] = -0.005; - - CHECK(a[0][0].as() ==std::string("Tenor")); - CHECK(a[2][3].as() == Approx(-0.005).epsilon(0.00000001)); - - CHECK(a[0][0].as() ==std::string("Tenor")); - CHECK(a[2][3].as() == Approx(-0.005).epsilon(0.00000001)); -} - -TEST_CASE("test_three_dim_array") -{ - json a = json::make_array<3>(4,3,2,0); - CHECK(a.size() == 4); - a[0][2][0] = 2; - a[0][2][1] = 3; - - CHECK(a[0][2][0].as() == 2); - CHECK(a[0][2][1].as() == 3); - CHECK(a[3][2][1].as() == 0); - - CHECK(a[0][2][0].as() == 2); - CHECK(a[0][2][1].as() == 3); - CHECK(a[3][2][1].as() == 0); -} - -TEST_CASE("test_array_assign_vector") -{ - std::vector vec; - vec.push_back("Toronto"); - vec.push_back("Vancouver"); - vec.push_back("Montreal"); - - json val; - val = vec; - - CHECK(val.size() == 3); - CHECK(val[0].as() ==std::string("Toronto")); - CHECK(val[1].as() ==std::string("Vancouver")); - CHECK(val[2].as() ==std::string("Montreal")); - -} - -TEST_CASE("test_array_assign_vector_of_bool") -{ - std::vector vec; - vec.push_back(true); - vec.push_back(false); - vec.push_back(true); - - json val; - val = vec; - - CHECK(val.size() == 3); - CHECK(val[0].as() == true); - CHECK(val[1].as() == false); - CHECK(val[2].as() == true); - -} - -TEST_CASE("test_array_add_null") -{ - json a = json::array(); - a.push_back(jsoncons::null_type()); - a.push_back(json::null()); - CHECK(a[0].is_null()); - CHECK(a[1].is_null()); -} - -TEST_CASE("test_array_from_container") -{ - std::vector vec; - vec.push_back(10); - vec.push_back(20); - vec.push_back(30); - - json val1 = vec; - REQUIRE(vec.size() == 3); - CHECK(vec[0] == 10); - CHECK(vec[1] == 20); - CHECK(vec[2] == 30); - - std::list list; - list.push_back(10.5); - list.push_back(20.5); - list.push_back(30.5); - - json val2 = list; - REQUIRE(val2.size() == 3); - CHECK(val2[0].as() == Approx(10.5).epsilon(0.000001)); - CHECK(val2[1].as() == Approx(20.5).epsilon(0.000001)); - CHECK(val2[2].as() == Approx(30.5).epsilon(0.000001)); -} - -TEST_CASE("test_array_as_vector_of_double") -{ - std::string s("[0,1.1,2,3.1]"); - json val = json::parse(s); - - std::vector v = val.as>(); - CHECK(v.size() == 4); - CHECK(v[0] == Approx(0.0).epsilon(0.0000000001)); - CHECK(v[1] == Approx(1.1).epsilon(0.0000000001)); - CHECK(v[2] == Approx(2.0).epsilon(0.0000000001)); - CHECK(v[3] == Approx(3.1).epsilon(0.0000000001)); -} - -TEST_CASE("test_array_as_vector_of_bool") -{ - std::string s("[true,false,true]"); - json val = json::parse(s); - - std::vector v = val.as>(); - CHECK(v.size() == 3); - CHECK(v[0] == true); - CHECK(v[1] == false); - CHECK(v[2] == true); -} - -TEST_CASE("test_array_as_vector_of_string") -{ - std::string s("[\"Hello\",\"World\"]"); - json val = json::parse(s); - - std::vector v = val.as>(); - CHECK(v.size() == 2); - CHECK(v[0] == "Hello"); - CHECK(v[1] == "World"); -} - -TEST_CASE("test_array_as_vector_of_char") -{ - std::string s("[20,30]"); - json val = json::parse(s); - - std::vector v = val.as>(); - CHECK(v.size() == 2); - CHECK(v[0] == 20); - CHECK(v[1] == 30); -} - -TEST_CASE("test_array_as_vector_of_int") -{ - std::string s("[0,1,2,3]"); - json val = json::parse(s); - - std::vector v = val.as>(); - CHECK(v.size() == 4); - CHECK(v[0]==0); - CHECK(v[1]==1); - CHECK(v[2]==2); - CHECK(v[3]==3); - - std::vector v1 = val.as>(); - CHECK(v1.size() == 4); - CHECK(v1[0]==0); - CHECK(v1[1]==1); - CHECK(v1[2]==2); - CHECK(v1[3]==3); - - std::vector v2 = val.as>(); - CHECK(v2.size() == 4); - CHECK(v2[0]==0); - CHECK(v2[1]==1); - CHECK(v2[2]==2); - CHECK(v2[3]==3); - - std::vector v3 = val.as>(); - CHECK(v3.size() == 4); - CHECK(v3[0]==0); - CHECK(v3[1]==1); - CHECK(v3[2]==2); - CHECK(v3[3]==3); - - std::vector v4 = val.as>(); - CHECK(v4.size() == 4); - CHECK(v4[0]==0); - CHECK(v4[1]==1); - CHECK(v4[2]==2); - CHECK(v4[3]==3); - - std::vector v5 = val.as>(); - CHECK(v5.size() == 4); - CHECK(v5[0]==0); - CHECK(v5[1]==1); - CHECK(v5[2]==2); - CHECK(v5[3]==3); -} - -TEST_CASE("test_array_as_vector_of_int_on_proxy") -{ - std::string s("[0,1,2,3]"); - json val = json::parse(s); - json root; - root["val"] = val; - std::vector v = root["val"].as>(); - CHECK(v.size() == 4); - CHECK(v[0]==0); - CHECK(v[1]==1); - CHECK(v[2]==2); - CHECK(v[3]==3); -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_as_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_as_tests.cpp deleted file mode 100644 index 9559887ee4..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_as_tests.cpp +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -TEST_CASE("json::as()") -{ - std::string s1("Short"); - jsoncons::json j1(s1); - CHECK(j1.as() == jsoncons::string_view(s1)); - - std::string s2("String to long for short string"); - jsoncons::json j2(s2); - CHECK(j2.as() == jsoncons::string_view(s2)); -} - -TEST_CASE("json::as()") -{ - SECTION("from integer") - { - jsoncons::json j(-1000); - CHECK(bool(j.as() == jsoncons::bignum(-1000))); - } - SECTION("from unsigned integer") - { - jsoncons::json j(1000u); - CHECK(bool(j.as() == jsoncons::bignum(1000u))); - } - SECTION("from double") - { - jsoncons::json j(1000.0); - CHECK(bool(j.as() == jsoncons::bignum(1000.0))); - } - SECTION("from bignum") - { - std::string s = "-18446744073709551617"; - jsoncons::json j(s, jsoncons::semantic_tag::bigint); - CHECK(bool(j.as() == jsoncons::bignum(s))); - } -} - -#if (defined(__GNUC__) || defined(__clang__)) && (!defined(__STRICT_ANSI__) && defined(_GLIBCXX_USE_INT128)) -TEST_CASE("json::as<__int128>()") -{ - std::string s = "-18446744073709551617"; - - jsoncons::detail::to_integer_result<__int128> result = jsoncons::detail::to_integer<__int128>(s.data(),s.size()); - REQUIRE(result.ec == jsoncons::detail::to_integer_errc()); - - jsoncons::json j(s); - - __int128 val = j.as<__int128>(); - CHECK(result.value == val); -} - -TEST_CASE("json::as()") -{ - std::string s = "18446744073709551616"; - - jsoncons::detail::to_integer_result result = jsoncons::detail::to_integer(s.data(),s.size()); - REQUIRE(result.ec == jsoncons::detail::to_integer_errc()); - - jsoncons::json j(s); - - unsigned __int128 val = j.as(); - CHECK(result.value == val); -} -#endif diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_checker_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_checker_tests.cpp deleted file mode 100644 index 00bb54e00c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_checker_tests.cpp +++ /dev/null @@ -1,716 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("test_fail1") -{ - std::string path = "./input/JSON_checker/fail1.json"; - std::fstream is(path); - REQUIRE(is); - CHECK_NOTHROW(json::parse(is)); -} - -TEST_CASE("test_fail2") -{ - std::string in_file = "./input/JSON_checker/fail2.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - } - CHECK(err == jsoncons::json_errc::unexpected_eof); -} - -TEST_CASE("test_fail3") -{ - std::string in_file = "./input/JSON_checker/fail3.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - } - CHECK(err == jsoncons::json_errc::expected_name); -} - -TEST_CASE("test_fail4") -{ - std::string in_file = "./input/JSON_checker/fail4.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - } - CHECK(err == jsoncons::json_errc::extra_comma); -} - -TEST_CASE("test_fail5") -{ - std::string in_file = "./input/JSON_checker/fail5.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - } - CHECK(err == jsoncons::json_errc::expected_value); -} - -TEST_CASE("test_fail6") -{ - std::string in_file = "./input/JSON_checker/fail6.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::expected_value); -} - -TEST_CASE("test_fail7") -{ - std::string in_file = "./input/JSON_checker/fail7.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - } - CHECK(err == jsoncons::json_errc::extra_character); -} - -TEST_CASE("test_fail8") -{ - std::string in_file = "./input/JSON_checker/fail8.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::extra_character); -} - -TEST_CASE("test_fail9") -{ - std::string in_file = "./input/JSON_checker/fail9.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::extra_comma); -} - -TEST_CASE("test_fail10") -{ - std::string in_file = "./input/JSON_checker/fail10.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - } - CHECK(err == jsoncons::json_errc::extra_character); -} - -TEST_CASE("test_fail11") -{ - std::string in_file = "./input/JSON_checker/fail11.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::expected_comma_or_right_brace); -} - -TEST_CASE("test_fail12") -{ - std::string in_file = "./input/JSON_checker/fail12.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::expected_value); -} - -TEST_CASE("test_fail13") -{ - std::string in_file = "./input/JSON_checker/fail13.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::leading_zero); -} - -TEST_CASE("test_fail14") -{ - std::string in_file = "./input/JSON_checker/fail14.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::invalid_number); -} - -TEST_CASE("test_fail15") -{ - std::string in_file = "./input/JSON_checker/fail15.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::illegal_escaped_character); -} - -TEST_CASE("test_fail16") -{ - std::string in_file = "./input/JSON_checker/fail16.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::expected_value); -} - -TEST_CASE("test_fail17") -{ - std::string in_file = "./input/JSON_checker/fail17.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::illegal_escaped_character); -} - -TEST_CASE("test_fail18") -{ - std::error_code err; - - std::string in_file = "./input/JSON_checker/fail18.json"; - std::ifstream is(in_file); - REQUIRE(is); - try - { - json_options options; - options.max_nesting_depth(19); - json::parse(is, options); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::max_depth_exceeded); -} - -TEST_CASE("test_fail19") -{ - std::string in_file = "./input/JSON_checker/fail19.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::expected_colon); -} - -TEST_CASE("test_fail20") -{ - std::string in_file = "./input/JSON_checker/fail20.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::expected_value); -} - -TEST_CASE("test_fail21") -{ - std::string in_file = "./input/JSON_checker/fail21.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::expected_colon); -} - -TEST_CASE("test_fail22") -{ - std::string in_file = "./input/JSON_checker/fail22.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::expected_comma_or_right_bracket); -} - -TEST_CASE("test_fail23") -{ - std::string in_file = "./input/JSON_checker/fail23.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::invalid_value); -} - -TEST_CASE("test_fail24") -{ - std::string in_file = "./input/JSON_checker/fail24.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - // Single quote - CHECK(err == jsoncons::json_errc::single_quote); -} - -TEST_CASE("test_fail25") -{ - std::string in_file = "./input/JSON_checker/fail25.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::illegal_character_in_string); -} - -TEST_CASE("test_fail26") -{ - std::string in_file = "./input/JSON_checker/fail26.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::illegal_escaped_character); -} - -TEST_CASE("test_fail27") -{ - std::string in_file = "./input/JSON_checker/fail27.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::illegal_character_in_string); -} - -TEST_CASE("test_fail28") -{ - std::string in_file = "./input/JSON_checker/fail28.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::illegal_escaped_character); -} - -TEST_CASE("test_fail29") -{ - std::string in_file = "./input/JSON_checker/fail29.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::expected_value); -} - -TEST_CASE("test_fail30") -{ - std::string in_file = "./input/JSON_checker/fail30.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::expected_value); -} - -TEST_CASE("test_fail31") -{ - std::string in_file = "./input/JSON_checker/fail31.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::expected_value); -} - -TEST_CASE("test_fail32") -{ - std::string in_file = "./input/JSON_checker/fail32.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::unexpected_eof); -} - -TEST_CASE("test_fail33") -{ - std::string in_file = "./input/JSON_checker/fail33.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - err = e.code(); - //std::cout << in_file << " " << e.what() << std::endl; - } - CHECK(err == jsoncons::json_errc::expected_comma_or_right_bracket); -} - -TEST_CASE("test_pass1") -{ - std::string in_file = "./input/JSON_checker/pass1.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - std::cout << in_file << " " << e.what() << std::endl; - throw; - } -} - -TEST_CASE("test_pass2") -{ - std::string in_file = "./input/JSON_checker/pass2.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - std::cout << in_file << " " << e.what() << std::endl; - throw; - } -} - -TEST_CASE("test_pass3") -{ - std::string in_file = "./input/JSON_checker/pass3.json"; - std::ifstream is(in_file); - REQUIRE(is); - - std::error_code err; - - try - { - json::parse(is); - } - catch (const ser_error& e) - { - std::cout << in_file << " " << e.what() << std::endl; - throw; - } -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_cursor_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_cursor_tests.cpp deleted file mode 100644 index 0675b039f5..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_cursor_tests.cpp +++ /dev/null @@ -1,336 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("json_cursor string_value test") -{ - std::string s = R"("Tom")"; - std::istringstream is(s); - - json_cursor reader(is); - - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::string_value); - CHECK(reader.current().as() == std::string("Tom")); - CHECK((reader.current().as() == jsoncons::string_view("Tom"))); - reader.next(); - CHECK(reader.done()); -} - -TEST_CASE("json_cursor string_value as test") -{ - std::string s = R"("-100")"; - std::istringstream is(s); - - json_cursor reader(is); - - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::string_value); - CHECK(reader.current().as() == -100); - reader.next(); - CHECK(reader.done()); -} - -TEST_CASE("json_cursor string_value as test") -{ - std::string s = R"("100")"; - std::istringstream is(s); - - json_cursor reader(is); - - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::string_value); - CHECK(reader.current().as() == 100); - CHECK(reader.current().as() == 100); - reader.next(); - CHECK(reader.done()); -} - -TEST_CASE("json_cursor null_value test") -{ - std::string s = "null"; - std::istringstream is(s); - - json_cursor reader(is); - - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::null_value); - reader.next(); - CHECK(reader.done()); -} - -TEST_CASE("json_cursor bool_value test") -{ - std::string s = "false"; - std::istringstream is(s); - - json_cursor reader(is); - - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::bool_value); - reader.next(); - CHECK(reader.done()); -} - -TEST_CASE("json_cursor int64_value test") -{ - std::string s = "-100"; - std::istringstream is(s); - - json_cursor reader(is); - - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::int64_value); - CHECK(reader.current().as() == -100); - reader.next(); - CHECK(reader.done()); -} - -TEST_CASE("json_cursor uint64_value test") -{ - std::string s = "100"; - std::istringstream is(s); - - json_cursor reader(is); - - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::uint64_value); - CHECK(reader.current().as() == 100); - CHECK(reader.current().as() == 100); - reader.next(); - CHECK(reader.done()); -} - -TEST_CASE("json_cursor string_value as bignum test") -{ - std::string s = "-18446744073709551617"; - std::istringstream is("\""+s+"\""); - - json_cursor reader(is); - - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::string_value); - CHECK(s == reader.current().as()); - bignum c = reader.current().as(); - CHECK(bool(bignum("-18446744073709551617") == c)); - reader.next(); - CHECK(reader.done()); -} - -TEST_CASE("json_cursor bigint value as bignum") -{ - std::string s = "-18446744073709551617"; - std::istringstream is(s); - - json_cursor reader(is); - - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::string_value); - CHECK(reader.current().get_semantic_tag() == semantic_tag::bigint); - bignum c = reader.current().as(); - CHECK(bool(bignum(s) == c)); - reader.next(); - CHECK(reader.done()); -} - -TEST_CASE("json_cursor double_value test") -{ - std::string s = "100.0"; - std::istringstream is(s); - - json_cursor reader(is); - - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::double_value); - reader.next(); - CHECK(reader.done()); -} - -TEST_CASE("json_cursor array_value test") -{ - std::string s = R"( - [ - { - "enrollmentNo" : 100, - "firstName" : "Tom", - "lastName" : "Cochrane", - "mark" : 55 - }, - { - "enrollmentNo" : 101, - "firstName" : "Catherine", - "lastName" : "Smith", - "mark" : 95}, - { - "enrollmentNo" : 102, - "firstName" : "William", - "lastName" : "Skeleton", - "mark" : 60 - } - ] - )"; - - std::istringstream is(s); - - json_cursor reader(is); - - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::begin_array); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::begin_object); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::name); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::uint64_value); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::name); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::string_value); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::name); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::string_value); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::name); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::uint64_value); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::end_object); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::begin_object); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::name); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::uint64_value); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::name); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::string_value); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::name); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::string_value); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::name); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::uint64_value); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::end_object); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::begin_object); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::name); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::uint64_value); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::name); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::string_value); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::name); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::string_value); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::name); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::uint64_value); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::end_object); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::end_array); - reader.next(); - CHECK(reader.done()); -} - -TEST_CASE("json_cursor object_value test") -{ - std::string s = R"( - { - "enrollmentNo" : 100, - "firstName" : "Tom", - "lastName" : "Cochrane", - "mark" : 55 - } - )"; - - json_cursor reader(s); - - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::begin_object); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::name); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::uint64_value); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::name); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::string_value); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::name); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::string_value); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::name); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::uint64_value); - reader.next(); - REQUIRE_FALSE(reader.done()); - CHECK(reader.current().event_type() == staj_event_type::end_object); - reader.next(); - CHECK(reader.done()); -} - - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_encode_and_decode_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_encode_and_decode_tests.cpp deleted file mode 100644 index 559f75e844..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_encode_and_decode_tests.cpp +++ /dev/null @@ -1,327 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("encode and decode json") -{ - json j(std::make_pair(false,std::string("foo"))); - - SECTION("string test") - { - std::string s; - encode_json(j, s); - json result = decode_json(s); - CHECK(result == j); - } - - SECTION("stream test") - { - std::stringstream ss; - encode_json(j, ss); - json result = decode_json(ss); - CHECK(result == j); - } -} - -TEST_CASE("encode and decode wjson") -{ - wjson j(std::make_pair(false,std::wstring(L"foo"))); - - SECTION("string test") - { - std::wstring s; - encode_json(j, s); - wjson result = decode_json(s); - CHECK(result == j); - } - - SECTION("stream test") - { - std::wstringstream ss; - encode_json(j, ss); - wjson result = decode_json(ss); - CHECK(result == j); - } -} - -TEST_CASE("convert_pair_test") -{ - auto val = std::make_pair(false,std::string("foo")); - std::string s; - - jsoncons::encode_json(val, s); - - auto result = jsoncons::decode_json>(s); - - CHECK(val == result); -} - -TEST_CASE("convert_vector_test") -{ - std::vector v = {1,2,3,4,5,6}; - - std::string s; - jsoncons::encode_json(v,s); - - auto result = jsoncons::decode_json>(s); - - REQUIRE(v.size() == result.size()); - for (size_t i = 0; i < result.size(); ++i) - { - CHECK(v[i] == result[i]); - } -} - -TEST_CASE("convert_map_test") -{ - std::map m = {{"a",1},{"b",2}}; - - std::string s; - jsoncons::encode_json(m,s); - auto result = jsoncons::decode_json>(s); - - REQUIRE(result.size() == m.size()); - CHECK(m["a"] == result["a"]); - CHECK(m["b"] == result["b"]); -} - -TEST_CASE("convert_array_test") -{ - std::array v{1,2,3,4}; - - std::string s; - jsoncons::encode_json(v,s); - std::array result = jsoncons::decode_json>(s); - REQUIRE(result.size() == v.size()); - for (size_t i = 0; i < result.size(); ++i) - { - CHECK(v[i] == result[i]); - } -} - -#if !(defined(__GNUC__) && __GNUC__ <= 5) -TEST_CASE("convert_tuple_test") -{ - typedef std::map> employee_collection; - - employee_collection employees = - { - {"John Smith",{"Hourly","Software Engineer",10000}}, - {"Jane Doe",{"Commission","Sales",20000}} - }; - - std::string s; - jsoncons::encode_json(employees, s, jsoncons::indenting::indent); - std::cout << "(1)\n" << s << std::endl; - auto employees2 = jsoncons::decode_json(s); - REQUIRE(employees2.size() == employees.size()); - - std::cout << "\n(2)\n"; - for (const auto& pair : employees2) - { - std::cout << pair.first << ": " << std::get<1>(pair.second) << std::endl; - } -} -#endif - -namespace ns { - -struct book -{ - std::string author; - std::string title; - double price; - - friend std::ostream& operator<<(std::ostream& os, const book& b) - { - std::cout << "author: " << b.author << ", title: " << b.title << ", price: " << b.price << "\n"; - return os; - } -}; - -} // namespace ns - -JSONCONS_MEMBER_TRAITS_DECL(ns::book,author,title,price) - -TEST_CASE("book_conversion_test") -{ - ns::book book_list{"Haruki Murakami", "Kafka on the Shore", 25.17}; - - std::string s; - encode_json(book_list, s, indenting::indent); - - std::cout << "s: " << s << "\n"; - -} - -namespace ns { - - struct reputon - { - std::string rater; - std::string assertion; - std::string rated; - double rating; - - friend bool operator==(const reputon& lhs, const reputon& rhs) - { - return lhs.rater == rhs.rater && lhs.assertion == rhs.assertion && - lhs.rated == rhs.rated && lhs.rating == rhs.rating; - } - - friend bool operator!=(const reputon& lhs, const reputon& rhs) - { - return !(lhs == rhs); - }; - }; - - class reputation_object - { - std::string application; - std::vector reputons; - - // Make json_type_traits specializations friends to give accesses to private members - JSONCONS_TYPE_TRAITS_FRIEND; - public: - reputation_object() - { - } - reputation_object(const std::string& application, const std::vector& reputons) - : application(application), reputons(reputons) - {} - - friend bool operator==(const reputation_object& lhs, const reputation_object& rhs) - { - return (lhs.application == rhs.application) && (lhs.reputons == rhs.reputons); - } - - friend bool operator!=(const reputation_object& lhs, const reputation_object& rhs) - { - return !(lhs == rhs); - }; - }; - -} // namespace ns - -// Declare the traits. Specify which data members need to be serialized. -JSONCONS_MEMBER_TRAITS_DECL(ns::reputon, rater, assertion, rated, rating) -JSONCONS_MEMBER_TRAITS_DECL(ns::reputation_object, application, reputons) - -TEST_CASE("reputation_object") -{ - ns::reputation_object val("hiking", { ns::reputon{"HikingAsylum.example.com","strong-hiker","Marilyn C",0.90} }); - - SECTION("1") - { - std::string s; - encode_json(val, s); - auto val2 = decode_json(s); - CHECK(val2 == val); - } - - SECTION("2") - { - std::string s; - encode_json(val, s, indenting::indent); - auto val2 = decode_json(s); - CHECK(val2 == val); - } - - SECTION("3") - { - std::string s; - json_options options; - encode_json(val, s, options, indenting::indent); - auto val2 = decode_json(s, options); - CHECK(val2 == val); - } - - SECTION("4") - { - std::string s; - encode_json(ojson(), val, s); - auto val2 = decode_json(ojson(), s); - CHECK(val2 == val); - } - - SECTION("5") - { - std::string s; - encode_json(ojson(), val, s, indenting::indent); - auto val2 = decode_json(ojson(), s); - CHECK(val2 == val); - } - - SECTION("6") - { - std::string s; - json_options options; - encode_json(ojson(), val, s, options, indenting::indent); - auto val2 = decode_json(ojson(), s, options); - CHECK(val2 == val); - } - - SECTION("os 1") - { - std::stringstream os; - encode_json(val, os); - auto val2 = decode_json(os); - CHECK(val2 == val); - } - - SECTION("os 2") - { - std::stringstream os; - encode_json(val, os, indenting::indent); - auto val2 = decode_json(os); - CHECK(val2 == val); - } - - SECTION("os 3") - { - std::stringstream os; - json_options options; - encode_json(val, os, options, indenting::indent); - auto val2 = decode_json(os, options); - CHECK(val2 == val); - } - - SECTION("os 4") - { - std::stringstream os; - encode_json(ojson(), val, os); - auto val2 = decode_json(ojson(), os); - CHECK(val2 == val); - } - - SECTION("os 5") - { - std::stringstream os; - encode_json(ojson(), val, os, indenting::indent); - auto val2 = decode_json(ojson(), os); - CHECK(val2 == val); - } - - SECTION("os 6") - { - std::stringstream os; - json_options options; - encode_json(ojson(), val, os, options, indenting::indent); - auto val2 = decode_json(ojson(), os, options); - CHECK(val2 == val); - } - -} - - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_encoder_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_encoder_tests.cpp deleted file mode 100644 index 99ea9ced5e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_encoder_tests.cpp +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("test_byte_string_serialization") -{ - const uint8_t bs[] = {'H','e','l','l','o'}; - json j(byte_string_view(bs,sizeof(bs))); - - std::ostringstream os; - os << j; - - std::string expected; - expected.push_back('\"'); - encode_base64url(bs,bs+sizeof(bs),expected); - expected.push_back('\"'); - - //std::cout << expected << " " << os.str() << std::endl; - - CHECK(os.str() == expected); -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_equal_to_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_equal_to_tests.cpp deleted file mode 100644 index d8a45a26e4..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_equal_to_tests.cpp +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("test_object_equals_basic") -{ - json o1; - o1["a"] = 1; - o1["b"] = 2; - o1["c"] = 3; - - json o2; - o2["c"] = 3; - o2["a"] = 1; - o2["b"] = 2; - - CHECK(o1 == o2); - CHECK(o2 == o1); - CHECK_FALSE((o1 != o2)); - CHECK_FALSE((o2 != o1)); -} - -TEST_CASE("test_object_equals_diff_vals") -{ - json o1; - o1["a"] = 1; - o1["b"] = 2; - o1["c"] = 3; - - json o2; - o2["a"] = 1; - o2["b"] = 4; - o2["c"] = 3; - - CHECK_FALSE((o1 == o2)); - CHECK_FALSE((o2 == o1)); - CHECK(o1 != o2); - CHECK(o2 != o1); -} - -TEST_CASE("test_object_equals_diff_el_names") -{ - json o1; - o1["a"] = 1; - o1["b"] = 2; - o1["c"] = 3; - - json o2; - o2["d"] = 1; - o2["e"] = 2; - o2["f"] = 3; - - CHECK_FALSE((o1 == o2)); - CHECK_FALSE((o2 == o1)); - CHECK(o1 != o2); - CHECK(o2 != o1); -} - -TEST_CASE("test_object_equals_diff_sizes") -{ - json o1; - o1["a"] = 1; - o1["b"] = 2; - o1["c"] = 3; - - json o2; - o2["a"] = 1; - o2["b"] = 2; - - CHECK_FALSE((o1 == o2)); - CHECK_FALSE((o2 == o1)); - CHECK(o1 != o2); - CHECK(o2 != o1); -} - -TEST_CASE("test_object_equals_subtle_offsets") -{ - json o1; - o1["a"] = 1; - o1["b"] = 1; - - json o2; - o2["b"] = 1; - o2["c"] = 1; - - CHECK_FALSE((o1 == o2)); - CHECK_FALSE((o2 == o1)); - CHECK(o1 != o2); - CHECK(o2 != o1); -} - -TEST_CASE("test_object_equals_empty_objects") -{ - json def_constructed_1; - json def_constructed_2; - json parsed_1 = json::parse("{}"); - json parsed_2 = json::parse("{}"); - json type_constructed_1 = json::object(); - json type_constructed_2 = json::object(); - - CHECK(def_constructed_1 == def_constructed_1); - CHECK(parsed_1 == parsed_2); - CHECK(type_constructed_1 == type_constructed_2); - - CHECK(def_constructed_1 == parsed_1); - CHECK(def_constructed_1 == type_constructed_1); - CHECK(parsed_1 == type_constructed_1); -} - -TEST_CASE("test_object_equals_empty_arrays") -{ - json parsed_1 = json::parse("[]"); - json parsed_2 = json::parse("[]"); - json type_constructed_1 = json::array(); - json type_constructed_2 = json::array(); - - CHECK(parsed_1 == parsed_2); - CHECK(type_constructed_1 == type_constructed_2); - - CHECK(parsed_1 == type_constructed_1); -} - -TEST_CASE("test_empty_object_equal") -{ - CHECK(json() == json(json::object())); - CHECK(json(json::object()) == json()); -} - -TEST_CASE("test_string_not_equals_empty_object") -{ - json o1("42"); - json o2; - - CHECK(o1 != o2); - CHECK(o2 != o1); -} - -TEST_CASE("test_byte_strings_equal") -{ - json o1(byte_string("123456789")); - json o2(byte_string{'1','2','3','4','5','6','7','8','9'}); - json o3(byte_string{'1','2','3','4','5','6','7','8'}); - - CHECK(o1 == o2); - CHECK(o2 == o1); - CHECK(o3 != o1); - CHECK(o2 != o3); -} - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_exception_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_exception_tests.cpp deleted file mode 100644 index 0d772f91cd..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_exception_tests.cpp +++ /dev/null @@ -1,74 +0,0 @@ - -// Copyright 2016 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("test_object_at") -{ - json a; - REQUIRE_THROWS_AS(a.at("key1"), std::out_of_range); - REQUIRE_THROWS_AS(static_cast(a).at("key1"), std::out_of_range); - - a["key1"] = "value1"; - REQUIRE_THROWS_AS(a.at("key2"), std::out_of_range); - REQUIRE_THROWS_AS(static_cast(a).at("key2"), std::out_of_range); - - json b = json::array(); - REQUIRE_THROWS_AS(b.at("key1"), std::runtime_error); - REQUIRE_THROWS_AS(static_cast(b).at("key1"), std::runtime_error); -} - -TEST_CASE("test_object_find") -{ - json b = json::array(); - b.resize(3); - REQUIRE_THROWS_AS(b.find("key1"), std::runtime_error); - REQUIRE_THROWS_AS(static_cast(b).find("key1"), std::runtime_error); - REQUIRE_THROWS_AS(b.find(std::string("key1")), std::runtime_error); - REQUIRE_THROWS_AS(static_cast(b).find(std::string("key1")), std::runtime_error); -} - -TEST_CASE("test_array_at") -{ - json a = json::array(); - REQUIRE_THROWS_AS(a.at(0), std::out_of_range); - REQUIRE_THROWS_AS(static_cast(a).at(0), std::out_of_range); - - a.resize(3); - REQUIRE_THROWS_AS(a.at(3), std::out_of_range); - REQUIRE_THROWS_AS(static_cast(a).at(3), std::out_of_range); -} - -TEST_CASE("test_object_set") -{ - json b = json::array(); - b.resize(3); - REQUIRE_THROWS_AS(b.insert_or_assign("key1","value1"), std::runtime_error); -} - -TEST_CASE("test_array_add") -{ - json b; - b["key1"] = "value1"; - REQUIRE_THROWS_AS(b.push_back(0), std::runtime_error); -} - -TEST_CASE("test_object_index") -{ - json b; - REQUIRE_THROWS_AS(b["key1"].as(), std::out_of_range); - - b["key1"] = "value1"; - REQUIRE_THROWS_AS(b["key2"].as(), std::out_of_range); -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_filter_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_filter_tests.cpp deleted file mode 100644 index 5ee27c18b3..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_filter_tests.cpp +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -struct warning -{ - std::string name; - size_t line_number; - size_t column_number; -}; - -class name_fix_up_filter : public json_filter -{ - std::string member_name_; -public: - std::vector warnings; - - name_fix_up_filter(json_content_handler& handler) - : json_filter(handler) - { - } - -private: - bool do_name(const string_view_type& name, - const ser_context& context) override - { - member_name_ = std::string(name); - if (member_name_ != "name") - { - this->to_handler().name(name,context); - } - return true; - } - - bool do_string_value(const string_view_type& s, - semantic_tag tag, - const ser_context& context) override - { - if (member_name_ == "name") - { - size_t end_first = s.find_first_of(" \t"); - size_t start_last = s.find_first_not_of(" \t", end_first); - this->to_handler().name("first-name", context); - string_view_type first = s.substr(0, end_first); - this->to_handler().string_value(first, tag, context); - if (start_last != string_view_type::npos) - { - this->to_handler().name("last-name", context); - string_view_type last = s.substr(start_last); - this->to_handler().string_value(last, tag, context); - } - else - { - warnings.push_back(warning{std::string(s), - context.line(), - context.column()}); - } - } - else - { - this->to_handler().string_value(s, tag, context); - } - return true; - } -}; - -TEST_CASE("test_filter") -{ - std::string in_file = "./input/address-book.json"; - std::string out_file = "./output/address-book-new.json"; - std::ifstream is(in_file, std::ofstream::binary); - std::ofstream os(out_file); - - json_encoder encoder(os); - name_fix_up_filter filter(encoder); - json_reader reader(is, filter); - reader.read_next(); - - CHECK(1 == filter.warnings.size()); - CHECK("John" ==filter.warnings[0].name); - CHECK(9 == filter.warnings[0].line_number); - CHECK(26 == filter.warnings[0].column_number); -} - -TEST_CASE("test_filter2") -{ - std::string in_file = "./input/address-book.json"; - std::string out_file = "./output/address-book-new.json"; - std::ifstream is(in_file, std::ofstream::binary); - std::ofstream os(out_file); - - json_encoder encoder(os); - - name_fix_up_filter filter2(encoder); - - rename_object_member_filter filter1("email","email2",filter2); - - json_reader reader(is, filter1); - reader.read_next(); - - CHECK(1 == filter2.warnings.size()); - CHECK("John" ==filter2.warnings[0].name); - CHECK(9 == filter2.warnings[0].line_number); - CHECK(26 == filter2.warnings[0].column_number); -} - -TEST_CASE("test_rename_name") -{ - json j; - try - { - j = json::parse(R"( -{"store": -{"book": [ -{"category": "reference", -"author": "Margaret Weis", -"title": "Dragonlance Series", -"price": 31.96}, {"category": "reference", -"author": "Brent Weeks", -"title": "Night Angel Trilogy", -"price": 14.70 -}]}} -)"); - } - catch (const ser_error& e) - { - std::cout << e.what() << std::endl; - } - CHECK(j["store"]["book"][0]["price"].as() == Approx(31.96).epsilon(0.001)); - - std::stringstream ss; - json_encoder encoder(ss); - rename_object_member_filter filter("price","price2",encoder); - j.dump(filter); - - json j2 = json::parse(ss); - CHECK(j2["store"]["book"][0]["price2"].as() == Approx(31.96).epsilon(0.001)); -} - -TEST_CASE("test_chained_filters") -{ - ojson j = ojson::parse(R"({"first":1,"second":2,"fourth":3,"fifth":4})"); - - json_decoder decoder; - - rename_object_member_filter filter2("fifth", "fourth", decoder); - rename_object_member_filter filter1("fourth", "third", filter2); - - j.dump(filter1); - ojson j2 = decoder.get_result(); - CHECK(j2.size() == 4); - CHECK(j2["first"] == 1); - CHECK(j2["second"] == 2); - CHECK(j2["third"] == 3); - CHECK(j2["fourth"] == 4); -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_integer_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_integer_tests.cpp deleted file mode 100644 index 63de637972..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_integer_tests.cpp +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("test_integer_limits") -{ - { - std::ostringstream os; - - os << "{\"max int64_t\":" << (std::numeric_limits::max)() << "}"; - json val = json::parse(os.str()); - REQUIRE(val["max int64_t"].is_int64()); - CHECK(val["max int64_t"].as() == (std::numeric_limits::max)()); - } - { - std::ostringstream os; - - os << "{\"min int64_t\":" << (std::numeric_limits::lowest)() << "}"; - json val = json::parse(os.str()); - REQUIRE(val["min int64_t"].is_int64()); - CHECK(val["min int64_t"].as() == (std::numeric_limits::lowest)()); - } - - // test overflow - { - std::ostringstream os; - - os << "{\"int overflow\":-" << (std::numeric_limits::max)() << "0}"; - json val = json::parse(os.str()); - REQUIRE(val["int overflow"].is_bignum()); - } - { - std::ostringstream os; - - os << "{\"max uint64_t\":" << (std::numeric_limits::max)() << "}"; - json val = json::parse(os.str()); - REQUIRE(val["max uint64_t"].is_uint64()); - CHECK(val["max uint64_t"].as() == (std::numeric_limits::max)()); - } - { - std::ostringstream os; - - os << "{\"uint overflow\":" << (std::numeric_limits::max)() << "0}"; - json val = json::parse(os.str()); - REQUIRE(val["uint overflow"].is_bignum()); - } -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_less_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_less_tests.cpp deleted file mode 100644 index fe87ca5d0d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_less_tests.cpp +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("json null less") -{ - json j1 = null_type(); - - SECTION("empty object") - { - json j2; - - CHECK(j1 < j2); - CHECK_FALSE(j2 < j1); - } - - SECTION("object") - { - json j2; - j2["a"] = 1; - j2["b"] = 3; - j2["c"] = 3; - - CHECK(j1 < j2); - CHECK_FALSE(j2 < j1); - } -} - -TEST_CASE("json empty object less") -{ - json j1; - - SECTION("empty object") - { - json j2; - - CHECK_FALSE(j1 < j2); - } - - SECTION("object with no members") - { - json j2 = json::object(); - - CHECK_FALSE(j1 < j2); - } - - SECTION("object with members") - { - json j2; - j2["a"] = 1; - j2["b"] = 3; - j2["c"] = 3; - - CHECK(j1 < j2); - } -} - -TEST_CASE("json bool less") -{ - json jtrue = true; - json jfalse = false; - - SECTION("bool") - { - CHECK(jfalse < jtrue); - CHECK_FALSE(jtrue < jfalse); - } - - SECTION("null") - { - json j = null_type(); - CHECK(j < jfalse); - CHECK(j < jtrue); - } -} - -TEST_CASE("json int64 less") -{ - json j1 = -100; -} - -TEST_CASE("json short string less") -{ - json j1 = "bcd"; - - SECTION("short string") - { - json j2 = "cde"; - CHECK(j1 < j2); - CHECK_FALSE(j2 < j1); - json j3 = "bcda"; - CHECK(j1 < j3); - CHECK_FALSE(j3 < j1); - } - - SECTION("long string") - { - json j2 = "string too long for short string"; - CHECK(j1 < j2); - CHECK_FALSE(j2 < j1); - - json j3 = "a string too long for short string"; - CHECK(j3 < j1); - CHECK_FALSE(j1 < j3); - } -} - -TEST_CASE("json long string less") -{ - json j1 = "a string too long for short string"; - - SECTION("short string") - { - json j2 = "a s"; - CHECK(j2 < j1); - CHECK_FALSE(j1 < j2); - json j3 = "bcd"; - CHECK(j1 < j3); - CHECK_FALSE(j3 < j1); - } - - SECTION("long string") - { - json j2 = "string too long for short string"; - CHECK(j1 < j2); - CHECK_FALSE(j2 < j1); - } -} - -TEST_CASE("json array of string less") -{ - json j1 = json::array(); - j1.push_back("b"); - j1.push_back("c"); - j1.push_back("d"); - - SECTION("array") - { - json j2 = json::array(); - j2.push_back("a"); - j2.push_back("b"); - j2.push_back("c"); - - CHECK(j2 < j1); - CHECK_FALSE(j1 < j2); - } -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_line_split_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_line_split_tests.cpp deleted file mode 100644 index ec77cded5d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_line_split_tests.cpp +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; -using namespace jsoncons::literals; - -bool are_equal(const std::string& s1, const std::string& s2) -{ - size_t len1 = s1.size(); - size_t len2 = s2.size(); - - size_t len = std::min(len1,len2); - - for (size_t i = 0; i < len; ++i) - { - if (s1[i] != s2[i]) - { - for (size_t j = 0; j <= i; ++j) - { - std::cout << s1[j]; - } - std::cout << "|"; - std::cout << "\n"; - std::cout << i << " s1: " << s1[i] << ", " << (int)s1[i] << " s2: " << s2[i] << ", " << (int)s2[i] << "\n"; - std::cout << s1 << "\n"; - std::cout << "---\n"; - std::cout << s2 << "\n"; - return false; - } - } - return true; -} - -TEST_CASE("json_encoder line split tests") -{ - json val = json::parse(R"( - { - "header" : {"properties": {}}, - "data": - { - "tags" : [], - "id" : [1,2,3], - "item": [[1,2,3]] - } - } -)"); - - SECTION("Default line splits") - { -std::string expected = R"({ - "data": { - "id": [1,2,3], - "item": [ - [1,2,3] - ], - "tags": [] - }, - "header": { - "properties": {} - } -})"; - json_options options; - options.spaces_around_comma(spaces_option::no_spaces); - std::ostringstream os; - os << pretty_print(val, options); - CHECK(os.str() == expected); - } - - SECTION("array_array same_line") - { - json_options options; - options.spaces_around_comma(spaces_option::no_spaces) - .array_array_line_splits(line_split_kind::same_line); - std::string expected = R"({ - "data": { - "id": [1,2,3], - "item": [[1,2,3]], - "tags": [] - }, - "header": { - "properties": {} - } -})"; - std::ostringstream os; - os << pretty_print(val,options); - - //std::cout << os.str() << "\n"; - CHECK(os.str() == expected); - } - - SECTION("array_array new_line") - { - json_options options; - options.spaces_around_comma(spaces_option::no_spaces) - .array_array_line_splits(line_split_kind::new_line); - std::string expected = R"({ - "data": { - "id": [1,2,3], - "item": [ - [1,2,3] - ], - "tags": [] - }, - "header": { - "properties": {} - } -})"; - std::ostringstream os; - os << pretty_print(val,options); - - //std::cout << os.str() << "\n"; - CHECK(os.str() == expected); - } - - SECTION("array_array multi_line") - { - json_options options; - options.spaces_around_comma(spaces_option::no_spaces) - .array_array_line_splits(line_split_kind::multi_line); - std::string expected = R"({ - "data": { - "id": [1,2,3], - "item": [ - [ - 1, - 2, - 3 - ] - ], - "tags": [] - }, - "header": { - "properties": {} - } -})"; - std::ostringstream os; - os << pretty_print(val,options); - //std::cout << os.str() << "\n"; - CHECK(os.str() == expected); - } - - SECTION("object_array same_line") - { - json_options options; - options.spaces_around_comma(spaces_option::no_spaces) - .object_array_line_splits(line_split_kind::same_line); - std::string expected = R"({ - "data": { - "id": [1,2,3], - "item": [ - [1,2,3] - ], - "tags": [] - }, - "header": { - "properties": {} - } -})"; - std::ostringstream os; - os << pretty_print(val,options); - //std::cout << os.str() << "\n"; - CHECK(os.str() == expected); - } - - SECTION("object_array new_line") - { - json_options options; - options.spaces_around_comma(spaces_option::no_spaces) - .object_array_line_splits(line_split_kind::new_line); - std::string expected = R"({ - "data": { - "id": [ - 1,2,3 - ], - "item": [ - [1,2,3] - ], - "tags": [] - }, - "header": { - "properties": {} - } -})"; - std::ostringstream os; - os << pretty_print(val,options); - //std::cout << os.str() << "\n"; - CHECK(os.str() == expected); - } - - SECTION("") - { - json_options options; - options.spaces_around_comma(spaces_option::no_spaces) - .object_array_line_splits(line_split_kind::multi_line); - std::string expected = R"({ - "data": { - "id": [ - 1, - 2, - 3 - ], - "item": [ - [1,2,3] - ], - "tags": [] - }, - "header": { - "properties": {} - } -})"; - std::ostringstream os; - os << pretty_print(val,options); - //std::cout << os.str() << "\n"; - CHECK(os.str() == expected); - } -} - -// array_array_line_splits_(line_split_kind::new_line) - -TEST_CASE("test_array_of_array_of_string_string_array") -{ - json j = R"( -[ - ["NY","LON", - ["TOR","LON"] - ] -] - )"_json; - - //std::cout << pretty_print(j) << std::endl; -} - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_literal_operator_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_literal_operator_tests.cpp deleted file mode 100644 index 6fcf305a4c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_literal_operator_tests.cpp +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; -using namespace jsoncons::literals; - -TEST_CASE("json_literal_operator_test1") -{ - json j = R"( -{ - "StartDate" : "2017-03-01", - "MaturityDate" : "2020-12-30", - "Currency" : "USD", - "DiscountCurve" : "USD-LIBOR", - "FixedRate" : 0.01, - "PayFrequency" : "6M", - "DayCountBasis" : "ACT/360", - "Notional" : 1000000 -} -)"_json; - - CHECK(j["Currency"] == "USD"); - -} - -TEST_CASE("ojson_literal_operator_test1") -{ - ojson j = R"( -{ - "StartDate" : "2017-03-01", - "MaturityDate" : "2020-12-30", - "Currency" : "USD", - "DiscountCurve" : "USD-LIBOR", - "FixedRate" : 0.01, - "PayFrequency" : "6M", - "DayCountBasis" : "ACT/360", - "Notional" : 1000000 -} -)"_ojson; - - CHECK(j["Currency"] == "USD"); - -} - -TEST_CASE("json_literal_operator_test2") -{ - wjson j = LR"( -{ - "StartDate" : "2017-03-01", - "MaturityDate" : "2020-12-30", - "Currency" : "USD", - "DiscountCurve" : "USD-LIBOR", - "FixedRate" : 0.01, - "PayFrequency" : "6M", - "DayCountBasis" : "ACT/360", - "Notional" : 1000000 -} -)"_json; - - CHECK(j[L"Currency"] == L"USD"); - -} - -TEST_CASE("ojson_literal_operator_test2") -{ - wojson j = LR"( -{ - "StartDate" : "2017-03-01", - "MaturityDate" : "2020-12-30", - "Currency" : "USD", - "DiscountCurve" : "USD-LIBOR", - "FixedRate" : 0.01, - "PayFrequency" : "6M", - "DayCountBasis" : "ACT/360", - "Notional" : 1000000 -} -)"_ojson; - - CHECK(j[L"Currency"] == L"USD"); - -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_member_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_member_tests.cpp deleted file mode 100644 index 638025dc25..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_member_tests.cpp +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("json(string_view)") -{ - json::string_view_type sv("Hello world."); - - json j(sv); - - CHECK(j.as() == sv); - CHECK(j.as_string_view() == sv); -} - -TEST_CASE("json(string, semantic_tag::datetime)") -{ - std::string s("2015-05-07 12:41:07-07:00"); - - json j(s, semantic_tag::datetime); - - CHECK(j.get_semantic_tag() == semantic_tag::datetime); - CHECK(j.as() == s); -} - - -TEST_CASE("json(string, semantic_tag::timestamp)") -{ - SECTION("positive integer") - { - int t = 10000; - json j(t, semantic_tag::timestamp); - - CHECK(j.get_semantic_tag() == semantic_tag::timestamp); - CHECK(j.as() == t); - } - SECTION("negative integer") - { - int t = -10000; - json j(t, semantic_tag::timestamp); - - CHECK(j.get_semantic_tag() == semantic_tag::timestamp); - CHECK(j.as() == t); - } - SECTION("floating point") - { - double t = 10000.1; - json j(t, semantic_tag::timestamp); - - CHECK(j.get_semantic_tag() == semantic_tag::timestamp); - CHECK(j.as() == t); - } - -} - -TEST_CASE("json get_allocator() tests") -{ - SECTION("short string") - { - json j("short"); - - CHECK(j.get_allocator() == json::allocator_type()); - } - SECTION("long string") - { - json::allocator_type alloc; - json j("string too long for short string", alloc); - - CHECK(j.get_allocator() == alloc); - } - SECTION("byte string") - { - json::allocator_type alloc; - json j(byte_string({'H','e','l','l','o'}),alloc); - - CHECK(j.get_allocator() == alloc); - } - SECTION("array") - { - json::allocator_type alloc; - json j = json::array(alloc); - - CHECK(j.get_allocator() == alloc); - } - SECTION("object") - { - json::allocator_type alloc; - json j(alloc); - - CHECK(j.get_allocator() == alloc); - } -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_object_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_object_tests.cpp deleted file mode 100644 index 6c8b805904..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_object_tests.cpp +++ /dev/null @@ -1,1219 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("json = json::object(first,last)") -{ - SECTION("copy map into json") - { - std::map m = {{"c",1},{"b",2},{"a",3}}; - - json j = json::object(m.begin(),m.end()); - - REQUIRE(j.size() == 3); - auto it = j.object_range().begin(); - CHECK(it++->key() == "a"); - CHECK(it++->key() == "b"); - CHECK(it++->key() == "c"); - } -} - -TEST_CASE("json insert(first,last) test") -{ - SECTION("copy map into json") - { - std::map m1 = {{"f",4},{"e",5},{"d",6}}; - std::map m2 = {{"c",1},{"b",2},{"a",3}}; - - json j; - j.insert(m1.begin(),m1.end()); - j.insert(m2.begin(),m2.end()); - - //std::cout << j << "\n"; - - REQUIRE(j.size() == 6); - auto it = j.object_range().begin(); - CHECK(it++->key() == "a"); - CHECK(it++->key() == "b"); - CHECK(it++->key() == "c"); - CHECK(it++->key() == "d"); - CHECK(it++->key() == "e"); - CHECK(it++->key() == "f"); - } - SECTION("copy map into ojson") - { - std::map m1 = {{"f",4},{"e",5},{"d",6}}; - std::map m2 = {{"c",1},{"b",2},{"a",3}}; - - ojson j; - j.insert(m1.begin(),m1.end()); - j.insert(m2.begin(),m2.end()); - - //std::cout << j << "\n"; - - REQUIRE(j.size() == 6); - auto it = j.object_range().begin(); - CHECK(it++->key() == "d"); - CHECK(it++->key() == "e"); - CHECK(it++->key() == "f"); - CHECK(it++->key() == "a"); - CHECK(it++->key() == "b"); - CHECK(it++->key() == "c"); - } - - // Fails with xenial-armhf -/* - SECTION("move map into json") - { - std::map m1 = {{"a",1},{"b",2},{"c",3}}; - std::map m2 = {{"d",4},{"e",5},{"f",6}}; - - json j; - j.insert(std::make_move_iterator(m1.begin()),std::make_move_iterator(m1.end())); - j.insert(std::make_move_iterator(m2.begin()),std::make_move_iterator(m2.end())); - - //std::cout << j << "\n"; - - REQUIRE(j.size() == 6); - auto it = j.object_range().begin(); - CHECK(it++->key() == "a"); - CHECK(it++->key() == "b"); - CHECK(it++->key() == "c"); - CHECK(it++->key() == "d"); - CHECK(it++->key() == "e"); - CHECK(it++->key() == "f"); - } -*/ -} - -TEST_CASE("json as") -{ - SECTION("empty object as string") - { - json j; - std::string s = j.as(); - CHECK("{}" == s); - } - - SECTION("key not found") - { - try - { - json j; - std::string s = j["empty"].as(); - CHECK(false); - } - catch (const std::out_of_range& e) - { - CHECK(e.what() == std::string("Key 'empty' not found")); - } - } -} - -TEST_CASE("parse_duplicate_names") -{ - json j1 = json::parse(R"({"first":1,"second":2,"third":3})"); - CHECK(3 == j1.size()); - CHECK(1 == j1["first"].as()); - CHECK(2 == j1["second"].as()); - CHECK(3 == j1["third"].as()); - - json j2 = json::parse(R"({"first":1,"second":2,"first":3})"); - CHECK(2 == j2.size()); - CHECK(1 == j2["first"].as()); - CHECK(2 == j2["second"].as()); -} - -TEST_CASE("test_erase_member") -{ - json o; - o["key"] = "Hello"; - - CHECK(o.size() == 1); - o.erase("key"); - CHECK(o.size() == 0); - - json a; - json b = json::object(); - b["input-file"] = "config_file"; - json b_copy = b; - - a["b"] = std::move(b); - - CHECK(true == a["b"].is_object()); - CHECK(a["b"] == b_copy); -} - -TEST_CASE("test_object_erase_range") -{ - json o; - o["key1"] = "value1"; - o["key2"] = "value2"; - o["key3"] = "value3"; - o["key4"] = "value4"; - - auto first = o.find("key2"); - auto last = o.find("key4"); - - o.erase(first,last); - - CHECK(2 == o.size()); - CHECK(1 == o.count("key1")); - CHECK(1 == o.count("key4")); -} - -TEST_CASE("test_empty_object") -{ - json a; - CHECK(a.size() == 0); - CHECK(a.is_object()); - CHECK(a.is()); - - json::object_iterator begin = a.object_range().begin(); - json::object_iterator end = a.object_range().end(); - - for (json::object_iterator it = begin; it != end; ++it) - { - CHECK(false); - } - - a["key"] = "Hello"; - CHECK(a.size() == 1); - CHECK(a.is_object()); - CHECK(a.is()); -} - -TEST_CASE("test_const_empty_object") -{ - const json b; - CHECK(b.size() == 0); - CHECK(b.is_object()); - CHECK(b.is()); - - json::const_object_iterator begin = b.object_range().begin(); - json::const_object_iterator end = b.object_range().end(); - - for (json::const_object_iterator it = begin; it != end; ++it) - { - CHECK(false); - } -} - -TEST_CASE("test_empty_object_reserve") -{ - json c; - CHECK(c.size() == 0); - CHECK(c.is_object()); - CHECK(c.is()); - c.reserve(100); - CHECK(c.capacity() == 100); - c["key"] = "Hello"; - CHECK(c.size() == 1); - CHECK(c.is_object()); - CHECK(c.is()); - CHECK(c.capacity() == 100); -} - -TEST_CASE("test_empty_object_copy") -{ - json a; - CHECK(a.size() == 0); - CHECK(a.is_object()); - CHECK(a.is()); - - json b = a; - CHECK(b.size() == 0); - CHECK(b.is_object()); - CHECK(b.is()); -} - -TEST_CASE("test_empty_object_assignment") -{ - json a; - CHECK(a.size() == 0); - CHECK(a.is_object()); - CHECK(a.is()); - - json b = json::make_array<1>(10); - CHECK(b.size() == 10); - CHECK(b.is_array()); - CHECK(b.is()); - - b = a; - CHECK(b.size() == 0); - CHECK(b.is_object()); - CHECK(b.is()); - - json c; - c["key"] = "value"; - CHECK(c.size() == 1); - CHECK(c.is_object()); - CHECK(c.is()); - c = a; - CHECK(c.size() == 0); - CHECK(c.is_object()); - CHECK(c.is()); -} - -TEST_CASE("test_get") -{ - json a; - - a["field1"] = "value1"; - - std::string s1 = a.at("field1").as(); - std::string s1a = a.at("field1").as(); - std::string s2 = a.get_with_default("field2","null"); - REQUIRE_THROWS_AS(a.at("field2"), std::out_of_range); - - CHECK(s1 == std::string("value1")); - CHECK(s1a == std::string("value1")); - - //std::cout << "s2=" << s2 << std::endl; - CHECK(std::string("null") == s2); -} - -TEST_CASE("test_proxy_get") -{ - json a; - - a["object1"] = json(); - a["object1"]["field1"] = "value1"; - - std::string s1 = a["object1"].at("field1").as(); - std::string s1a = a["object1"].at("field1").as(); - std::string s2 = a["object1"].get_with_default("field2",json::null()).as(); - CHECK(a["object1"].get_with_default("field2", json::null()).is_null()); - //std::cout << s2 << std::endl; - REQUIRE_THROWS_AS(a["object1"].at("field2").as(), std::out_of_range); - - CHECK(std::string("value1") == s1); - CHECK(std::string("value1") == s1a); - CHECK(std::string("null") == s2); -} - -TEST_CASE("test_proxy_get_with_default") -{ - json a; - - a["object1"] = json(); - a["object1"]["field1"] = "3.7"; - a["object1"]["field2"] = 1.5; - - std::string s1 = a["object1"].get_with_default("field1","default"); - std::string s2 = a["object1"].get_with_default("field2","1.0"); - std::string s3 = a["object1"].get_with_default("field3","1.0"); - std::string s4 = a["object1"].get_with_default("field2","1.0"); - std::string s5 = a["object1"].get_with_default("field3","1.0"); - double d1 = a["object1"].get_with_default("field1",1.0); - double d2 = a["object1"].get_with_default("field2",1.0); - double d3 = a["object1"].get_with_default("field3",1.0); - - CHECK(std::string("3.7") == s1); - CHECK(std::string("1.5") == s2); - CHECK(std::string("1.0") == s3); - CHECK(std::string("1.5") == s4); - CHECK(std::string("1.0") == s5); - CHECK(3.7 == d1); - CHECK(1.5 == d2); - CHECK(1 == d3); -} - -TEST_CASE("test_set_and_proxy_set") -{ - json a; - - a.insert_or_assign("object1",json()); - a.insert_or_assign("field1","value1"); - a["object1"].insert_or_assign("field2","value2"); - - CHECK(std::string("value1") == a["field1"].as()); - CHECK(std::string("value2") == a["object1"]["field2"].as()); -} - -TEST_CASE("test_emplace_and_proxy_set") -{ - json a; - - a.try_emplace("object1",json()); - a.try_emplace("field1","value1"); - a["object1"].try_emplace("field2","value2"); - - CHECK(std::string("value1") == a["field1"].as()); - CHECK(std::string("value2") == a["object1"]["field2"].as()); -} - -TEST_CASE("test_const_member_read") -{ - json a; - - a["field1"] = 10; - - a["field2"]; - - const json b(a); - - int val1 = b["field1"].as(); - CHECK(val1 == 10); - REQUIRE_THROWS_AS(b["field2"], std::out_of_range); -} - -TEST_CASE("test_proxy_const_member_read") -{ - json a; - - a["object1"] = json(); - a["object1"]["field1"] = "value1"; - a["object1"]["field2"]; // No throw yet - - const json b(a); - - std::string s1 = b["object1"]["field1"].as(); - REQUIRE_THROWS_AS(b["object1"]["field2"], std::out_of_range); - - CHECK(s1 == std::string("value1")); -} - -TEST_CASE("test_object_equals") -{ - json a; - a["field1"] = "value1"; - - json b; - b["field1"] = "value1"; - - CHECK(a == b); - - json c; - c["field1"] = 10; - - CHECK_FALSE(a == c); -} - -TEST_CASE("test_json_object_iterator_1") -{ - json a; - a["name1"] = "value1"; - a["name2"] = "value2"; - a["name3"] = "value3"; - - json::object_iterator it = a.object_range().begin(); - CHECK((*it).key() == "name1"); - CHECK((*it).value() == json("value1")); - ++it; - CHECK((*it).key() == "name2"); - CHECK((*it).value() == json("value2")); - - CHECK((*(it++)).key() == "name2"); - CHECK((*it).key() == "name3"); - CHECK((*it).value() == json("value3")); - - CHECK((*(it--)).key() == "name3"); - CHECK((*it).value() == json("value2")); - CHECK((*(--it)).value() == json("value1")); - - json::key_value_type member = *it; - CHECK(member.key() == "name1"); - CHECK(member.value() == json("value1")); -} - -TEST_CASE("test_json_object_iterator_2") -{ - json a; - a["name1"] = "value1"; - a["name2"] = "value2"; - a["name3"] = "value3"; - - json::const_object_iterator it = a.object_range().begin(); - CHECK((*it).key() == "name1"); - CHECK((*it).value() == json("value1")); - ++it; - CHECK((*it).key() == "name2"); - CHECK((*it).value() == json("value2")); - - CHECK((*(it++)).key() == "name2"); - CHECK((*it).key() == "name3"); - CHECK((*it).value() == json("value3")); - - CHECK((*(it--)).key() == "name3"); - CHECK((*it).value() == json("value2")); - - CHECK((*(--it)).value() == json("value1")); - - json::key_value_type member = *it; - CHECK(member.key() == "name1"); - CHECK(member.value() == json("value1")); -} - -TEST_CASE("test_json_object_iterator_3") -{ - json a; - a["name1"] = "value1"; - a["name2"] = "value2"; - a["name3"] = "value3"; - - json::const_object_iterator it = static_cast(a).object_range().begin(); - CHECK((it == a.object_range().begin())); - CHECK_FALSE((it == a.object_range().end())); - CHECK((*it).key() == "name1"); - CHECK((*it).value() == json("value1")); - ++it; - CHECK_FALSE((it == a.object_range().end())); - CHECK((*it).key() == "name2"); - CHECK((*it).value() == json("value2")); - - CHECK((*(it++)).key() == "name2"); - CHECK_FALSE((it == a.object_range().end())); - CHECK((*it).key() == "name3"); - CHECK((*it).value() == json("value3")); - - CHECK((*(it--)).key() == "name3"); - CHECK((*it).value() == json("value2")); - - CHECK((*(--it)).value() == json("value1")); - CHECK((it == a.object_range().begin())); - - json::key_value_type member = *it; - CHECK(member.key() == "name1"); - CHECK(member.value() == json("value1")); - - //*it = member; // Don't want this to compile -} - -TEST_CASE("test_object_key_proxy") -{ - json a; - a["key1"] = "value1"; - - json b; - b["key2"] = json(); - b["key2"]["key3"] = std::move(a); - - CHECK_FALSE((a.is_object() || a.is_array() || a.is_string())); -} - -// accessor tests - - -TEST_CASE("test_get_with_string_default") -{ - json example; - - std::string s("too long string for short string"); - std::string result = example.get_with_default("test", s); - CHECK(s == result); -} - -TEST_CASE("test_compare_with_string") -{ - json a; - a["key"] = "value"; - a["key1"] = "value1"; - a["key2"] = "value2"; - CHECK(a["key"] == a["key"]); - CHECK_FALSE((a["key"] == a["key1"])); - CHECK_FALSE((a["key"] == a["key2"])); -} - -TEST_CASE("test_count") -{ - json a; - a["key1"] = "value1"; - a["key2"] = "value2"; - - CHECK(1 == a.count("key1")); - CHECK(1 == a.count("key2")); - CHECK(0 == a.count("key3")); - - json b = json::parse( - "{\"key1\":\"a value\",\"key1\":\"another value\"}" - ); - CHECK(1 == b.count("key1")); -} - -TEST_CASE("test_find") -{ - json obj; - - json::object_iterator it = obj.find("key"); - CHECK((it == obj.object_range().end())); - - obj["key1"] = 10; - obj["key2"] = true; - obj["key3"] = 'c'; - obj["key4"] = "value4"; - - json::object_iterator it2 = obj.find("key"); - CHECK((it2 == obj.object_range().end())); - - json::object_iterator it3 = obj.find("key4"); - CHECK_FALSE((it3 == obj.object_range().end())); - CHECK(std::string("value4") ==it3->value().as()); -} - -TEST_CASE("test_as") -{ - json obj; - obj["field1"] = 10; - obj["field2"] = true; - obj["char_field"] = 'c'; - obj["string_field"] = "char"; - - std::string s = obj["field1"].as(); - CHECK(std::string("10") == s); - int int_val = obj["field2"].as(); - CHECK(1 == int_val); - int short_val = obj["field2"].as(); - CHECK(short_val == 1); - int ushort_val = obj["field2"].as(); - CHECK(ushort_val == static_cast(1)); - char char_val = obj["field2"].as(); - CHECK(int(char_val) == 1); - - CHECK(obj["char_field"].is()); - CHECK_FALSE(obj["string_field"].is()); - - json parent; - parent["child"] = obj; - s = parent["child"]["field1"].as(); - CHECK(s == std::string("10")); - int_val = parent["child"]["field2"].as(); - CHECK(int_val == 1); - short_val = parent["child"]["field2"].as(); - CHECK(short_val == 1); - - //json::object x = parent["child"].as(); - // Compile time error, "as not supported" - - json empty; - CHECK(empty.is_object()); - CHECK(empty.empty()); - - //json::object y = empty.as(); - // Compile time error, "as not supported" -} - -TEST_CASE("test_as2") -{ - json obj; - obj["field1"] = "10"; - obj["field2"] = "-10"; - obj["field3"] = "10.1"; - - CHECK(10 == obj["field1"].as()); - CHECK(-10 ==obj["field2"].as()); - CHECK(10.1 == obj["field3"].as()); -} - -TEST_CASE("test_is") -{ - json obj; - obj["field1"] = 10; - obj["field2"] = -10; - obj["field3"] = 10U; - - CHECK(obj["field1"].get_storage_type() == jsoncons::storage_type::int64_val); - CHECK(obj["field2"].get_storage_type() == jsoncons::storage_type::int64_val); - CHECK(obj["field3"].get_storage_type() == jsoncons::storage_type::uint64_val); - - CHECK_FALSE(obj["field1"].is()); - CHECK(obj["field1"].is()); - CHECK(obj["field1"].is()); - CHECK(obj["field1"].is()); - CHECK(obj["field1"].is()); - CHECK(obj["field1"].is()); - CHECK(obj["field1"].is()); - CHECK(obj["field1"].is()); - CHECK_FALSE(obj["field1"].is()); - - CHECK_FALSE(obj["field2"].is()); - CHECK(obj["field2"].is()); - CHECK(obj["field2"].is()); - CHECK(obj["field2"].is()); - CHECK(obj["field2"].is()); - CHECK_FALSE(obj["field2"].is()); - CHECK_FALSE(obj["field2"].is()); - CHECK_FALSE(obj["field2"].is()); - CHECK_FALSE(obj["field2"].is()); - CHECK_FALSE(obj["field2"].is()); - - CHECK_FALSE(obj["field3"].is()); - CHECK(obj["field3"].is()); - CHECK(obj["field3"].is()); - CHECK(obj["field3"].is()); - CHECK(obj["field3"].is()); - CHECK(obj["field3"].is()); - CHECK(obj["field3"].is()); - CHECK(obj["field3"].is()); - CHECK_FALSE(obj["field3"].is()); -} - -TEST_CASE("test_is2") -{ - json obj = json::parse("{\"field1\":10}"); - - CHECK(obj["field1"].get_storage_type() == jsoncons::storage_type::uint64_val); - - CHECK_FALSE(obj["field1"].is()); - CHECK(obj["field1"].is()); - CHECK(obj["field1"].is()); - CHECK(obj["field1"].is()); - CHECK(obj["field1"].is()); - CHECK(obj["field1"].is()); - CHECK(obj["field1"].is()); - CHECK_FALSE(obj["field1"].is()); -} - -TEST_CASE("test_is_type") -{ - json obj; - CHECK(obj.is_object()); - CHECK(obj.is()); - - // tests for proxy is_type methods - obj["string"] = "val1"; - - CHECK(obj.is_object()); - CHECK(obj.is()); - - CHECK(obj["string"].is_string()); - CHECK(obj["string"].is()); - - obj["double"] = 10.7; - CHECK(obj["double"].is_double()); - CHECK(obj["double"].is()); - - obj["int"] = -10; - CHECK(obj["int"].is_int64()); - CHECK(obj["int"].is()); - - obj["uint"] = 10u; - CHECK(obj["uint"].is_uint64()); - CHECK(obj["uint"].is()); - - obj["long"] = static_cast(10); - CHECK(obj["long"].is_int64()); - CHECK(obj["long"].is()); - - obj["ulong"] = static_cast(10); - CHECK(obj["ulong"].is_uint64()); - CHECK(obj["ulong"].is()); - - obj["longlong"] = static_cast(10); - CHECK(obj["longlong"].is_int64()); - CHECK(obj["longlong"].is()); - - obj["ulonglong"] = static_cast(10); - CHECK(obj["ulonglong"].is_uint64()); - CHECK(obj["ulonglong"].is()); - - obj["true"] = true; - CHECK(obj["true"].is_bool()); - CHECK(obj["true"].is()); - - obj["false"] = false; - CHECK(obj["false"].is_bool()); - CHECK(obj["false"].is()); - - obj["null1"] = json::null(); - CHECK(obj["null1"].is_null()); - - obj["object"] = json(); - CHECK(obj["object"].is_object()); - CHECK(obj["object"].is()); - - obj["array"] = json::array(); - CHECK(obj["array"].is_array()); - CHECK(obj["array"].is()); - - // tests for json is_type methods - - json str = obj["string"]; - CHECK(str.is()); - CHECK(str.is()); -} - -TEST_CASE("test_object_get_defaults") -{ - json obj; - - obj["field1"] = 1; - obj["field3"] = "Toronto"; - - double x1 = obj.contains("field1") ? obj["field1"].as() : 10.0; - double x2 = obj.contains("field2") ? obj["field2"].as() : 20.0; - - - CHECK(x1 == 1.0); - CHECK(x2 == 20.0); - - std::string s1 = obj.get_with_default("field3", "Montreal"); - std::string s2 = obj.get_with_default("field4", "San Francisco"); - - CHECK(s1 =="Toronto"); - CHECK(s2 == "San Francisco"); -} - -TEST_CASE("test_object_accessing") -{ - json obj; - obj["first_name"] = "Jane"; - obj["last_name"] = "Roe"; - obj["events_attended"] = 10; - obj["accept_waiver_of_liability"] = true; - - CHECK(obj["first_name"].as() == "Jane"); - CHECK(obj.at("last_name").as() == "Roe"); - CHECK(obj["events_attended"].as() == 10); - CHECK(obj["accept_waiver_of_liability"].as()); -} - -TEST_CASE("test_value_not_found_and_defaults") -{ - json obj; - obj["first_name"] = "Jane"; - obj["last_name"] = "Roe"; - - try - { - auto val = obj["outdoor_experience"].as(); - CHECK(false); - } - catch (const std::out_of_range& e) - { - CHECK(e.what() == std::string("Key 'outdoor_experience' not found")); - } - - //REQUIRE_THROWS_AS((obj["outdoor_experience"].as()),jsoncons::key_not_found); - //REQUIRE_THROWS_WITH((obj["outdoor_experience"].as()),"Key 'outdoor_experience' not found"); - - std::string experience = obj.contains("outdoor_experience") ? obj["outdoor_experience"].as() : ""; - - CHECK(experience == ""); - - try - { - auto val = obj["first_aid_certification"].as(); - CHECK(false); - } - catch (const std::out_of_range& e) - { - CHECK(e.what() == std::string("Key 'first_aid_certification' not found")); - } - - //REQUIRE_THROWS_AS(obj["first_aid_certification"].as(),std::out_of_range); - //REQUIRE_THROWS_WITH(obj["first_aid_certification"].as(),"Key 'first_aid_certification' not found"); -} - -TEST_CASE("test_set_override") -{ - json obj; - obj["first_name"] = "Jane"; - obj["height"] = 0.9; - - obj["first_name"] = "Joe"; - obj["height"] = "0.3"; - - CHECK(obj["first_name"] == "Joe"); - CHECK(obj["height"].as() == Approx(0.3).epsilon(0.00000000001)); -} - -TEST_CASE("try_emplace tests") -{ - json j = json::parse(R"( - { - "a" : 1, - "b" : 2 - } - )"); - - json expected = json::parse(R"( - { - "a" : 1, - "b" : 2, - "c" : 3 - } - )"); - - SECTION("try_emplace(const string_view_type& name, Args&&... args)") - { - j.try_emplace("c",3); - - CHECK(j == expected); - } - - SECTION("try_emplace(iterator hint, const string_view_type& name, Args&&... args)") - { - json::object_iterator it = j.object_range().begin(); - - j.try_emplace(it,"c",3); - - CHECK(j == expected); - } -} - - -// merge tests - -TEST_CASE("test_json_merge") -{ -json j = json::parse(R"( -{ - "a" : 1, - "b" : 2 -} -)"); -json j2 = j; - -const json source = json::parse(R"( -{ - "a" : 2, - "c" : 3 -} -)"); - -const json expected = json::parse(R"( -{ - "a" : 1, - "b" : 2, - "c" : 3 -} -)"); - - j.merge(source); - CHECK(j.size() == 3); - CHECK(j == expected); - - j2.merge(j2.object_range().begin()+1,source); - CHECK(j2.size() == 3); - CHECK(j2 == expected); - - //std::cout << j << std::endl; -} - -TEST_CASE("test_json_merge_move") -{ -json j = json::parse(R"( -{ - "a" : "1", - "b" : [1,2,3] -} -)"); - json j2 = j; - -json source = json::parse(R"( -{ - "a" : "2", - "c" : [4,5,6] -} -)"); - -json expected = json::parse(R"( -{ - "a" : "1", - "b" : [1,2,3], - "c" : [4,5,6] -} -)"); - - json source2 = source; - - j.merge(std::move(source)); - CHECK(j.size() == 3); - CHECK(j == expected); - - j2.merge(std::move(source2)); - CHECK(j2.size() == 3); - CHECK(j2 == expected); - - //std::cout << source << std::endl; -} - -// merge_or_update tests - -TEST_CASE("test_json_merge_or_update") -{ -json j = json::parse(R"( -{ - "a" : 1, - "b" : 2 -} -)"); -json j2 = j; - -const json source = json::parse(R"( -{ - "a" : 2, - "c" : 3 -} -)"); - -const json expected = json::parse(R"( -{ - "a" : 2, - "b" : 2, - "c" : 3 -} -)"); - - j.merge_or_update(source); - CHECK(j.size() == 3); - CHECK(j == expected); - - j2.merge_or_update(j2.object_range().begin()+1,source); - CHECK(j2.size() == 3); - CHECK(j2 == expected); - - //std::cout << j << std::endl; -} - -TEST_CASE("test_json_merge_or_update_move") -{ -json j = json::parse(R"( -{ - "a" : "1", - "b" : [1,2,3] -} -)"); - json j2 = j; - -json source = json::parse(R"( -{ - "a" : "2", - "c" : [4,5,6] -} -)"); - -json expected = json::parse(R"( -{ - "a" : "2", - "b" : [1,2,3], - "c" : [4,5,6] -} -)"); - - json source2 = source; - - j.merge_or_update(std::move(source)); - CHECK(j.size() == 3); - CHECK(j == expected); - - j2.merge_or_update(std::move(source2)); - CHECK(j2.size() == 3); - CHECK(j2 == expected); - - //std::cout << source << std::endl; -} - -TEST_CASE("ojson parse_duplicate_names") -{ - ojson oj1 = ojson::parse(R"({"first":1,"second":2,"third":3})"); - CHECK(3 == oj1.size()); - CHECK(1 == oj1["first"].as()); - CHECK(2 == oj1["second"].as()); - CHECK(3 == oj1["third"].as()); - - ojson oj2 = ojson::parse(R"({"first":1,"second":2,"first":3})"); - CHECK(2 == oj2.size()); - CHECK(1 == oj2["first"].as()); - CHECK(2 == oj2["second"].as()); -} -TEST_CASE("test_ojson_merge") -{ -ojson j = ojson::parse(R"( -{ - "a" : 1, - "b" : 2 -} -)"); - -const ojson source = ojson::parse(R"( -{ - "a" : 2, - "c" : 3, - "d" : 4, - "b" : 5, - "e" : 6 -} -)"); - - SECTION("merge j with source") - { - const ojson expected = ojson::parse(R"( - { - "a" : 1, - "b" : 2, - "c" : 3, - "d" : 4, - "e" : 6 - } - )"); - j.merge(source); - CHECK(j == expected); - } - - SECTION("merge j") - { - const ojson expected = ojson::parse(R"( -{"a":1,"c":3,"d":4,"b":2,"e":6} - )"); - j.merge(j.object_range().begin()+1,source); - CHECK(j == expected); - } - - //std::cout << j << std::endl; -} - -TEST_CASE("test_ojson_merge_move") -{ -ojson j = ojson::parse(R"( -{ - "a" : "1", - "d" : [1,2,3] -} -)"); - -ojson source = ojson::parse(R"( -{ - "a" : "2", - "c" : [4,5,6] -} -)"); - SECTION("merge source into j") - { - ojson expected = ojson::parse(R"( - { - "a" : "1", - "d" : [1,2,3], - "c" : [4,5,6] - } - )"); - - j.merge(std::move(source)); - CHECK(j == expected); - } - SECTION("merge source into j at begin") - { - ojson expected = ojson::parse(R"( - { - "a" : "1", - "c" : [4,5,6], - "d" : [1,2,3] - } - )"); - - j.merge(j.object_range().begin(),std::move(source)); - CHECK(j == expected); - } - - - //std::cout << "(1)\n" << j << std::endl; - //std::cout << "(2)\n" << source << std::endl; -} - -TEST_CASE("test_ojson_merge_or_update") -{ -ojson j = ojson::parse(R"( -{ - "a" : 1, - "b" : 2 -} -)"); - -const ojson source = ojson::parse(R"( -{ - "a" : 2, - "c" : 3 -} -)"); - - SECTION("merge_or_update source into j") - { - const ojson expected = ojson::parse(R"( - { - "a" : 2, - "b" : 2, - "c" : 3 - } - )"); - j.merge_or_update(source); - CHECK(j == expected); - } - - SECTION("merge_or_update source into j at pos 1") - { - const ojson expected = ojson::parse(R"( - { - "a" : 2, - "c" : 3, - "b" : 2 - } - )"); - j.merge_or_update(j.object_range().begin()+1,source); - CHECK(j == expected); - } - - //std::cout << j << std::endl; -} - -TEST_CASE("test_ojson_merge_or_update_move") -{ -ojson j = ojson::parse(R"( -{ - "a" : "1", - "d" : [1,2,3] -} -)"); - -ojson source = ojson::parse(R"( -{ - "a" : "2", - "c" : [4,5,6] -} -)"); - - SECTION("merge or update j from source") - { - ojson expected = ojson::parse(R"( - { - "a" : "2", - "d" : [1,2,3], - "c" : [4,5,6] - } - )"); - - j.merge_or_update(std::move(source)); - CHECK(j == expected); - } - - SECTION("merge or update j from source at pos") - { - ojson expected = ojson::parse(R"( - { - "a" : "2", - "c" : [4,5,6], - "d" : [1,2,3] - } - )"); - - j.merge_or_update(j.object_range().begin(),std::move(source)); - CHECK(j.size() == 3); - CHECK(j == expected); - } - - - //std::cout << "(1)\n" << j << std::endl; - //std::cout << "(2)\n" << source << std::endl; -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_options_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_options_tests.cpp deleted file mode 100644 index 01090a746d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_options_tests.cpp +++ /dev/null @@ -1,438 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("test_default_nan_replacement") -{ - json obj; - obj["field1"] = std::sqrt(-1.0); - obj["field2"] = 1.79e308 * 1000; - obj["field3"] = -1.79e308 * 1000; - - std::ostringstream os; - os << print(obj); - std::string expected = R"({"field1":null,"field2":null,"field3":null})"; - - CHECK(os.str() == expected); -} - -TEST_CASE("test_write_nan_replacement") -{ - json j; - j["field1"] = std::sqrt(-1.0); - j["field2"] = 1.79e308 * 1000; - j["field3"] = -1.79e308 * 1000; - - json_options options; - options.inf_to_num("1e9999"); - - std::ostringstream os; - os << print(j, options); - std::string expected = R"({"field1":null,"field2":1e9999,"field3":-1e9999})"; - - CHECK(os.str() == expected); -} - -TEST_CASE("test_read_write_read_nan_replacement") -{ - json j; - j["field1"] = std::sqrt(-1.0); - j["field2"] = 1.79e308 * 1000; - j["field3"] = -1.79e308 * 1000; - - json_options options; - options.nan_to_str("NaN") - .inf_to_str("Inf"); - - std::ostringstream os; - os << pretty_print(j, options); - - json j2 = json::parse(os.str(),options); - - json expected; - expected["field1"] = std::nan(""); - expected["field2"] = std::numeric_limits::infinity(); - expected["field3"] = -std::numeric_limits::infinity(); - - CHECK(expected.to_string(options) == j.to_string(options)); - CHECK(expected.to_string() == j.to_string()); -} - -TEST_CASE("object_array empty array") -{ - std::string s = R"( -{ - "foo": [] -} - )"; - - SECTION("same_line") - { - json j = json::parse(s); - - json_options options; - options.object_array_line_splits(line_split_kind::same_line); - - std::ostringstream os; - os << pretty_print(j, options); - - std::string expected = R"({ - "foo": [] -})"; - CHECK(os.str() == expected); - } - - SECTION("new_line") - { - json j = json::parse(s); - - json_options options; - options.object_array_line_splits(line_split_kind::new_line); - - std::ostringstream os; - os << pretty_print(j, options); - - std::string expected = R"({ - "foo": [] -})"; - CHECK(os.str() == expected); - } - - SECTION("multi_line") - { - json j = json::parse(s); - - json_options options; - options.object_array_line_splits(line_split_kind::multi_line); - - std::ostringstream os; - os << pretty_print(j, options); - - std::string expected = R"({ - "foo": [] -})"; - CHECK(os.str() == expected); - } - -} - -TEST_CASE("object_array with/without line_length_limit") -{ - std::string s = R"( -{ - "foo": ["bar", "baz", [1, 2, 3]], - "qux": [1, 2, 3, null, 123, 45.3, 342334, 234] -} - )"; - - SECTION("same_line") - { - std::string expected = R"({ - "foo": ["bar","baz", - [1,2,3] - ], - "qux": [1,2,3,null,123,45.3,342334,234] -})"; - - json j = json::parse(s); - - json_options options; - options.line_length_limit(120) - .spaces_around_comma(spaces_option::no_spaces) - .object_array_line_splits(line_split_kind::same_line); - - std::ostringstream os; - os << pretty_print(j, options); - - CHECK(os.str() == expected); - } - - SECTION("new_line") - { - std::string expected = R"({ - "foo": [ - "bar","baz", - [1,2,3] - ], - "qux": [ - 1,2,3,null,123,45.3,342334,234 - ] -})"; - - json j = json::parse(s); - - json_options options; - options.line_length_limit(120) - .spaces_around_comma(spaces_option::no_spaces) - .object_array_line_splits(line_split_kind::new_line); - - std::ostringstream os; - os << pretty_print(j, options); - - //std::cout << pretty_print(j, options) << "\n"; - CHECK(os.str() == expected); - } - - SECTION("multi_line") - { - std::string expected = R"({ - "foo": [ - "bar", - "baz", - [1,2,3] - ], - "qux": [ - 1, - 2, - 3, - null, - 123, - 45.3, - 342334, - 234 - ] -})"; - - json j = json::parse(s); - - json_options options; - options.spaces_around_comma(spaces_option::no_spaces) - .object_array_line_splits(line_split_kind::multi_line); - - std::ostringstream os; - os << pretty_print(j, options); - - //std::cout << pretty_print(j, options) << "\n"; - CHECK(os.str() == expected); - } - - SECTION("same_line with line length limit") - { - std::string expected = R"({ - "foo": ["bar","baz", - [1,2,3] - ], - "qux": [1,2,3,null, - 123,45.3,342334, - 234 - ] -})"; - - json j = json::parse(s); - - json_options options; - options.line_length_limit(20) - .spaces_around_comma(spaces_option::no_spaces) - .object_array_line_splits(line_split_kind::same_line); - - std::ostringstream os; - os << pretty_print(j, options); - - //std::cout << pretty_print(j, options) << "\n"; - CHECK(os.str() == expected); - } - - SECTION("new_line with line length limit") // Revisit 234 - { - std::string expected = R"({ - "foo": [ - "bar","baz", - [1,2,3] - ], - "qux": [ - 1,2,3,null,123, - 45.3,342334,234 - ] -})"; - - json j = json::parse(s); - - json_options options; - options.line_length_limit(20) - .spaces_around_comma(spaces_option::no_spaces) - .object_array_line_splits(line_split_kind::new_line); - - std::ostringstream os; - os << pretty_print(j, options); - - //std::cout << pretty_print(j, options) << "\n"; - CHECK(os.str() == expected); - } -} - -TEST_CASE("array_object with/without line_length_limit") -{ - std::string s = R"( -[ - { - "author": "Graham Greene", - "title": "The Comedians" - }, - { - "author": "Koji Suzuki", - "title": "ring" - }, - { - "author": "Haruki Murakami", - "title": "A Wild Sheep Chase" - } -] - )"; - SECTION("same_line") - { - std::string expected = R"([{"author": "Graham Greene","title": "The Comedians"},{"author": "Koji Suzuki","title": "ring"},{"author": "Haruki Murakami", - "title": "A Wild Sheep Chase"}])"; - - json j = json::parse(s); - - json_options options; - options.line_length_limit(120) - .spaces_around_comma(spaces_option::no_spaces) - .array_object_line_splits(line_split_kind::same_line); - - std::ostringstream os; - os << pretty_print(j, options); - - //std::cout << pretty_print(j, options) << "\n"; - CHECK(os.str() == expected); - } - - SECTION("new_line") - { - std::string expected = R"([ - {"author": "Graham Greene","title": "The Comedians"}, - {"author": "Koji Suzuki","title": "ring"}, - {"author": "Haruki Murakami","title": "A Wild Sheep Chase"} -])"; - - json j = json::parse(s); - - json_options options; - options.line_length_limit(120) - .spaces_around_comma(spaces_option::no_spaces) - .array_object_line_splits(line_split_kind::new_line); - - std::ostringstream os; - os << pretty_print(j, options); - - //std::cout << pretty_print(j, options) << "\n"; - CHECK(os.str() == expected); - } - - SECTION("multi_line (default)") - { - std::string expected = R"([ - { - "author": "Graham Greene", - "title": "The Comedians" - }, - { - "author": "Koji Suzuki", - "title": "ring" - }, - { - "author": "Haruki Murakami", - "title": "A Wild Sheep Chase" - } -])"; - - json j = json::parse(s); - - json_options options; - options.spaces_around_comma(spaces_option::no_spaces); - - std::ostringstream os; - os << pretty_print(j, options); - - //std::cout << pretty_print(j, options) << "\n"; - CHECK(os.str() == expected); - } - SECTION("same_line with line length limit") - { - std::string expected = R"([{"author": "Graham Greene", - "title": "The Comedians"}, - {"author": "Koji Suzuki", - "title": "ring"}, - {"author": "Haruki Murakami", - "title": "A Wild Sheep Chase"}])"; - - json j = json::parse(s); - - json_options options; - options.line_length_limit(20) - .spaces_around_comma(spaces_option::no_spaces) - .array_object_line_splits(line_split_kind::same_line); - - std::ostringstream os; - os << pretty_print(j, options); - - //std::cout << pretty_print(j, options) << "\n"; - CHECK(os.str() == expected); - } - SECTION("new_line with line length limit") - { - std::string expected = R"([ - {"author": "Graham Greene", - "title": "The Comedians"}, - {"author": "Koji Suzuki", - "title": "ring"}, - {"author": "Haruki Murakami", - "title": "A Wild Sheep Chase"} -])"; - json j = json::parse(s); - - json_options options; - options.line_length_limit(20) - .spaces_around_comma(spaces_option::no_spaces) - .array_object_line_splits(line_split_kind::new_line); - - std::ostringstream os; - os << pretty_print(j, options); - CHECK(os.str() == expected); - - //std::cout << pretty_print(j, options) << "\n"; - } -} - -TEST_CASE("json_options tests") -{ - SECTION("pad_inside_array_brackets") - { - std::string s = R"({ - "foo": [ 1, 2 ] -})"; - - json j = json::parse(s); - - json_options options; - options.pad_inside_array_brackets(true); - - std::ostringstream os; - j.dump(os, options, indenting::indent); - CHECK(os.str() == s); - } - SECTION("pad_inside_object_braces") - { - std::string s = R"([{ "foo": 1 }])"; - - json j = json::parse(s); - - json_options options; - options.pad_inside_object_braces(true) - .array_object_line_splits(line_split_kind::same_line); - - std::ostringstream os; - j.dump(os, options, indenting::indent); - CHECK(os.str() == s); - } -} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_parse_error_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_parse_error_tests.cpp deleted file mode 100644 index 1089292b48..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_parse_error_tests.cpp +++ /dev/null @@ -1,239 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -void test_parse_error(const std::string& text, std::error_code ec) -{ - REQUIRE_THROWS(json::parse(text)); - try - { - json::parse(text); - } - catch (const ser_error& e) - { - if (e.code() != ec) - { - std::cout << text << std::endl; - std::cout << e.code().value() << " " << e.what() << std::endl; - } - CHECK(ec == e.code()); - } -} - -void test_parse_ec(const std::string& text, std::error_code expected) -{ - std::error_code ec; - - std::istringstream is(text); - json_decoder decoder; - json_reader reader(is,decoder); - - reader.read(ec); - //std::cerr << text << std::endl; - //std::cerr << ec.message() - // << " at line " << reader.line() - // << " and column " << reader.column() << std::endl; - - CHECK(ec); - CHECK(ec == expected); -} - -TEST_CASE("test_parse_missing_separator") -{ - std::string jtext = R"({"field1"{}})"; - - test_parse_error(jtext, jsoncons::json_errc::expected_colon); - test_parse_ec(jtext, jsoncons::json_errc::expected_colon); -} - -TEST_CASE("test_invalid_value") -{ - std::string jtext = R"({"field1":ru})"; - - test_parse_error(jtext,jsoncons::json_errc::expected_value); - test_parse_ec(jtext, jsoncons::json_errc::expected_value); -} - - -TEST_CASE("test_unexpected_end_of_file") -{ - std::string jtext = R"({"field1":{})"; - - test_parse_error(jtext, jsoncons::json_errc::unexpected_eof); - test_parse_ec(jtext, jsoncons::json_errc::unexpected_eof); -} - -TEST_CASE("test_value_not_found") -{ - std::string jtext = R"({"name":})"; - - test_parse_error(jtext, jsoncons::json_errc::expected_value); - test_parse_ec(jtext, jsoncons::json_errc::expected_value); -} - -TEST_CASE("test_escaped_characters") -{ - std::string input("[\"\\n\\b\\f\\r\\t\"]"); - std::string expected("\n\b\f\r\t"); - - json o = json::parse(input); - CHECK(expected == o[0].as()); -} - - -TEST_CASE("test_expected_colon") -{ - test_parse_error("{\"name\" 10}", jsoncons::json_errc::expected_colon); - test_parse_error("{\"name\" true}", jsoncons::json_errc::expected_colon); - test_parse_error("{\"name\" false}", jsoncons::json_errc::expected_colon); - test_parse_error("{\"name\" null}", jsoncons::json_errc::expected_colon); - test_parse_error("{\"name\" \"value\"}", jsoncons::json_errc::expected_colon); - test_parse_error("{\"name\" {}}", jsoncons::json_errc::expected_colon); - test_parse_error("{\"name\" []}", jsoncons::json_errc::expected_colon); -} - -TEST_CASE("test_expected_name") -{ - test_parse_error("{10}", jsoncons::json_errc::expected_name); - test_parse_error("{true}", jsoncons::json_errc::expected_name); - test_parse_error("{false}", jsoncons::json_errc::expected_name); - test_parse_error("{null}", jsoncons::json_errc::expected_name); - test_parse_error("{{}}", jsoncons::json_errc::expected_name); - test_parse_error("{[]}", jsoncons::json_errc::expected_name); -} - -TEST_CASE("test_expected_value") -{ - test_parse_error("[tru]", jsoncons::json_errc::invalid_value); - test_parse_error("[fa]", jsoncons::json_errc::invalid_value); - test_parse_error("[n]", jsoncons::json_errc::invalid_value); -} - -TEST_CASE("test_parse_primitive_pass") -{ - json val; - CHECK_NOTHROW((val=json::parse("null"))); - CHECK(val == json::null()); - CHECK_NOTHROW((val=json::parse("false"))); - CHECK(val == json(false)); - CHECK_NOTHROW((val=json::parse("true"))); - CHECK(val == json(true)); - CHECK_NOTHROW((val=json::parse("10"))); - CHECK(val == json(10)); - CHECK_NOTHROW((val=json::parse("1.999"))); - CHECK(val == json(1.999)); - CHECK_NOTHROW((val=json::parse("\"string\""))); - CHECK(val == json("string")); -} - -TEST_CASE("test_parse_empty_structures") -{ - json val; - CHECK_NOTHROW((val=json::parse("{}"))); - CHECK_NOTHROW((val=json::parse("[]"))); - CHECK_NOTHROW((val=json::parse("{\"object\":{},\"array\":[]}"))); - CHECK_NOTHROW((val=json::parse("[[],{}]"))); -} - -TEST_CASE("test_parse_primitive_fail") -{ - test_parse_error("null {}", jsoncons::json_errc::extra_character); - test_parse_error("n ", jsoncons::json_errc::invalid_value); - test_parse_error("nu ", jsoncons::json_errc::invalid_value); - test_parse_error("nul ", jsoncons::json_errc::invalid_value); - test_parse_error("false {}", jsoncons::json_errc::extra_character); - test_parse_error("fals ", jsoncons::json_errc::invalid_value); - test_parse_error("true []", jsoncons::json_errc::extra_character); - test_parse_error("tru ", jsoncons::json_errc::invalid_value); - test_parse_error("10 {}", jsoncons::json_errc::extra_character); - test_parse_error("1a ", jsoncons::json_errc::invalid_number); - test_parse_error("1.999 []", jsoncons::json_errc::extra_character); - test_parse_error("1e0-1", jsoncons::json_errc::invalid_number); - test_parse_error("\"string\"{}", jsoncons::json_errc::extra_character); - test_parse_error("\"string\"[]", jsoncons::json_errc::extra_character); -} - -TEST_CASE("test_multiple") -{ - std::string in="{\"a\":1,\"b\":2,\"c\":3}{\"a\":4,\"b\":5,\"c\":6}"; - //std::cout << in << std::endl; - - std::istringstream is(in); - - jsoncons::json_decoder decoder; - json_reader reader(is,decoder); - - REQUIRE_FALSE(reader.eof()); - reader.read_next(); - CHECK_FALSE(reader.eof()); - json val = decoder.get_result(); - CHECK(1 == val["a"].as()); - - REQUIRE_FALSE(reader.eof()); - reader.read_next(); - CHECK(reader.eof()); - json val2 = decoder.get_result(); - CHECK(4 == val2["a"].as()); -} - -TEST_CASE("test_uinteger_overflow") -{ - uint64_t m = (std::numeric_limits::max)(); - std::string s1 = std::to_string(m); - std::string s2 = s1; - s2.push_back('0'); - - json j1 = json::parse(s1); - CHECK(j1.is_uint64()); - CHECK(m == j1.as()); - - json j2 = json::parse(s2); - CHECK_FALSE(j2.is_uint64()); - CHECK(j2.is_bignum()); - CHECK(s2 == j2.as()); -} -TEST_CASE("test_negative_integer_overflow") -{ - int64_t m = (std::numeric_limits::lowest)(); - std::string s1 = std::to_string(m); - std::string s2 = s1; - s2.push_back('0'); - - json j1 = json::parse(s1); - CHECK(m == j1.as()); - - json j2 = json::parse(s2); - CHECK_FALSE(j2.is_int64()); - CHECK(j2.is_bignum()); - CHECK(s2 == j2.as()); -} - -TEST_CASE("test_positive_integer_overflow") -{ - int64_t m = (std::numeric_limits::max)(); - std::string s1 = std::to_string(m); - std::string s2 = s1; - s2.push_back('0'); - - json j1 = json::parse(s1); - CHECK(m == j1.as()); - - json j2 = json::parse(s2); - CHECK_FALSE(j2.is_int64()); - CHECK(j2.is_bignum()); - CHECK(s2 == j2.as()); -} - - - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_parser_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_parser_tests.cpp deleted file mode 100644 index a215f46738..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_parser_tests.cpp +++ /dev/null @@ -1,279 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("Test cyrillic.json") -{ - std::string path = "./input/cyrillic.json"; - std::fstream is(path); - if (!is) - { - std::cout << "Cannot open " << path << std::endl; - } - REQUIRE(is); - json j = json::parse(is); -} - -TEST_CASE("test_object2") -{ -json source = json::parse(R"( -{ - "a" : "2", - "c" : [4,5,6] -} -)"); - - std::cout << source << std::endl; -} - -TEST_CASE("test_object_with_three_members") -{ - std::string input = "{\"A\":\"Jane\", \"B\":\"Roe\",\"C\":10}"; - json val = json::parse(input); - - CHECK(true == val.is_object()); - CHECK(3 == val.size()); -} - -TEST_CASE("test_double") -{ - json val = json::parse("42.229999999999997"); -} - -TEST_CASE("test_array_of_integer") -{ - std::string s = "[1,2,3]"; - json j1 = json::parse(s); - CHECK(true == j1.is_array()); - CHECK(3 == j1.size()); - - std::istringstream is(s); - json j2 = json::parse(is); - CHECK(true == j2.is_array()); - CHECK(3 == j2.size()); -} - -TEST_CASE("test_skip_bom") -{ - std::string s = "\xEF\xBB\xBF[1,2,3]"; - json j1 = json::parse(s); - CHECK(true == j1.is_array()); - CHECK(3 == j1.size()); - - std::istringstream is(s); - json j2 = json::parse(is); - CHECK(true == j2.is_array()); - CHECK(3 == j2.size()); -} - -TEST_CASE("test_parse_empty_object") -{ - jsoncons::json_decoder decoder; - json_parser parser; - - parser.reset(); - - static std::string s("{}"); - - parser.update(s.data(),s.length()); - parser.parse_some(decoder); - parser.finish_parse(decoder); - CHECK(parser.done()); - - json j = decoder.get_result(); -} - -TEST_CASE("test_parse_array") -{ - jsoncons::json_decoder decoder; - json_parser parser; - - parser.reset(); - - static std::string s("[]"); - - parser.update(s.data(),s.length()); - parser.parse_some(decoder); - parser.finish_parse(decoder); - CHECK(parser.done()); - - json j = decoder.get_result(); -} - -TEST_CASE("test_parse_string") -{ - jsoncons::json_decoder decoder; - json_parser parser; - - parser.reset(); - - static std::string s("\"\""); - - parser.update(s.data(),s.length()); - parser.parse_some(decoder); - parser.finish_parse(decoder); - CHECK(parser.done()); - - json j = decoder.get_result(); -} - -TEST_CASE("test_parse_integer") -{ - jsoncons::json_decoder decoder; - json_parser parser; - - parser.reset(); - - static std::string s("10"); - - parser.update(s.data(),s.length()); - parser.parse_some(decoder); - parser.finish_parse(decoder); - CHECK(parser.done()); - - json j = decoder.get_result(); -} - -TEST_CASE("test_parse_integer_space") -{ - jsoncons::json_decoder decoder; - json_parser parser; - - parser.reset(); - - static std::string s("10 "); - - parser.update(s.data(),s.length()); - parser.parse_some(decoder); - parser.finish_parse(decoder); - CHECK(parser.done()); - - json j = decoder.get_result(); -} - -TEST_CASE("test_parse_double_space") -{ - jsoncons::json_decoder decoder; - json_parser parser; - - parser.reset(); - - static std::string s("10.0 "); - - parser.update(s.data(),s.length()); - parser.parse_some(decoder); - parser.finish_parse(decoder); - CHECK(parser.done()); - - json j = decoder.get_result(); -} - -TEST_CASE("test_parse_false") -{ - jsoncons::json_decoder decoder; - json_parser parser; - - parser.reset(); - - static std::string s("false"); - - parser.update(s.data(),s.length()); - parser.parse_some(decoder); - parser.finish_parse(decoder); - CHECK(parser.done()); - - json j = decoder.get_result(); -} - -TEST_CASE("test_parse_true") -{ - jsoncons::json_decoder decoder; - json_parser parser; - - parser.reset(); - - static std::string s("true"); - - parser.update(s.data(),s.length()); - parser.parse_some(decoder); - parser.finish_parse(decoder); - CHECK(parser.done()); - - json j = decoder.get_result(); -} - -TEST_CASE("test_parse_null") -{ - jsoncons::json_decoder decoder; - json_parser parser; - - parser.reset(); - - static std::string s("null"); - - parser.update(s.data(),s.length()); - parser.parse_some(decoder); - parser.finish_parse(decoder); - CHECK(parser.done()); - - json j = decoder.get_result(); -} - -TEST_CASE("test_parse_array_string") -{ - jsoncons::json_decoder decoder; - json_parser parser; - - parser.reset(); - - static std::string s1("[\"\""); - - parser.update(s1.data(),s1.length()); - parser.parse_some(decoder); - CHECK_FALSE(parser.done()); - static std::string s2("]"); - parser.update(s2.data(), s2.length()); - parser.parse_some(decoder); - parser.finish_parse(decoder); - CHECK(parser.done()); - - json j = decoder.get_result(); -} - -TEST_CASE("test_incremental_parsing") -{ - jsoncons::json_decoder decoder; - json_parser parser; - - parser.reset(); - - parser.update("[fal",4); - parser.parse_some(decoder); - CHECK_FALSE(parser.done()); - CHECK(parser.source_exhausted()); - parser.update("se]",3); - parser.parse_some(decoder); - - parser.finish_parse(decoder); - CHECK(parser.done()); - - json j = decoder.get_result(); - REQUIRE(j.is_array()); - CHECK_FALSE(j[0].as()); -} - - - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_reader_exception_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_reader_exception_tests.cpp deleted file mode 100644 index 86bb9b822e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_reader_exception_tests.cpp +++ /dev/null @@ -1,252 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("test_filename_invalid") -{ - std::string in_file = "./input/json-exception--1.json"; - std::ifstream is(in_file); - - json_decoder decoder; - - try - { - json_reader reader(is,decoder); - reader.read_next(); - } - catch (const std::exception&) - { - } - //CHECK(false == decoder.is_valid()); -} - -TEST_CASE("test_exception_left_brace") -{ - std::string in_file = "./input/json-exception-1.json"; - std::ifstream is(in_file); - - json_decoder decoder; - try - { - json_reader reader(is,decoder); - reader.read_next(); - } - catch (const ser_error& e) - { - CHECK(e.code() == json_errc::expected_comma_or_right_bracket); - CHECK(14 == e.line()); - CHECK(30 == e.column()); - } - CHECK(false == decoder.is_valid()); -} -TEST_CASE("test_exception_right_brace") -{ - std::string in_file = "./input/json-exception-2.json"; - std::ifstream is(in_file); - - json_decoder decoder; - try - { - json_reader reader(is,decoder); - reader.read_next(); // must throw - CHECK(0 != 0); - } - catch (const ser_error& e) - { - //std::cout << e.what() << std::endl; - CHECK(e.code() == json_errc::expected_comma_or_right_brace); - CHECK(17 == e.line()); - CHECK(9 == e.column()); - } - CHECK(false == decoder.is_valid()); -} - -TEST_CASE("test_exception_array_eof") -{ - std::istringstream is("[100"); - - json_decoder decoder; - try - { - json_reader reader(is,decoder); - reader.read_next(); // must throw - CHECK(0 != 0); - } - catch (const ser_error& e) - { - CHECK(e.code() == json_errc::unexpected_eof); - CHECK(1 == e.line()); - CHECK(5 == e.column()); - } - CHECK(false == decoder.is_valid()); -} - -TEST_CASE("test_exception_unicode_eof") -{ - std::istringstream is("[\"\\u"); - - json_decoder decoder; - try - { - json_reader reader(is,decoder); - reader.read_next(); // must throw - CHECK(0 != 0); - } - catch (const ser_error& e) - { - //std::cout << e.what() << std::endl; - CHECK(e.code() == json_errc::unexpected_eof); - CHECK(1 == e.line()); - CHECK(5 == e.column()); - } - CHECK(false == decoder.is_valid()); -} - -TEST_CASE("test_exception_tru_eof") -{ - std::istringstream is("[tru"); - - json_decoder decoder; - try - { - json_reader reader(is,decoder); - reader.read_next(); // must throw - CHECK(0 != 0); - } - catch (const ser_error& e) - { - //std::cout << e.what() << std::endl; - CHECK(e.code() == json_errc::unexpected_eof); - CHECK(1 == e.line()); - CHECK(5 == e.column()); - } - CHECK(false == decoder.is_valid()); -} - -TEST_CASE("test_exception_fals_eof") -{ - std::istringstream is("[fals"); - - json_decoder decoder; - try - { - json_reader reader(is,decoder); - reader.read_next(); // must throw - CHECK(0 != 0); - } - catch (const ser_error& e) - { - //std::cout << e.what() << std::endl; - CHECK(e.code() == json_errc::unexpected_eof); - CHECK(1 == e.line()); - CHECK(6 == e.column()); - } - CHECK(false == decoder.is_valid()); -} - -TEST_CASE("test_exception_nul_eof") -{ - std::istringstream is("[nul"); - - json_decoder decoder; - try - { - json_reader reader(is,decoder); - reader.read_next(); // must throw - CHECK(0 != 0); - } - catch (const ser_error& e) - { - //std::cout << e.what() << std::endl; - CHECK(e.code() == json_errc::unexpected_eof); - CHECK(1 == e.line()); - CHECK(5 == e.column()); - } - CHECK(false == decoder.is_valid()); -} - -TEST_CASE("test_exception_true_eof") -{ - std::istringstream is("[true"); - - json_decoder decoder; - try - { - json_reader reader(is,decoder); - reader.read_next(); // must throw - CHECK(0 != 0); - } - catch (const ser_error& e) - { - CHECK(e.code() == json_errc::unexpected_eof); - CHECK(1 == e.line()); - CHECK(6 == e.column()); - } - CHECK(false == decoder.is_valid()); -} - -TEST_CASE("test_exception_false_eof") -{ - std::istringstream is("[false"); - - json_decoder decoder; - try - { - json_reader reader(is,decoder); - reader.read_next(); // must throw - CHECK(0 != 0); - } - catch (const ser_error& e) - { - CHECK(e.code() == json_errc::unexpected_eof); - CHECK(1 == e.line()); - CHECK(7 == e.column()); - } - CHECK(false == decoder.is_valid()); -} - -TEST_CASE("test_exception_null_eof") -{ - std::istringstream is("[null"); - - json_decoder decoder; - try - { - json_reader reader(is,decoder); - reader.read_next(); // must throw - CHECK(0 != 0); - } - catch (const ser_error& e) - { - CHECK(e.code() == json_errc::unexpected_eof); - CHECK(1 == e.line()); - CHECK(6 == e.column()); - } - CHECK(false == decoder.is_valid()); -} - -TEST_CASE("test_exception") -{ - std::string input("{\"field1\":\n\"value}"); - REQUIRE_THROWS_AS(json::parse(input),ser_error); - try - { - json::parse(input); - } - catch (const ser_error& e) - { - CHECK((e.code() == json_errc::unexpected_eof && e.line() == 2 && e.column() == 9)); - } -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_reader_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_reader_tests.cpp deleted file mode 100644 index a27c7a1ceb..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_reader_tests.cpp +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -void test_json_reader_error(const std::string& text, std::error_code ec) -{ - REQUIRE_THROWS(json::parse(text)); - try - { - json::parse(text); - } - catch (const ser_error& e) - { - if (e.code() != ec) - { - std::cout << text << std::endl; - std::cout << e.code().value() << " " << e.what() << std::endl; - } - CHECK(ec == e.code()); - } -} - -void test_json_reader_ec(const std::string& text, std::error_code expected) -{ - std::error_code ec; - - std::istringstream is(text); - json_decoder decoder; - json_reader reader(is,decoder); - - reader.read(ec); - //std::cerr << text << std::endl; - //std::cerr << ec.message() - // << " at line " << reader.line() - // << " and column " << reader.column() << std::endl; - - CHECK(ec); - CHECK(ec == expected); -} - -TEST_CASE("test_missing_separator") -{ - std::string jtext = R"({"field1"{}})"; - - test_json_reader_error(jtext, jsoncons::json_errc::expected_colon); - test_json_reader_ec(jtext, jsoncons::json_errc::expected_colon); -} - -TEST_CASE("test_read_invalid_value") -{ - std::string jtext = R"({"field1":ru})"; - - test_json_reader_error(jtext,jsoncons::json_errc::expected_value); - test_json_reader_ec(jtext, jsoncons::json_errc::expected_value); -} - -TEST_CASE("test_read_unexpected_end_of_file") -{ - std::string jtext = R"({"field1":{})"; - - test_json_reader_error(jtext, jsoncons::json_errc::unexpected_eof); - test_json_reader_ec(jtext, jsoncons::json_errc::unexpected_eof); -} - - -TEST_CASE("test_read_value_not_found") -{ - std::string jtext = R"({"name":})"; - - test_json_reader_error(jtext, jsoncons::json_errc::expected_value); - test_json_reader_ec(jtext, jsoncons::json_errc::expected_value); -} - -TEST_CASE("test_read_escaped_characters") -{ - std::string input("[\"\\n\\b\\f\\r\\t\"]"); - std::string expected("\n\b\f\r\t"); - - json o = json::parse(input); - CHECK(expected == o[0].as()); -} - - -TEST_CASE("test_read_expected_colon") -{ - test_json_reader_error("{\"name\" 10}", jsoncons::json_errc::expected_colon); - test_json_reader_error("{\"name\" true}", jsoncons::json_errc::expected_colon); - test_json_reader_error("{\"name\" false}", jsoncons::json_errc::expected_colon); - test_json_reader_error("{\"name\" null}", jsoncons::json_errc::expected_colon); - test_json_reader_error("{\"name\" \"value\"}", jsoncons::json_errc::expected_colon); - test_json_reader_error("{\"name\" {}}", jsoncons::json_errc::expected_colon); - test_json_reader_error("{\"name\" []}", jsoncons::json_errc::expected_colon); -} - -TEST_CASE("test_read_expected_name") -{ - test_json_reader_error("{10}", jsoncons::json_errc::expected_name); - test_json_reader_error("{true}", jsoncons::json_errc::expected_name); - test_json_reader_error("{false}", jsoncons::json_errc::expected_name); - test_json_reader_error("{null}", jsoncons::json_errc::expected_name); - test_json_reader_error("{{}}", jsoncons::json_errc::expected_name); - test_json_reader_error("{[]}", jsoncons::json_errc::expected_name); -} - -TEST_CASE("test_read_expected_value") -{ - test_json_reader_error("[tru]", jsoncons::json_errc::invalid_value); - test_json_reader_error("[fa]", jsoncons::json_errc::invalid_value); - test_json_reader_error("[n]", jsoncons::json_errc::invalid_value); -} - -TEST_CASE("test_read_primitive_pass") -{ - json val; - CHECK_NOTHROW((val=json::parse("null"))); - CHECK(val == json::null()); - CHECK_NOTHROW((val=json::parse("false"))); - CHECK(val == json(false)); - CHECK_NOTHROW((val=json::parse("true"))); - CHECK(val == json(true)); - CHECK_NOTHROW((val=json::parse("10"))); - CHECK(val == json(10)); - CHECK_NOTHROW((val=json::parse("1.999"))); - CHECK(val == json(1.999)); - CHECK_NOTHROW((val=json::parse("\"string\""))); - CHECK(val == json("string")); -} - -TEST_CASE("test_read_empty_structures") -{ - json val; - CHECK_NOTHROW((val=json::parse("{}"))); - CHECK_NOTHROW((val=json::parse("[]"))); - CHECK_NOTHROW((val=json::parse("{\"object\":{},\"array\":[]}"))); - CHECK_NOTHROW((val=json::parse("[[],{}]"))); -} - -TEST_CASE("test_read_primitive_fail") -{ - test_json_reader_error("null {}", jsoncons::json_errc::extra_character); - test_json_reader_error("n ", jsoncons::json_errc::invalid_value); - test_json_reader_error("nu ", jsoncons::json_errc::invalid_value); - test_json_reader_error("nul ", jsoncons::json_errc::invalid_value); - test_json_reader_error("false {}", jsoncons::json_errc::extra_character); - test_json_reader_error("fals ", jsoncons::json_errc::invalid_value); - test_json_reader_error("true []", jsoncons::json_errc::extra_character); - test_json_reader_error("tru ", jsoncons::json_errc::invalid_value); - test_json_reader_error("10 {}", jsoncons::json_errc::extra_character); - test_json_reader_error("1a ", jsoncons::json_errc::invalid_number); - test_json_reader_error("1.999 []", jsoncons::json_errc::extra_character); - test_json_reader_error("1e0-1", jsoncons::json_errc::invalid_number); - test_json_reader_error("\"string\"{}", jsoncons::json_errc::extra_character); - test_json_reader_error("\"string\"[]", jsoncons::json_errc::extra_character); -} - -TEST_CASE("test_read_multiple") -{ - std::string in="{\"a\":1,\"b\":2,\"c\":3}{\"a\":4,\"b\":5,\"c\":6}"; - //std::cout << in << std::endl; - - std::istringstream is(in); - - jsoncons::json_decoder decoder; - json_reader reader(is,decoder); - - REQUIRE_FALSE(reader.eof()); - reader.read_next(); - json val = decoder.get_result(); - CHECK(1 == val["a"].as()); - REQUIRE_FALSE(reader.eof()); - reader.read_next(); - json val2 = decoder.get_result(); - CHECK(4 == val2["a"].as()); - CHECK(reader.eof()); -} - -TEST_CASE("json_reader read from string test") -{ - std::string s = R"( -{ - "store": { - "book": [ - { - "category": "reference", - "author": "Margaret Weis", - "title": "Dragonlance Series", - "price": 31.96 - }, - { - "category": "reference", - "author": "Brent Weeks", - "title": "Night Angel Trilogy", - "price": 14.70 - } - ] - } -} -)"; - - json_decoder decoder; - json_reader reader(s, decoder); - reader.read(); - json j = decoder.get_result(); - - REQUIRE(j.is_object()); - REQUIRE(j.size() == 1); - REQUIRE(j[0].is_object()); - REQUIRE(j[0].size() == 1); - REQUIRE(j[0][0].is_array()); - REQUIRE(j[0][0].size() == 2); - CHECK(j[0][0][0]["category"].as() == std::string("reference")); - CHECK(j[0][0][1]["author"].as() == std::string("Brent Weeks")); -} - - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_swap_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_swap_tests.cpp deleted file mode 100644 index 481b71bcf9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_swap_tests.cpp +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -void check_swap(const json& j1, const json& j2) -{ - json j3 = j1; - json j4 = j2; - - j3.swap(j4); - CHECK(j1 == j4); - CHECK(j2 == j3); -} - -TEST_CASE("test_swap") -{ - json j1 = json::null(); - json j2 = false; - json j3 = -2000; - json j4 = 2000U; - json j5 = 2000.1234; - json j6 = "Small"; - json j7 = "String too large for small string"; - json j8 = json::parse("[1,2,3,4]"); - json j9; - json j10 = json::object(); - j10["Name"] = "John Smith"; - - check_swap(j1,j1); - check_swap(j1,j2); - check_swap(j1,j3); - check_swap(j1,j4); - check_swap(j1,j5); - check_swap(j1,j6); - check_swap(j1,j7); - check_swap(j1,j8); - check_swap(j1,j9); - check_swap(j1,j10); - - check_swap(j2,j1); - check_swap(j2,j2); - check_swap(j2,j3); - check_swap(j2,j4); - check_swap(j2,j5); - check_swap(j2,j6); - check_swap(j2,j7); - check_swap(j2,j8); - check_swap(j2,j9); - check_swap(j2,j10); - - check_swap(j3,j1); - check_swap(j3,j2); - check_swap(j3,j3); - check_swap(j3,j4); - check_swap(j3,j5); - check_swap(j3,j6); - check_swap(j3,j7); - check_swap(j3,j8); - check_swap(j3,j9); - check_swap(j3,j10); - - check_swap(j4,j1); - check_swap(j4,j2); - check_swap(j4,j3); - check_swap(j4,j4); - check_swap(j4,j5); - check_swap(j4,j6); - check_swap(j4,j7); - check_swap(j4,j8); - check_swap(j4,j9); - check_swap(j4,j10); - - check_swap(j5,j1); - check_swap(j5,j2); - check_swap(j5,j3); - check_swap(j5,j4); - check_swap(j5,j5); - check_swap(j5,j6); - check_swap(j5,j7); - check_swap(j5,j8); - check_swap(j5,j9); - check_swap(j5,j10); - - check_swap(j6,j1); - check_swap(j6,j2); - check_swap(j6,j3); - check_swap(j6,j4); - check_swap(j6,j5); - check_swap(j6,j6); - check_swap(j6,j7); - check_swap(j6,j8); - check_swap(j6,j9); - check_swap(j6,j10); - - check_swap(j7,j1); - check_swap(j7,j2); - check_swap(j7,j3); - check_swap(j7,j4); - check_swap(j7,j5); - check_swap(j7,j6); - check_swap(j7,j7); - check_swap(j7,j8); - check_swap(j7,j9); - check_swap(j7,j10); - - check_swap(j8,j1); - check_swap(j8,j2); - check_swap(j8,j3); - check_swap(j8,j4); - check_swap(j8,j5); - check_swap(j8,j6); - check_swap(j8,j7); - check_swap(j8,j8); - check_swap(j8,j9); - check_swap(j8,j10); - - check_swap(j9,j1); - check_swap(j9,j2); - check_swap(j9,j3); - check_swap(j9,j4); - check_swap(j9,j5); - check_swap(j9,j6); - check_swap(j9,j7); - check_swap(j9,j8); - check_swap(j9,j9); - check_swap(j9,j10); - - check_swap(j10,j1); - check_swap(j10,j2); - check_swap(j10,j3); - check_swap(j10,j4); - check_swap(j10,j5); - check_swap(j10,j6); - check_swap(j10,j7); - check_swap(j10,j8); - check_swap(j10,j9); - check_swap(j10,j10); -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_type_traits_container_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_type_traits_container_tests.cpp deleted file mode 100644 index 5c87bd43c3..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_type_traits_container_tests.cpp +++ /dev/null @@ -1,373 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("test_json_as_pair") -{ - json j = json::array{false,1}; - auto t = j.as>(); - CHECK(std::get<0>(t) == false); - CHECK(std::get<1>(t) == 1); -} - -TEST_CASE("test_tuple_to_json") -{ - auto t = std::make_tuple(false,1,"foo"); - json j(t); - - REQUIRE(j.is_array()); - REQUIRE(j.size() == 3); - CHECK(false == j[0].as()); - CHECK(1 == j[1].as()); - CHECK(std::string("foo") == j[2].as()); -} - -TEST_CASE("test_json_as_tuple") -{ - json j = json::array{false,1,"foo"}; - - auto t = j.as>(); - - CHECK(std::get<0>(t) == false); - CHECK(std::get<1>(t) == 1); - CHECK(std::get<2>(t) == std::string("foo")); -} - -TEST_CASE("test_characters") -{ - const json a = "short"; - const json b = "a long string"; - - CHECK(true == a.is()); - CHECK(true == b.is()); - - std::string s2 = a.as(); - std::string t2 = b.as(); - - json c = json::array{ "short","a long string" }; - auto u = c.as>(); -} - -// vector - -TEST_CASE("test_is_json_vector") -{ - json a = json::array{0,1,2,3,4}; - - CHECK(true == a.is>()); -} - -TEST_CASE("test_as_vector") -{ - json a = json::array{0,1,2,3,4}; - std::vector v = a.as>(); - - CHECK(v[0] == 0); - CHECK(v[1] == 1); - CHECK(v[2] == 2); - CHECK(v[3] == 3); - CHECK(v[4] == 4); -} - -TEST_CASE("test_assign_vector") -{ - std::vector v {0,1,2,3,4}; - json a = v; - - CHECK(a[0] == json(0)); - CHECK(a[1] == json(1)); - CHECK(a[2] == json(2)); - CHECK(a[3] == json(3)); - CHECK(a[4] == json(4)); -} - -TEST_CASE("test_as_vector_of_bool") -{ - json a = json::parse("[true,false,true]"); - - std::vector v = a.as>(); - - CHECK(v[0] == true); - CHECK(v[1] == false); - CHECK(v[2] == true); -} - -TEST_CASE("test_assign_vector_of_bool") -{ - std::vector v = {true,false,true}; - json a = v; - - CHECK(a[0] == json(true)); - CHECK(a[1] == json(false)); - CHECK(a[2] == json(true)); - - json b; - b = v; - - CHECK(b[0] == json(true)); - CHECK(b[1] == json(false)); - CHECK(b[2] == json(true)); -} - -TEST_CASE("test_construct_vector_of_bool") -{ - std::vector v = {true,false,true}; - json a = v; - - CHECK(a[0] == json(true)); - CHECK(a[1] == json(false)); - CHECK(a[2] == json(true)); -} - -TEST_CASE("test_construct_const_vector_of_bool") -{ - const std::vector v = {true,false,true}; - json a = v; - - CHECK(a[0] == json(true)); - CHECK(a[1] == json(false)); - CHECK(a[2] == json(true)); -} - -// valarray - -TEST_CASE("test_is_json_valarray") -{ - json a = json::array{0,1,2,3,4}; - - CHECK(true == a.is>()); -} - -TEST_CASE("test_as_valarray") -{ - json a = json::array{0,1,2,3,4}; - std::valarray v = a.as>(); - - CHECK(v[0] == 0); - CHECK(v[1] == 1); - CHECK(v[2] == 2); - CHECK(v[3] == 3); - CHECK(v[4] == 4); -} - -TEST_CASE("test_assign_valarray") -{ - std::valarray v {0,1,2,3,4}; - json a = v; - - CHECK(a[0] == json(0)); - CHECK(a[1] == json(1)); - CHECK(a[2] == json(2)); - CHECK(a[3] == json(3)); - CHECK(a[4] == json(4)); -} - -TEST_CASE("test_is_json_map") -{ - json a; - a["a"] = 0; - a["b"] = 1; - a["c"] = 2; - - CHECK(true == (a.is >())); -} - -TEST_CASE("test_is_json_map2") -{ - json a; - a["a"] = "0"; - a["b"] = "1"; - a["c"] = "2"; - - CHECK(true == (a["a"].is_string())); - - json b("0"); - CHECK(true == (b.is())); - - CHECK(true == (a["a"].is())); - - CHECK(true == (a.is >())); -} - -TEST_CASE("test_as_map") -{ - json o; - o["first"] = "first"; - o["second"] = "second"; - - auto m = o.as>(); - CHECK(std::string("first") == m.at("first")); - CHECK(std::string("second") == m.at("second")); - - json o2(m); - CHECK(o == o2); - - json o3; - o3 = m; - CHECK(o == o3); -} - -TEST_CASE("test_as_map2") -{ - json o; - o["first"] = 1; - o["second"] = true; - o["third"] = jsoncons::null_type(); - - auto m = o.as>(); - CHECK(std::string("1") == m.at("first")); - CHECK(std::string("true") == m.at("second")); - CHECK(std::string("null") == m.at("third")); - - json o2(m); - CHECK(std::string("1") == o2["first"].as()); -} - -TEST_CASE("test_from_stl_container") -{ - std::vector a_vector{1, 2, 3, 4}; - json j_vector(a_vector); - CHECK((1 == j_vector[0].as())); - CHECK(2 == j_vector[1].as()); - CHECK(3 == j_vector[2].as()); - CHECK(4 == j_vector[3].as()); - - std::vector a_vector2{1ul, 2ul, 3ul, 4ul}; - json j_vec2(a_vector2); - CHECK(1 == j_vec2[0].as()); - CHECK(2 == j_vec2[1].as()); - CHECK(3 == j_vec2[2].as()); - CHECK(4 == j_vec2[3].as()); - - std::deque a_deque{1.123, 2.234, 3.456, 4.567}; - json j_deque(a_deque); - CHECK(1.123 == j_deque[0].as()); - CHECK(2.234 == j_deque[1].as()); - CHECK(3.456 == j_deque[2].as()); - CHECK(4.567 == j_deque[3].as()); - - std::list a_list{true, true, false, true}; - json j_list(a_list); - CHECK(true == j_list[0].as()); - CHECK(true == j_list[1].as()); - CHECK(false == j_list[2].as()); - CHECK(true == j_list[3].as()); - - std::forward_lista_flist {12345678909876, 23456789098765, 34567890987654, 45678909876543}; - json j_flist(a_flist); - CHECK(static_cast(12345678909876) == j_flist[0].as()); - CHECK(static_cast(23456789098765) == j_flist[1].as()); - CHECK(static_cast(34567890987654) == j_flist[2].as()); - CHECK(static_cast(45678909876543) == j_flist[3].as()); - - std::array a_array {{1, 2, 3, 4}}; - json j_array(a_array); - CHECK(1 == j_array[0].as()); - CHECK(2 == j_array[1].as()); - CHECK(3 == j_array[2].as()); - CHECK(4 == j_array[3].as()); - - std::set a_set{"one", "two", "three", "four", "one"}; - json j_set(a_set); // only one entry for "one" is used - // ["four", "one", "three", "two"] - - std::unordered_set a_uset{"one", "two", "three", "four", "one"}; - json j_uset(a_uset); // only one entry for "one" is used - // maybe ["two", "three", "four", "one"] - - std::multiset a_mset{"one", "two", "one", "four"}; - json j_mset(a_mset); // only one entry for "one" is used - // maybe ["one", "two", "four"] - - std::unordered_multiset a_umset {"one", "two", "one", "four"}; - json j_umset(a_umset); // both entries for "one" are used - // maybe ["one", "two", "one", "four"] - - std::map a_map{ {"one", 1}, {"two", 2}, {"three", 3} }; - json j_map(a_map); - CHECK(1 == j_map["one"].as()); - CHECK(2 == j_map["two"].as()); - CHECK(3 == j_map["three"].as()); - - std::unordered_map a_umap{ {"one", 1.2}, {"two", 2.3}, {"three", 3.4} }; - json j_umap(a_umap); - CHECK(1.2 == j_umap["one"].as()); - CHECK(2.3 == j_umap["two"].as()); - CHECK(3.4 == j_umap["three"].as()); - - std::multimap a_mmap{ {"one", true}, {"two", true}, {"three", false}, {"three", true} }; - json j_mmap(a_mmap); // one entry for key "three" - CHECK(true == j_mmap.find("one")->value().as()); - CHECK(true == j_mmap.find("two")->value().as()); - CHECK(false == j_mmap.find("three")->value().as()); - - std::unordered_multimap a_ummap { {"one", true}, {"two", true}, /*{"three", false},*/ {"three", true} }; - json j_ummap(a_ummap); // two entries for key "three" - CHECK(true == j_ummap.find("one")->value().as()); - CHECK(true == j_ummap.find("two")->value().as()); - CHECK(true == j_ummap.find("three")->value().as()); -} - -//own vector will always be of an even length -struct own_vector : std::vector { using std::vector::vector; }; - -namespace jsoncons { -template -struct json_type_traits { - static bool is(const Json& j) noexcept - { - return j.is_object() && j.size() % 2 == 0; - } - static own_vector as(const Json& j) - { - own_vector v; - for (auto& item : j.object_range()) - { - std::string s(item.key()); - v.push_back(std::strtol(s.c_str(),nullptr,10)); - v.push_back(item.value().template as()); - } - return v; - } - static Json to_json(const own_vector& val){ - Json j; - for(size_t i=0;i -struct is_json_type_traits_declared : public std::true_type -{}; -} // jsoncons - -TEST_CASE("own_vector json_type_traits") -{ - json j = json::object{ {"1",2},{"3",4} }; - REQUIRE(j.is()); - auto v = j.as(); - REQUIRE(v.size() == 4); - json j2 = v; - CHECK(j2 == j); -} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_type_traits_macros_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_type_traits_macros_tests.cpp deleted file mode 100644 index 76490605e0..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_type_traits_macros_tests.cpp +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -namespace json_type_traits_macros_tests { - - struct book - { - std::string author; - std::string title; - double price; - }; - struct book2 - { - std::string author; - std::string title; - double price; - std::string isbn; - }; - class book3 - { - std::string author_; - std::string title_; - double price_; - public: - book3() = default; - - book3(const std::string& author, - const std::string& title, - double price) - : author_(author), title_(title), price_(price) - { - } - - book3(const book3&) = default; - book3(book3&&) = default; - book3& operator=(const book3&) = default; - book3& operator=(book3&&) = default; - - const std::string& author() const - { - return author_; - } - - const std::string& title() const - { - return title_; - } - - double price() const - { - return price_; - } - }; - -} // namespace jsoncons_member_traits_decl_tests - -namespace ns = json_type_traits_macros_tests; - -JSONCONS_GETTER_CTOR_TRAITS_DECL(ns::book3, author, title, price) -JSONCONS_MEMBER_TRAITS_DECL(ns::book,author,title,price) -JSONCONS_MEMBER_TRAITS_DECL(ns::book2,author,title,price,isbn) - -TEST_CASE("JSONCONS_MEMBER_TRAITS_DECL tests") -{ - std::string an_author = "Haruki Murakami"; - std::string a_title = "Kafka on the Shore"; - double a_price = 25.17; - - ns::book book{an_author, a_title, a_price}; - - SECTION("book") - { - std::string s; - - encode_json(book, s); - - json j = decode_json(s); - - REQUIRE(j.is() == true); - REQUIRE(j.is() == false); - - CHECK(j["author"].as() == an_author); - CHECK(j["title"].as() == a_title); - CHECK(j["price"].as() == Approx(a_price).epsilon(0.001)); - - json j2(book); - - CHECK(j == j2); - - ns::book val = j.as(); - - CHECK(val.author == book.author); - CHECK(val.title == book.title); - CHECK(val.price == Approx(book.price).epsilon(0.001)); - } -} - -TEST_CASE("JSONCONS_GETTER_CTOR_TRAITS_DECL tests") -{ - std::string an_author = "Haruki Murakami"; - std::string a_title = "Kafka on the Shore"; - double a_price = 25.17; - - SECTION("is") - { - json j; - j["author"] = an_author; - j["title"] = a_title; - j["price"] = a_price; - - bool val = j.is(); - CHECK(val == true); - } - SECTION("to_json") - { - ns::book3 book(an_author,a_title,a_price); - - json j(book); - - CHECK(j["author"].as() == an_author); - CHECK(j["title"].as() == a_title); - CHECK(j["price"].as() == Approx(a_price).epsilon(0.001)); - } - - SECTION("as") - { - json j; - j["author"] = an_author; - j["title"] = a_title; - j["price"] = a_price; - - ns::book3 book = j.as(); - - CHECK(book.author() == an_author); - CHECK(book.title() == a_title); - CHECK(book.price() == Approx(a_price).epsilon(0.001)); - } -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_type_traits_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_type_traits_tests.cpp deleted file mode 100644 index 321218da89..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_type_traits_tests.cpp +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; -// own vector will always be of an even length -struct own_vector : std::vector { using std::vector::vector; }; - -namespace jsoncons { -template -struct json_type_traits { - static bool is(const Json&) noexcept { return true; } - static own_vector as(const Json&) { return own_vector(); } - static Json to_json(const own_vector& val) { - Json j; - for (uint64_t i = 0; i(p); - - root["Test"] = q; -} - -TEST_CASE("test_uint8_t") -{ - uint8_t x = 10; - - json o; - o["u"] = x; - - CHECK(o["u"].is_number()); - - uint8_t y = o["u"].as(); - - CHECK(y == 10); -} - -TEST_CASE("test_float_assignment") -{ - float x = 10.5; - - json o; - o["float"] = x; - - CHECK(o["float"].is_number()); - - float y = o["float"].as(); - - CHECK(10.5 == Approx(y).epsilon(0.00001)); -} - -TEST_CASE("test_float") -{ - float x = 10.5; - - json o(x); - - CHECK(o.is()); - - float y = o.as(); - - CHECK(10.5 == Approx(y).epsilon(0.00001)); -} - -TEST_CASE("test_unsupported_type") -{ - json o; - - //o["u"] = Info; - // compile error -} - -TEST_CASE("test_as_json_value") -{ - json a; - - a["first"] = "first"; - a["second"] = "second"; - - CHECK(true == a.is()); - - json b = a.as(); - CHECK("first" == b["first"].as()); - CHECK("second" == b["second"].as()); -} - -TEST_CASE("test_byte_string_as_vector") -{ - json a(byte_string{'H','e','l','l','o'}); - - REQUIRE(a.is_byte_string()); - - auto bs = a.as(); - - REQUIRE(5 == bs.size()); - CHECK('H' == bs[0]); - CHECK('e' == bs[1]); - CHECK('l' == bs[2]); - CHECK('l' == bs[3]); - CHECK('o' == bs[4]); -} -/* -TEST_CASE("test_own_vector") -{ - jsoncons::json j = own_vector({0,9,8,7}); - std::cout << j; -} -*/ - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_variant_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_variant_tests.cpp deleted file mode 100644 index b7e163dc3a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/json_variant_tests.cpp +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("test_variant") -{ - json::variant var1(int64_t(-100), semantic_tag::none); - CHECK(storage_type::int64_val == var1.get_storage_type()); - json::variant var2(uint64_t(100), semantic_tag::none); - CHECK(storage_type::uint64_val == var2.get_storage_type()); - json::variant var3("Small string", 12, semantic_tag::none); - CHECK(storage_type::short_string_val == var3.get_storage_type()); - json::variant var4("Too long to fit in small string", 31, semantic_tag::none); - CHECK(storage_type::long_string_val == var4.get_storage_type()); - json::variant var5(true, semantic_tag::none); - CHECK(storage_type::bool_val == var5.get_storage_type()); - json::variant var6(semantic_tag::none); - CHECK(storage_type::empty_object_val == var6.get_storage_type()); - json::variant var7{ null_type(), semantic_tag::none }; - CHECK(storage_type::null_val == var7.get_storage_type()); - json::variant var8{ json::object(json::allocator_type()), semantic_tag::none }; - CHECK(storage_type::object_val == var8.get_storage_type()); - json::variant var9(123456789.9, semantic_tag::none); - CHECK(storage_type::double_val == var9.get_storage_type()); -} - -TEST_CASE("test_move_constructor") -{ - int64_t val1 = -100; - json::variant var1(val1, semantic_tag::none); - json::variant var2(std::move(var1)); - //CHECK(storage_type::null_val == var1.get_storage_type()); - CHECK(storage_type::int64_val == var2.get_storage_type()); - CHECK(var2.int64_data_cast()->value() == val1); - - uint64_t val3 = 9999; - json::variant var3(val3, semantic_tag::none); - json::variant var4(std::move(var3)); - //CHECK(storage_type::null_val == var3.get_storage_type()); - CHECK(storage_type::uint64_val == var4.get_storage_type()); - CHECK(var4.uint64_data_cast()->value() == val3); - - double val5 = 123456789.9; - json::variant var5(val5, semantic_tag::none); - json::variant var6(std::move(var5)); - //CHECK(storage_type::null_val == var5.get_storage_type()); - CHECK(storage_type::double_val == var6.get_storage_type()); - CHECK(var6.double_data_cast()->value() == val5); - - std::string val7("Too long for small string"); - json::variant var7(val7.data(), val7.length(), semantic_tag::none); - json::variant var8(std::move(var7)); - //CHECK(storage_type::null_val == var7.get_storage_type()); - CHECK(storage_type::long_string_val == var8.get_storage_type()); - CHECK(val7 == var8.string_data_cast()->data()); - CHECK(val7.length() == var8.string_data_cast()->length()); - - std::string val9("Small string"); - json::variant var9(val9.data(), val9.length(), semantic_tag::none); - json::variant var10(std::move(var9)); - //CHECK(storage_type::null_val == var9.get_storage_type()); - CHECK(storage_type::short_string_val == var10.get_storage_type()); - CHECK(val9 == var10.short_string_data_cast()->data()); - CHECK(val9.length() == var10.short_string_data_cast()->length()); - - bool val11 = true; - json::variant var11(val11, semantic_tag::none); - json::variant var12(std::move(var11)); - //CHECK(storage_type::null_val == var11.get_storage_type()); - CHECK(storage_type::bool_val == var12.get_storage_type()); - CHECK(var12.bool_data_cast()->value() == val11); - - std::string val13("Too long for small string"); - json::variant var13(val13.data(), val13.length(), semantic_tag::none); - json::variant var14(std::move(var13)); - //CHECK(storage_type::null_val == var13.get_storage_type()); - CHECK(storage_type::long_string_val == var14.get_storage_type()); - CHECK(val13 == var14.string_data_cast()->data()); - - json::object val15 = { {"first",1},{"second",2} }; - json::variant var15(val15, semantic_tag::none); - json::variant var16(std::move(var15)); - CHECK(storage_type::null_val == var15.get_storage_type()); - CHECK(storage_type::object_val == var16.get_storage_type()); - CHECK(val15 == var16.object_data_cast()->value()); - - json::array val17 = {1,2,3,4}; - json::variant var17(val17, semantic_tag::none); - json::variant var18(std::move(var17)); - CHECK(storage_type::null_val == var17.get_storage_type()); - CHECK(storage_type::array_val == var18.get_storage_type()); - CHECK(val17 == var18.array_data_cast()->value()); -} - -TEST_CASE("test_copy_constructor") -{ - int64_t val1 = 123456789; - json::variant var1(val1, semantic_tag::none); - json::variant var2(var1); - CHECK(storage_type::int64_val == var1.get_storage_type()); - CHECK(storage_type::int64_val == var2.get_storage_type()); - CHECK(var2.int64_data_cast()->value() == val1); - - uint64_t val3 = 123456789; - json::variant var3(val3, semantic_tag::none); - json::variant var4(var3); - CHECK(storage_type::uint64_val == var3.get_storage_type()); - CHECK(storage_type::uint64_val == var4.get_storage_type()); - CHECK(var4.uint64_data_cast()->value() == val3); - - double val5 = 123456789.9; - json::variant var5(val5, semantic_tag::none); - json::variant var6(var5); - CHECK(storage_type::double_val == var5.get_storage_type()); - CHECK(storage_type::double_val == var6.get_storage_type()); - CHECK(var6.double_data_cast()->value() == val5); - - std::string val9 = "Small string"; - json::variant var9(val9.data(), val9.length(), semantic_tag::none); - json::variant var10(var9); - CHECK(storage_type::short_string_val == var9.get_storage_type()); - CHECK(storage_type::short_string_val == var10.get_storage_type()); - CHECK(var10.short_string_data_cast()->data() == val9); - - bool val11 = true; - json::variant var11(val11, semantic_tag::none); - json::variant var12(var11); - CHECK(storage_type::bool_val == var11.get_storage_type()); - CHECK(storage_type::bool_val == var12.get_storage_type()); - CHECK(var12.bool_data_cast()->value() == val11); - - std::string val13 = "Too long for small string"; - json::variant var13(val13.data(), val13.length(), semantic_tag::none); - json::variant var14(var13); - CHECK(storage_type::long_string_val == var13.get_storage_type()); - CHECK(storage_type::long_string_val == var14.get_storage_type()); - CHECK(var14.string_data_cast()->data() == val13); - - json::object val15 = { {"first",1},{"second",2} }; - json::variant var15(val15, semantic_tag::none); - json::variant var16(var15); - CHECK(storage_type::object_val == var15.get_storage_type()); - CHECK(storage_type::object_val == var16.get_storage_type()); - CHECK(val15 == var16.object_data_cast()->value()); - - json::array val17 = {1,2,3,4}; - json::variant var17(val17, semantic_tag::none); - json::variant var18(var17); - CHECK(storage_type::array_val == var17.get_storage_type()); - CHECK(storage_type::array_val == var18.get_storage_type()); - CHECK(val17 == var18.array_data_cast()->value()); -} - -TEST_CASE("test_equals") -{ - json::variant var1(semantic_tag::none); - json::variant var2{ json::object(), semantic_tag::none }; - CHECK((var1 == var1 && var2 == var2)); - CHECK((var1 == var2 && var2 == var1)); - - json::variant var3{semantic_tag::none }; - CHECK((var3 == var1 && var1 == var3)); - json::variant var4{ json::object({{"first",1},{"second",2}}), semantic_tag::none }; - json::variant var5{ json::object({ { "first",1 },{ "second",2 } }), semantic_tag::none }; - CHECK((var3 != var4 && var4 != var3)); - CHECK((var2 != var4 && var4 != var2)); - CHECK(var4 == var4); - CHECK(var4 == var5); - CHECK(var5 == var4); - - json::variant var6(int64_t(100), semantic_tag::none); - json::variant var7(uint64_t(100), semantic_tag::none); - CHECK((var6 == var7 && var7 == var6)); - - json::variant var8(100.0, semantic_tag::none); - CHECK((var8 == var8 && var6 == var8 && var8 == var6 && var7 == var8 && var8 == var7)); - - std::string val9("small string"); - std::string val11("small string 2"); - json::variant var9(val9.data(), val9.length(), semantic_tag::none); - json::variant var10(val9.data(),val9.length(), semantic_tag::none); - json::variant var11(val11.data(),val11.length(), semantic_tag::none); - - std::string val12("too long for small string"); - std::string val14("too long for small string 2"); - json::variant var12(val12.data(),val12.length(), semantic_tag::none); - json::variant var13(val12.data(),val12.length(), semantic_tag::none); - json::variant var14(val14.data(),val14.length(), semantic_tag::none); - CHECK((var9 == var10 && var10 == var9)); - CHECK((var9 != var11 && var11 != var9)); - CHECK((var12 == var13 && var13 == var12)); - CHECK((var12 != var14 && var14 != var12)); - - json::variant var15(val9.data(),val9.length(), semantic_tag::none, std::allocator()); - CHECK((var9 == var15 && var15 == var9)); - - json::variant var16(static_cast(0), semantic_tag::none); - json::variant var17(static_cast(0), semantic_tag::none); - CHECK(var16 == var17); - CHECK(var17 == var16); -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsoncons_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsoncons_tests.cpp deleted file mode 100644 index f7e0c3317b..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsoncons_tests.cpp +++ /dev/null @@ -1,230 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("test_1") -{ - basic_json j; - - std::basic_ostringstream os; - - std::cout << sizeof(json) << std::endl; - - //os << j << U"\n"; -} - -TEST_CASE("test_shrink_to_fit") -{ - json val = json::make_array(3); - val.reserve(100); - val[0].reserve(100); - val[0]["key"] = "value"; - val.shrink_to_fit(); - CHECK(3 == val.size()); - CHECK(1 == val[0].size()); -} - -TEST_CASE("test_for_each_value") -{ - std::string input = "{\"A\":\"Jane\", \"B\":\"Roe\",\"C\":10}"; - json val = json::parse(input); - - json::object_iterator it = val.object_range().begin(); - - CHECK(it->value().is_string()); - ++it; - CHECK(it->value().is_string()); - ++it; - CHECK(it->value().get_storage_type() == jsoncons::storage_type::uint64_val); - ++it; - CHECK((it == val.object_range().end())); -} - -TEST_CASE("test_assignment") -{ - json root; - - root["double_1"] = 10.0; - - json double_1 = root["double_1"]; - - CHECK(double_1.as() == Approx(10.0).epsilon(0.000001)); - - root["myobject"] = json(); - root["myobject"]["double_2"] = 7.0; - root["myobject"]["bool_2"] = true; - root["myobject"]["int_2"] = 0LL; - root["myobject"]["string_2"] = "my string"; - root["myarray"] = json::array(); - - json double_2 = root["myobject"]["double_2"]; - - CHECK(double_2.as() == Approx(7.0).epsilon(0.000001)); - CHECK(double_2.as() == 7); - CHECK(root["myobject"]["bool_2"].as()); - CHECK(root["myobject"]["int_2"].as() == 0); - CHECK(root["myobject"]["string_2"].as() == std::string("my string")); - - CHECK(root["myobject"]["bool_2"].as()); - CHECK(root["myobject"]["int_2"].as() == 0); - CHECK(root["myobject"]["string_2"].as() == std::string("my string")); - -} - -TEST_CASE("test_array") -{ - json root; - - root["addresses"]; - - std::vector addresses; - json address1; - address1["city"] = "San Francisco"; - address1["state"] = "CA"; - address1["zip"] = "94107"; - address1["country"] = "USA"; - addresses.push_back(address1); - - json address2; - address2["city"] = "Sunnyvale"; - address2["state"] = "CA"; - address2["zip"] = "94085"; - address2["country"] = "USA"; - addresses.push_back(address2); - - root["addresses"] = addresses; - - CHECK(root["addresses"].size() == 2); - -} - -TEST_CASE("test_null") -{ - json nullval = json::null(); - CHECK(nullval.is_null()); - CHECK(nullval.is()); - - json obj; - obj["field"] = json::null(); - CHECK(obj["field"] == json::null()); -} - -TEST_CASE("test_to_string") -{ - std::ostringstream os; - os << "{" - << "\"string\":\"value\"" - << ",\"null\":null" - << ",\"bool1\":false" - << ",\"bool2\":true" - << ",\"integer\":12345678" - << ",\"neg-integer\":-87654321" - << ",\"double\":123456.01" - << ",\"neg-double\":-654321.01" - << ",\"exp\":2.00600e+03" - << ",\"minus-exp\":1.00600e-010" - << ",\"escaped-string\":\"\\\\\\n\"" - << "}"; - - - json root = json::parse(os.str()); - - CHECK(root["null"].is_null()); - CHECK(root["null"].is()); - CHECK_FALSE(root["bool1"].as()); - CHECK(root["bool2"].as()); - CHECK(root["integer"].as() == 12345678); - CHECK(root["integer"].as() == 12345678); - CHECK(root["neg-integer"].as() == -87654321); - CHECK(root["double"].as() == Approx(123456.01).epsilon(0.0000001)); - CHECK(root["escaped-string"].as() == std::string("\\\n")); - - CHECK_FALSE(root["bool1"].as()); - CHECK(root["bool2"].as()); - CHECK(root["integer"].as() == 12345678); - CHECK(root["integer"].as() == 12345678); - CHECK(root["neg-integer"].as() == -87654321); - CHECK(root["double"].as() == Approx(123456.01).epsilon(0.0000001)); - CHECK(root["escaped-string"].as() == std::string("\\\n")); -} - -TEST_CASE("test_u0000") -{ - std::string inputStr("[\"\\u0040\\u0040\\u0000\\u0011\"]"); - //std::cout << "Input: " << inputStr << std::endl; - json arr = json::parse(inputStr); - - std::string s = arr[0].as(); - REQUIRE(4 == s.length()); - CHECK(static_cast(s[0]) == 0x40); - CHECK(static_cast(s[1]) == 0x40); - CHECK(static_cast(s[2]) == 0x00); - CHECK(static_cast(s[3]) == 0x11); - - std::ostringstream os; - os << arr; - std::string s2 = os.str(); - - //std::cout << std::hex << "Output: " << os.str() << std::endl; - -} - -TEST_CASE("test_uHHHH") -{ - std::string inputStr("[\"\\u007F\\u07FF\\u0800\"]"); - json arr = json::parse(inputStr); - - std::string s = arr[0].as(); - REQUIRE(s.length() == 6); - CHECK(static_cast(s[0]) == 0x7f); - CHECK(static_cast(s[1]) == 0xdf); - CHECK(static_cast(s[2]) == 0xbf); - CHECK(static_cast(s[3]) == 0xe0); - CHECK(static_cast(s[4]) == 0xa0); - CHECK(static_cast(s[5]) == 0x80); - - std::ostringstream os; - json_options options; - options.escape_all_non_ascii(true); - arr.dump(os, options); - std::string outputStr = os.str(); - - json arr2 = json::parse(outputStr); - std::string s2 = arr2[0].as(); - REQUIRE(s2.length() == 6); - CHECK(static_cast(s2[0]) == 0x7f); - CHECK(static_cast(s2[1]) == 0xdf); - CHECK(static_cast(s2[2]) == 0xbf); - CHECK(static_cast(s2[3]) == 0xe0); - CHECK(static_cast(s2[4]) == 0xa0); - CHECK(static_cast(s2[5]) == 0x80); - -} - -TEST_CASE("test_multiline_comments") -{ - std::string path = "./input/json-multiline-comment.json"; - std::fstream is(path); - if (!is) - { - std::cout << "Cannot open " << path << std::endl; - return; - } - json j = json::parse(is); - - CHECK(j.is_array()); - CHECK(j.is()); - CHECK(j.size() == 0); -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpatch/jsonpatch_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpatch/jsonpatch_tests.cpp deleted file mode 100644 index 3d4de8375c..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpatch/jsonpatch_tests.cpp +++ /dev/null @@ -1,496 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; -using namespace jsoncons::literals; - -void check_patch(json& target, const json& patch, std::error_code expected_ec, const json& expected) -{ - std::error_code ec; - jsonpatch::apply_patch(target, patch, ec); - if (ec != expected_ec || expected != target) - { - std::cout << "target:\n" << target << std::endl; - } - CHECK(ec == expected_ec); - CHECK(target == expected); -} - -TEST_CASE("add_an_object_member") -{ - json target = R"( - { "foo": "bar"} - )"_json; - - json patch = R"( - [ - { "op": "add", "path": "/baz", "value": "qux" } - ] - )"_json; - - json expected = R"( - {"baz":"qux","foo":"bar"} - )"_json; - - check_patch(target,patch,std::error_code(),expected); -} - -TEST_CASE("add_an_array_element") -{ - json target = R"( - { "foo": [ "bar", "baz" ] } - )"_json; - - json patch = R"( - [ - { "op": "add", "path": "/foo/1", "value": "qux" } - ] - )"_json; - - json expected = R"( - { "foo": [ "bar", "qux", "baz" ] } - )"_json; - - check_patch(target,patch,std::error_code(),expected); -} - -TEST_CASE("remove_an_object_member") -{ - json target = R"( - { - "baz": "qux", - "foo": "bar" - } - )"_json; - - json patch = R"( - [ - { "op": "remove", "path": "/baz" } - ] - )"_json; - - json expected = R"( - { "foo": "bar" } - )"_json; - - check_patch(target,patch,std::error_code(),expected); -} - -TEST_CASE("remove_an_array_element") -{ - json target = R"( - { "foo": [ "bar", "qux", "baz" ] } - )"_json; - - json patch = R"( - [ - { "op": "remove", "path": "/foo/1" } - ] - )"_json; - - json expected = R"( - { "foo": [ "bar", "baz" ] } - )"_json; - - check_patch(target,patch,std::error_code(),expected); -} - -TEST_CASE("replace_a_value") -{ - json target = R"( - { - "baz": "qux", - "foo": "bar" - } - )"_json; - - json patch = R"( - [ - { "op": "replace", "path": "/baz", "value": "boo" } - ] - )"_json; - - json expected = R"( - { - "baz": "boo", - "foo": "bar" - } - )"_json; - - check_patch(target,patch,std::error_code(),expected); -} - -TEST_CASE("move_a_value") -{ - json target = R"( - { - "foo": { - "bar": "baz", - "waldo": "fred" - }, - "qux": { - "corge": "grault" - } - } - )"_json; - - json patch = R"( - [ - { "op": "move", "from": "/foo/waldo", "path": "/qux/thud" } - ] - )"_json; - - json expected = R"( - { - "foo": { - "bar": "baz" - }, - "qux": { - "corge": "grault", - "thud": "fred" - } - } - )"_json; - - check_patch(target,patch,std::error_code(),expected); -} - -TEST_CASE("move_an_array_element") -{ - json target = R"( - { "foo": [ "all", "grass", "cows", "eat" ] } - )"_json; - - json patch = R"( - [ - { "op": "move", "from": "/foo/1", "path": "/foo/3" } - ] - )"_json; - - json expected = R"( - { "foo": [ "all", "cows", "eat", "grass" ] } - )"_json; - - check_patch(target,patch,std::error_code(),expected); -} - -TEST_CASE("add_to_nonexistent_target") -{ - json target = R"( - { "foo": "bar" } - )"_json; - - json patch = R"( - [ - { "op": "add", "path": "/baz/bat", "value": "qux" } - ] - )"_json; - - json expected = target; - - check_patch(target,patch,jsonpatch::jsonpatch_errc::add_failed,expected); -} - -TEST_CASE("testing_a_value_success") -{ - json target = R"( - { - "baz": "qux", - "foo": [ "a", 2, "c" ] - } - )"_json; - - json patch = R"( - [ - { "op": "test", "path": "/baz", "value": "qux" }, - { "op": "test", "path": "/foo/1", "value": 2 } - ] - )"_json; - - json expected = target; - - check_patch(target,patch,std::error_code(),expected); -} - -TEST_CASE("testing_a_value_error") -{ - json target = R"( - { "baz": "qux" } - - )"_json; - - json patch = R"( - [ - { "op": "test", "path": "/baz", "value": "bar" } - ] - )"_json; - - json expected = target; - - check_patch(target,patch,jsonpatch::jsonpatch_errc::test_failed,expected); -} - -TEST_CASE("adding_nested_member_object") -{ - json target = R"( - { "foo": "bar" } - - )"_json; - - json patch = R"( - [ - { "op": "add", "path": "/child", "value": { "grandchild": { } } } - ] - )"_json; - - json expected = R"( - { - "foo": "bar", - "child": { - "grandchild": { - } - } - } - - )"_json; - - check_patch(target,patch,std::error_code(),expected); -} - -TEST_CASE("tilde_escape_ordering") -{ - json target = R"( - { - "/": 9, - "~1": 10 - } - - )"_json; - - json patch = R"( - [ - {"op": "test", "path": "/~01", "value": 10} - ] - )"_json; - - json expected = R"( - { - "/": 9, - "~1": 10 - } - - )"_json; - - check_patch(target,patch,std::error_code(),expected); -} - -TEST_CASE("comparing_strings_and_numbers") -{ - json target = R"( - { - "/": 9, - "~1": 10 - } - - )"_json; - - json patch = R"( - [ - {"op": "test", "path": "/~01", "value": "10"} - ] - )"_json; - - json expected = target; - - check_patch(target,patch,jsonpatch::jsonpatch_errc::test_failed,expected); -} - -TEST_CASE("adding_an_array_value") -{ - json target = R"( - { "foo": ["bar"] } - - )"_json; - - json patch = R"( - [ - { "op": "add", "path": "/foo/-", "value": ["abc", "def"] } - ] - )"_json; - - json expected = R"( - { "foo": ["bar", ["abc", "def"]] } - - )"_json; - - check_patch(target,patch,std::error_code(),expected); -} - -TEST_CASE("test_add_add") -{ - json target = R"( - { "foo": "bar"} - )"_json; - - json patch = R"( - [ - { "op": "add", "path": "/baz", "value": "qux" }, - { "op": "add", "path": "/foo", "value": [ "bar", "baz" ] } - ] - )"_json; - - json expected = R"( - { "baz":"qux", "foo": [ "bar", "baz" ]} - )"_json; - - check_patch(target,patch,std::error_code(),expected); -} - -TEST_CASE("test_add_add_add_fail") -{ - json target = R"( - { "foo": "bar"} - )"_json; - - json patch = R"( - [ - { "op": "add", "path": "/baz", "value": "qux" }, - { "op": "add", "path": "/foo", "value": [ "bar", "baz" ] }, - { "op": "add", "path": "/baz/bat", "value": "qux" } // nonexistent target - ] - )"_json; - - json expected = target; - - check_patch(target,patch,jsonpatch::jsonpatch_errc::add_failed,expected); -} - -TEST_CASE("test_add_remove_remove_fail") -{ - json target = R"( - { - "baz": "boo", - "foo": [ "bar", "qux", "baz" ] - } - )"_json; - - json patch = R"( - [ - { "op": "add", "path": "/baz", "value": "qux" }, - { "op": "remove", "path": "/foo/2" }, - { "op": "remove", "path": "/foo/2" } // nonexistent target - ] - )"_json; - - json expected = target; - - check_patch(target,patch,jsonpatch::jsonpatch_errc::remove_failed,expected); -} - -TEST_CASE("test_move_copy_replace_remove_fail") -{ - json target = R"( - { - "baz": ["boo"], - "foo": [ "bar", "qux", "baz" ] - } - )"_json; - - json patch = R"( - [ - { "op": "add", "path": "/baz/-", "value": "xyz" }, - { "op": "move", "from": "/foo/1", "path" : "/baz/-" }, - { "op": "copy", "from": "/baz/0", "path" : "/foo/-" }, - { "op": "replace", "path": "/foo/2", "value" : "qux" }, - { "op": "remove", "path": "/foo/3" } // nonexistent target - ] - )"_json; - - json expected = target; - - check_patch(target,patch,jsonpatch::jsonpatch_errc::remove_failed,expected); - -} - -TEST_CASE("test_diff1") -{ - json source = R"( - {"/": 9, "~1": 10, "foo": "bar"} - )"_json; - - json target = R"( - { "baz":"qux", "foo": [ "bar", "baz" ]} - )"_json; - - auto patch = jsonpatch::from_diff(source, target); - - check_patch(source,patch,std::error_code(),target); -} - -TEST_CASE("test_diff2") -{ - json source = R"( - { - "/": 3, - "foo": "bar" - } - )"_json; - - json target = R"( - { - "/": 9, - "~1": 10 - } - )"_json; - - auto patch = jsonpatch::from_diff(source, target); - - check_patch(source,patch,std::error_code(),target); -} - -TEST_CASE("add_when_new_items_in_target_array1") -{ - jsoncons::json source = R"( - {"/": 9, "foo": [ "bar"]} - )"_json; - - jsoncons::json target = R"( - { "baz":"qux", "foo": [ "bar", "baz" ]} - )"_json; - - jsoncons::json patch = jsoncons::jsonpatch::from_diff(source, target); - - check_patch(source,patch,std::error_code(),target); -} - -TEST_CASE("add_when_new_items_in_target_array2") -{ - jsoncons::json source = R"( - {"/": 9, "foo": [ "bar", "bar"]} - )"_json; - - jsoncons::json target = R"( - { "baz":"qux", "foo": [ "bar", "baz" ]} - )"_json; - - jsoncons::json patch = jsoncons::jsonpatch::from_diff(source, target); - - check_patch(source,patch,std::error_code(),target); -} - - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/JSONPathTestSuite_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/JSONPathTestSuite_tests.cpp deleted file mode 100644 index 1c09931ddd..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/JSONPathTestSuite_tests.cpp +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2013-2018 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if defined(_MSC_VER) && _MSC_VER >= 1900 -#include -namespace fs = std::experimental::filesystem; -#else -//#include -//namespace fs = std::filesystem; -#endif - -using namespace jsoncons; -using namespace jsoncons::jsonpath; - -#if defined(_MSC_VER) && _MSC_VER >= 1900 -TEST_CASE("JSONPath Test Suite") -{ - ojson document; - std::map query_dictionary; - std::map expected_dictionary; - - std::string path = "./input/JSONPathTestSuite"; - for (auto& p : fs::directory_iterator(path)) - { - if (fs::exists(p) && fs::is_regular_file(p)) - { - if (p.path().filename() == "document.json") - { - try - { - std::ifstream is(p.path().c_str()); - document = ojson::parse(is); - } - catch(const std::exception& e) - { - std::cerr << e.what() << std::endl; - return; - } - } - else if (p.path().extension() == ".jsonpath") - { - std::string s; - char buffer[4096]; - std::ifstream is(p.path().c_str()); - while (is.read(buffer, sizeof(buffer))) - { - s.append(buffer, sizeof(buffer)); - } - s.append(buffer, (size_t)is.gcount()); - query_dictionary[p.path().stem()] = s; - } - else if (p.path().extension() == ".json") - { - try - { - ojson j; - std::ifstream is(p.path().c_str()); - j = ojson::parse(is); - expected_dictionary[p.path().stem()] = j; - } - catch (const jsoncons::ser_error& e) - { - std::cerr << e.what() << std::endl; - return; - } - } - } - } - - for (auto pair : query_dictionary) - { - auto it = expected_dictionary.find(pair.first); - if (it != expected_dictionary.end()) - { - try - { - ojson result = json_query(document, pair.second); - CHECK(it->second == result); - } - catch (const jsoncons::jsonpath::jsonpath_error& e) - { - ojson result = json_query(document, pair.second); - std::cerr << pair.first << " " << pair.second << " " << e.what() << std::endl; - } - catch (const std::exception& e) - { - std::cerr << e.what() << std::endl; - } - } - else - { - std::cout << "Expected value for " << pair.first << "not found \n"; - std::cout << pair.second << '\n'; - ojson result = json_query(document,pair.second); - std::cout << pretty_print(result) << std::endl; - } - } -} -#endif - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_error_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_error_tests.cpp deleted file mode 100644 index 6664e708dd..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_error_tests.cpp +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -struct jsonpath_fixture -{ - static const char* store_text() - { - static const char* text = "{ \"store\": {\"book\": [ { \"category\": \"reference\",\"author\": \"Nigel Rees\",\"title\": \"Sayings of the Century\",\"price\": 8.95},{ \"category\": \"fiction\",\"author\": \"Evelyn Waugh\",\"title\": \"Sword of Honour\",\"price\": 12.99},{ \"category\": \"fiction\",\"author\": \"Herman Melville\",\"title\": \"Moby Dick\",\"isbn\": \"0-553-21311-3\",\"price\": 8.99},{ \"category\": \"fiction\",\"author\": \"J. R. R. Tolkien\",\"title\": \"The Lord of the Rings\",\"isbn\": \"0-395-19395-8\",\"price\": 22.99}],\"bicycle\": {\"color\": \"red\",\"price\": 19.95}}}"; - return text; - } - static const char* store_text_empty_isbn() - { - static const char* text = "{ \"store\": {\"book\": [ { \"category\": \"reference\",\"author\": \"Nigel Rees\",\"title\": \"Sayings of the Century\",\"price\": 8.95},{ \"category\": \"fiction\",\"author\": \"Evelyn Waugh\",\"title\": \"Sword of Honour\",\"price\": 12.99},{ \"category\": \"fiction\",\"author\": \"Herman Melville\",\"title\": \"Moby Dick\",\"isbn\": \"0-553-21311-3\",\"price\": 8.99},{ \"category\": \"fiction\",\"author\": \"J. R. R. Tolkien\",\"title\": \"The Lord of the Rings\",\"isbn\": \"\",\"price\": 22.99}],\"bicycle\": {\"color\": \"red\",\"price\": 19.95}}}"; - return text; - } - static const char* book_text() - { - static const char* text = "{ \"category\": \"reference\",\"author\": \"Nigel Rees\",\"title\": \"Sayings of the Century\",\"price\": 8.95}"; - return text; - } - - json book() - { - json root = json::parse(jsonpath_fixture::store_text()); - json book = root["store"]["book"]; - return book; - } - - json bicycle() - { - json root = json::parse(jsonpath_fixture::store_text()); - json bicycle = root["store"]["bicycle"]; - return bicycle; - } -}; - -void test_error_code(const json& root, const std::string& path, std::error_code value, size_t line, size_t column) -{ - REQUIRE_THROWS_AS(jsonpath::json_query(root,path),jsonpath::jsonpath_error); - try - { - json result = jsonpath::json_query(root,path); - } - catch (const jsonpath::jsonpath_error& e) - { - if (e.code() != value) - { - std::cout << path << "\n"; - } - CHECK(e.code() == value); - CHECK(e.line() == line); - CHECK(e.column() == column); - } -} - -/*TEST_CASE("test_root_error") -{ - json root = json::parse(jsonpath_fixture::store_text()); - test_error_code(root, "..*", jsonpath_errc::expected_root,1,1);` -}*/ - -TEST_CASE("test_right_bracket_error") -{ - - json root = json::parse(jsonpath_fixture::store_text()); - test_error_code(root, "$['store']['book'[*]", jsonpath::jsonpath_errc::unexpected_end_of_input,1,21); -} - -TEST_CASE("jsonpath missing dot") -{ - json root = json::parse(jsonpath_fixture::store_text()); - test_error_code(root, "$.store.book[?(@category == 'fiction')][?(@.price < 15)].title", jsonpath::jsonpath_errc::expected_separator,1,17); -} - -TEST_CASE("test_dot_dot_dot") -{ - - json root = json::parse(jsonpath_fixture::store_text()); - test_error_code(root, "$.store...price", jsonpath::jsonpath_errc::expected_name,1,10); -} - -TEST_CASE("test_dot_star_name") -{ - - json root = json::parse(jsonpath_fixture::store_text()); - test_error_code(root, "$.store.*price", jsonpath::jsonpath_errc::expected_separator,1,10); -} -TEST_CASE("test_filter_error") -{ - json root = json::parse(jsonpath_fixture::store_text()); - std::string path = "$..book[?(.price<10)]"; - test_error_code(root, path, jsonpath::jsonpath_errc::parse_error_in_filter,1,17); -} - -TEST_CASE("jsonpath slice errors") -{ - json root = json::parse(R"( -[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19] -)"); - - SECTION("$[") - { - std::string path = "$["; - test_error_code(root, path, jsonpath::jsonpath_errc::unexpected_end_of_input,1,3); - } - SECTION("$[1") - { - std::string path = "$[1"; - test_error_code(root, path, jsonpath::jsonpath_errc::unexpected_end_of_input,1,4); - } - SECTION("$[1:") - { - std::string path = "$[1:"; - test_error_code(root, path, jsonpath::jsonpath_errc::unexpected_end_of_input,1,5); - } - SECTION("$[1:1") - { - std::string path = "$[1:1"; - test_error_code(root, path, jsonpath::jsonpath_errc::unexpected_end_of_input,1,6); - } - - SECTION("$[-:]") - { - std::string path = "$[-:]"; - test_error_code(root, path, jsonpath::jsonpath_errc::expected_slice_start,1,4); - } - - SECTION("$[-1:-]") - { - std::string path = "$[-1:-]"; - test_error_code(root, path, jsonpath::jsonpath_errc::expected_slice_end,1,7); - } - - SECTION("$[-1:-1:-]") - { - std::string path = "$[-1:-1:-]"; - test_error_code(root, path, jsonpath::jsonpath_errc::expected_slice_step,1,10); - } -} - - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_filter_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_filter_tests.cpp deleted file mode 100644 index b946b563de..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_filter_tests.cpp +++ /dev/null @@ -1,488 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; -using namespace jsoncons::jsonpath; -using namespace jsoncons::jsonpath::detail; - -struct jsonpath_filter_fixture -{ - static const char* store_text() - { - static const char* text = "{ \"store\": {\"book\": [ { \"category\": \"reference\",\"author\": \"Nigel Rees\",\"title\": \"Sayings of the Century\",\"price\": 8.95},{ \"category\": \"fiction\",\"author\": \"Evelyn Waugh\",\"title\": \"Sword of Honour\",\"price\": 12.99},{ \"category\": \"fiction\",\"author\": \"Herman Melville\",\"title\": \"Moby Dick\",\"isbn\": \"0-553-21311-3\",\"price\": 8.99},{ \"category\": \"fiction\",\"author\": \"J. R. R. Tolkien\",\"title\": \"The Lord of the Rings\",\"isbn\": \"0-395-19395-8\",\"price\": 22.99}],\"bicycle\": {\"color\": \"red\",\"price\": 19.95}}}"; - return text; - } - static const char* book_text() - { - static const char* text = "{ \"category\": \"reference\",\"author\": \"Nigel Rees\",\"title\": \"Sayings of the Century\",\"price\": 8.95}"; - return text; - } - - json book() - { - json root = json::parse(jsonpath_filter_fixture::store_text()); - json book = root["store"]["book"]; - return book; - } - - json bicycle() - { - json root = json::parse(jsonpath_filter_fixture::store_text()); - json bicycle = root["store"]["bicycle"]; - return bicycle; - } -}; - -TEST_CASE("test_div") -{ - const char* pend; - jsonpath_filter_parser parser; - - json context = json::array(); - context.push_back(3); - - std::string s1 = "(3/1)"; - auto expr1 = parser.parse(context,s1.c_str(), s1.c_str()+ s1.length(), &pend); - auto result1 = expr1.eval(context); - CHECK(json(3) == result1); - - std::string s2 = "(3/@.length)"; - auto expr2 = parser.parse(context,s2.c_str(), s2.c_str()+ s2.length(), &pend); - auto result2 = expr2.eval(context); - CHECK(json(3) == result2); - - std::string s3 = "(5/2)"; - auto expr3 = parser.parse(context,s3.c_str(), s3.c_str()+ s3.length(), &pend); - auto result3 = expr3.eval(context); - CHECK(json(2.5) == result3); - - std::string s4 = "(@.length/3)"; - auto expr4 = parser.parse(context,s4.c_str(), s4.c_str()+ s4.length(), &pend); - auto result4 = expr4.eval(context); - CHECK(0.333333 == Approx(result4.as()).epsilon(0.001)); - - std::string s5 = "(@.0/@.length)"; - auto expr5 = parser.parse(context,s5.c_str(), s5.c_str()+ s5.length(), &pend); - auto result5 = expr5.eval(context); - CHECK(json(3) == result5); -} - -TEST_CASE("test_mult") -{ - const char* pend; - jsonpath_filter_parser parser; - - json context = json::array(); - context.push_back(1); - context.push_back(2); - - std::string s1 = "(3*1)"; - auto expr1 = parser.parse(context, s1.c_str(), s1.c_str()+ s1.length(), &pend); - auto result1 = expr1.eval(context); - CHECK(json(3) == result1); - - std::string s2 = "(3*@.length)"; - auto expr2 = parser.parse(context, s2.c_str(), s2.c_str()+ s2.length(), &pend); - auto result2 = expr2.eval(context); - CHECK(json(6) == result2); - - std::string s3 = "(5*2)"; - auto expr3 = parser.parse(context, s3.c_str(), s3.c_str()+ s3.length(), &pend); - auto result3 = expr3.eval(context); - CHECK(json(10) == result3); - - std::string s4 = "(@.length*3)"; - auto expr4 = parser.parse(context, s4.c_str(), s4.c_str()+ s4.length(), &pend); - auto result4 = expr4.eval(context); - CHECK(json(6) == result4); - - std::string s5 = "(@.length*@.1)"; - auto expr5 = parser.parse(context, s5.c_str(), s5.c_str()+ s5.length(), &pend); - auto result5 = expr5.eval(context); - CHECK(json(4) == result5); -} - -TEST_CASE("test_minus") -{ - const char* pend; - jsonpath_filter_parser parser; - - json context = json::array(); - context.push_back(10.0); - - std::string s1 = "(3-1)"; - auto expr1 = parser.parse(context, s1.c_str(), s1.c_str()+ s1.length(), &pend); - auto result1 = expr1.eval(context); - CHECK(json(2) == result1); - - std::string s2 = "(3-@.length)"; - auto expr2 = parser.parse(context, s2.c_str(), s2.c_str()+ s2.length(), &pend); - auto result2 = expr2.eval(context); - CHECK(json(2) == result2); - - std::string s3 = "(3.5-1.0)"; - auto expr3 = parser.parse(context, s3.c_str(), s3.c_str()+ s3.length(), &pend); - auto result3 = expr3.eval(context); - CHECK(json(2.5) == result3); - - std::string s4 = "(@.length-3)"; - auto expr4 = parser.parse(context, s4.c_str(), s4.c_str()+ s4.length(), &pend); - auto result4 = expr4.eval(context); - CHECK(json(-2) ==result4); - - std::string s5 = "(@.length-@.0)"; - auto expr5 = parser.parse(context, s5.c_str(), s5.c_str()+ s5.length(), &pend); - auto result5 = expr5.eval(context); - CHECK(json(-9) ==result5); -} - -TEST_CASE("test_lt") -{ - const char* pend; - jsonpath_filter_parser parser; - - json context = json::array(); - context.push_back(100); - context.push_back(1); - - std::string s1 = "(3 < 1)"; - auto expr1 = parser.parse(context, s1.c_str(), s1.c_str()+ s1.length(), &pend); - auto result1 = expr1.eval(context); - CHECK(result1 == json(false)); - - std::string s2 = "(3 < @.length)"; - auto expr2 = parser.parse(context, s2.c_str(), s2.c_str()+ s2.length(), &pend); - auto result2 = expr2.eval(context); - CHECK(result2 == json(false)); - - std::string s3 = "(@.length < 3)"; - auto expr3 = parser.parse(context, s3.c_str(), s3.c_str()+ s3.length(), &pend); - auto result3 = expr3.eval(context); - CHECK(result3 == json(true)); - - std::string s4 = "(@.length < @.length)"; - auto expr4 = parser.parse(context, s4.c_str(), s4.c_str()+ s4.length(), &pend); - auto result4 = expr4.eval(context); - CHECK(result4 == json(false)); - - std::string s5 = "(@.length < @.0)"; - auto expr5 = parser.parse(context, s5.c_str(), s5.c_str()+ s5.length(), &pend); - auto result5 = expr5.eval(context); - CHECK(json(true) == result5); - - std::string s6 = "(@.length < @.1)"; - auto expr6 = parser.parse(context, s6.c_str(), s6.c_str()+ s6.length(), &pend); - auto result6 = expr6.eval(context); - CHECK(json(false) == result6); -} - -TEST_CASE("test_lte") -{ - const char* pend; - jsonpath_filter_parser parser; - - json context = json::array(); - context.push_back(1); - - std::string s1 = "(3 <= 1)"; - auto expr1 = parser.parse(context, s1.c_str(), s1.c_str()+ s1.length(), &pend); - auto result1 = expr1.eval(context); - CHECK(result1 == json(false)); - - std::string s2 = "(3 <= @.length)"; - auto expr2 = parser.parse(context, s2.c_str(), s2.c_str()+ s2.length(), &pend); - auto result2 = expr2.eval(context); - CHECK(result2 == json(false)); -} - -TEST_CASE("test_gt") -{ - const char* pend; - jsonpath_filter_parser parser; - - json context = json::array(); - context.push_back(1); - - std::string s1 = "(3 > 1)"; - auto expr1 = parser.parse(context, s1.c_str(), s1.c_str()+ s1.length(), &pend); - auto result1 = expr1.eval(context); - CHECK(result1 == json(true)); - - std::string s2 = "(3 > @.length)"; - auto expr2 = parser.parse(context, s2.c_str(), s2.c_str()+ s2.length(), &pend); - auto result2 = expr2.eval(context); - CHECK(result2 == json(true)); -} - -TEST_CASE("test_gte") -{ - const char* pend; - jsonpath_filter_parser parser; - - json context = json::array(); - context.push_back(1); - - std::string s1 = "(3 >= 1)"; - auto expr1 = parser.parse(context, s1.c_str(), s1.c_str()+ s1.length(), &pend); - auto result1 = expr1.eval(context); - CHECK(result1 == json(true)); - - std::string s2 = "(3 >= @.length)"; - auto expr2 = parser.parse(context, s2.c_str(), s2.c_str()+ s2.length(), &pend); - auto result2 = expr2.eval(context); - CHECK(result2 == json(true)); -} - -TEST_CASE("test_eq") -{ - const char* pend; - jsonpath_filter_parser parser; - - json context = json::array(); - context.push_back(1); - - std::string s1 = "(3 == 1)"; - auto expr1 = parser.parse(context, s1.c_str(), s1.c_str()+ s1.length(), &pend); - auto result1 = expr1.eval(context); - CHECK(result1 == json(false)); - - std::string s2 = "(3 == @.length)"; - auto expr2 = parser.parse(context, s2.c_str(), s2.c_str()+ s2.length(), &pend); - auto result2 = expr2.eval(context); - CHECK(result2 == json(false)); - - std::string s3 = "(1 == 1)"; - auto expr3 = parser.parse(context, s3.c_str(), s3.c_str()+ s3.length(), &pend); - auto result3 = expr3.eval(context); - CHECK(result3 == json(true)); - - std::string s4 = "(1 == @.length)"; - auto expr4 = parser.parse(context, s4.c_str(), s4.c_str()+ s4.length(), &pend); - auto result4 = expr4.eval(context); - CHECK(result4 == json(true)); -} - -TEST_CASE("test_precedence") -{ - const char* pend; - jsonpath_filter_parser parser; - - json context = json::array(); - context.push_back(1); - context.push_back(2); - - std::string s1 = "(@.0 == 1 && @.1 == 2)"; - auto expr1 = parser.parse(context, s1.c_str(), s1.c_str()+ s1.length(), &pend); - auto result1 = expr1.eval(context); - CHECK(result1 == json(true)); - - std::string s2 = "((@.0 == 1) && (@.1 == 2))"; - auto expr2 = parser.parse(context, s2.c_str(), s2.c_str()+ s2.length(), &pend); - auto result2 = expr2.eval(context); - CHECK(result2 == json(true)); - - std::string s3 = "(@.0 == 2 && @.1 == 2)"; - auto expr3 = parser.parse(context, s3.c_str(), s3.c_str()+ s3.length(), &pend); - auto result3 = expr3.eval(context); - CHECK(result3 == json(false)); - - std::string s4 = "((@.0 == 1) && (@.1 == 1))"; - auto expr4 = parser.parse(context, s4.c_str(), s4.c_str()+ s4.length(), &pend); - auto result4 = expr4.eval(context); - CHECK(result4 == json(false)); -} - -TEST_CASE("test_ne") -{ - const char* pend; - jsonpath_filter_parser parser; - - json context = json::array(); - context.push_back(1); - - std::string s1 = "(3 != 1)"; - auto expr1 = parser.parse(context, s1.c_str(), s1.c_str()+ s1.length(), &pend); - auto result1 = expr1.eval(context); - CHECK(result1 == json(true)); - - std::string s2 = "(3 != @.length)"; - auto expr2 = parser.parse(context, s2.c_str(), s2.c_str()+ s2.length(), &pend); - auto result2 = expr2.eval(context); - CHECK(result2 == json(true)); - - std::string s3 = "(1 != 1)"; - auto expr3 = parser.parse(context, s3.c_str(), s3.c_str()+ s3.length(), &pend); - auto result3 = expr3.eval(context); - CHECK(result3 == json(false)); - - std::string s4 = "(1 != @.length)"; - auto expr4 = parser.parse(context, s4.c_str(), s4.c_str()+ s4.length(), &pend); - auto result4 = expr4.eval(context); - CHECK(result4 == json(false)); -} - -TEST_CASE("test_jsonpath_filter") -{ - const char* pend; - jsonpath_filter_parser parser; - json parent = json::array(); - parent.push_back(1); - parent.push_back(2); - - std::string expr1 = "(1 + 1)"; - auto res1 = parser.parse(parent, expr1.c_str(), expr1.c_str()+ expr1.length(), &pend); - auto result1 = res1.eval(parent); - CHECK(json(2) == result1); - - std::string expr2 = "(1 - 1)"; - auto res2 = parser.parse(parent, expr2.c_str(), expr2.c_str()+ expr2.length(), &pend); - auto result2 = res2.eval(parent); - CHECK(json(0) == result2); - - std::string expr3 = "(@.length - 1)"; - auto res3 = parser.parse(parent, expr3.c_str(), expr3.c_str()+ expr3.length(), &pend); - auto result3 = res3.eval(parent); - CHECK(json(1) == result3); - -} - -TEST_CASE("test_jsonpath_filter_exclaim") -{ - const char* pend; - jsonpath_filter_parser parser; - json parent = json::array(); - parent.push_back(1); - parent.push_back(2); - - std::string expr1 = "(!(1 + 1))"; - auto res1 = parser.parse(parent, expr1.c_str(), expr1.c_str()+ expr1.length(), &pend); - auto result1 = res1.eval(parent); - CHECK(result1 == json(false)); - - std::string expr2 = "(!0)"; - auto res2 = parser.parse(parent, expr2.c_str(), expr2.c_str()+ expr2.length(), &pend); - auto result2= res2.eval(parent); - CHECK(result2 == json(true)); -} - - -TEST_CASE("test_jsonpath_index_expression") -{ - json root = json::parse(jsonpath_filter_fixture::store_text()); - //std::cout << pretty_print(root) << std::endl; - //std::cout << "$..book[(@.length-1)]" << std::endl; - - json result = json_query(root,"$..book[(@.length-1)]"); - - CHECK(1 == result.size()); - CHECK(root["store"]["book"][3] == result[0]); - - // std::cout << pretty_print(result) << std::endl; -} - -TEST_CASE("test_jsonpath_filter_negative_numbers") -{ - const char* pend; - jsonpath_filter_parser parser; - json parent = json::array(); - parent.push_back(1); - parent.push_back(2); - - std::string expr1 = "(-1 + 1)"; - auto res1 = parser.parse(parent, expr1.c_str(), expr1.c_str()+ expr1.length(), &pend); - auto result1 = res1.eval(parent); - CHECK(json(0) == result1); - - std::string expr2 = "(1 + -1)"; - auto res2 = parser.parse(parent, expr2.c_str(), expr2.c_str()+ expr2.length(), &pend); - auto result2 = res2.eval(parent); - CHECK(json(0) == result2); - - std::string expr3 = "(-1 - -1)"; - auto res3 = parser.parse(parent, expr3.c_str(), expr3.c_str()+ expr3.length(), &pend); - auto result3 = res3.eval(parent); - CHECK(json(0) == result3); - - std::string expr4 = "(-1 - -3)"; - auto res4 = parser.parse(parent, expr4.c_str(), expr4.c_str()+ expr4.length(), &pend); - auto result4 = res4.eval(parent); - CHECK(json(2) == result4); - - std::string expr5 = "((-2 < -1) && (-3 > -4))"; - auto res5 = parser.parse(parent, expr5.c_str(), expr5.c_str()+ expr5.length(), &pend); - auto result5 = res5.eval(parent); - CHECK(json(true) == result5); - - std::string expr6 = "((-2 < -1) || (-4 > -3))"; - auto res6 = parser.parse(parent, expr6.c_str(), expr6.c_str()+ expr6.length(), &pend); - auto result6 = res6.eval(parent); - CHECK(json(true) == result6); - - std::string expr7 = "(-2 < -1 && -3 > -4)"; - auto res7 = parser.parse(parent, expr7.c_str(), expr7.c_str()+ expr7.length(), &pend); - auto result7 = res7.eval(parent); - CHECK(json(true) == result7); - - std::string expr8 = "(-2 < -1 || -4 > -3)"; - auto res8 = parser.parse(parent, expr8.c_str(), expr8.c_str()+ expr8.length(), &pend); - auto result8 = res8.eval(parent); - CHECK(json(true) == result8); -} - -TEST_CASE("test_jsonpath_filter_uni") -{ - const char* pend; - jsonpath_filter_parser parser; - json parent = json::array(); - parent.push_back(1); - parent.push_back(2); - - std::string expr1 = "(0)"; - auto res = parser.parse(parent, expr1.c_str(), expr1.c_str()+ expr1.length(), &pend); - auto result1 = res.eval(parent); - - //std::cout << (int)result1.data_type() << std::endl; - CHECK(result1 == json(0)); - - CHECK(json(0) == result1); -} - -#if !(defined(__GNUC__) && (__GNUC__ == 4 && __GNUC_MINOR__ < 9)) -// GCC 4.8 has broken regex support: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53631 -TEST_CASE("test_jsonpath_filter_regex") -{ - const char* pend; - jsonpath_filter_parser parser; - json parent = json::array(); - parent.push_back(1); - parent.push_back(2); - - std::string expr1 = "('today I go' =~ /today.*?/)"; - auto res1 = parser.parse(parent, expr1.c_str(), expr1.c_str()+ expr1.length(), &pend); - auto result1 = res1.eval(parent); - CHECK(result1 == json(true)); - - std::string expr2 = "('today I go' =~ /Today.*?/)"; - auto res2 = parser.parse(parent, expr2.c_str(), expr2.c_str()+ expr2.length(), &pend); - auto result2 = res2.eval(parent); - CHECK(result2 == json(false)); - - std::string expr3 = "('today I go' =~ /Today.*?/i)"; - auto res3 = parser.parse(parent, expr3.c_str(), expr3.c_str()+ expr3.length(), &pend); - auto result3 = res3.eval(parent); - CHECK(result3 == json(true)); -} -#endif - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_function_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_function_tests.cpp deleted file mode 100644 index d46c09129e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_function_tests.cpp +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("jsonpath function tests") -{ - const json store = json::parse(R"( - { - "store": { - "book": [ - { - "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { - "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - }, - { - "category": "fiction", - "author": "Herman Melville", - "title": "Moby Dick", - "isbn": "0-553-21311-3", - "price": 8.99 - }, - { - "category": "fiction", - "author": "J. R. R. Tolkien", - "title": "The Lord of the Rings", - "isbn": "0-395-19395-8", - "price": 22.99 - } - ], - "bicycle": { - "color": "red", - "price": 19.95 - } - } - } - )"); - - SECTION("count") - { - json result = jsonpath::json_query(store,"count($.store.book[*])"); - - size_t expected = 4; - - REQUIRE(result.size() == 1); - CHECK(result[0].as() == expected); - } -#if 0 - SECTION("keys") - { - json result = json_query(store,"keys($.store.book[0])[*]"); - - json expected = json::parse("[\"author\",\"category\",\"price\",\"title\"]"); - - REQUIRE(result.size() == 4); - CHECK(result == expected); - } - SECTION("sum") - { - json result = json_query(store,"sum($.store.book[*].price)"); - double expected = 53.92; - REQUIRE(result.size() == 1); - CHECK(result[0].as() == Approx(expected).epsilon(0.000001)); - } - SECTION("sum in filter") - { - json result = json_query(store,"$.store.book[?(@.price > sum($.store.book[*].price) / count($.store.book[*]))].title"); - std::string expected = "The Lord of the Rings"; - REQUIRE(result.size() == 1); - CHECK(result[0].as() == expected); - } - SECTION("avg") - { - json result = json_query(store,"avg($.store.book[*].price)"); - - double expected = 13.48; - - REQUIRE(result.size() == 1); - CHECK(result[0].as() == Approx(expected).epsilon(0.000001)); - } - SECTION("avg in filter") - { - json result = json_query(store,"$.store.book[?(@.price > avg($.store.book[*].price))].title"); - std::string expected = "The Lord of the Rings"; - REQUIRE(result.size() == 1); - CHECK(result[0].as() == expected); - } - - SECTION("prod") - { - json result = json_query(store,"prod($.store.book[*].price)"); - - double expected = 24028.731766049998; - - REQUIRE(result.size() == 1); - CHECK(result[0].as() == Approx(expected).epsilon(0.000001)); - } - - SECTION("min") - { - json result = json_query(store,"min($.store.book[*].price)"); - - double expected = 8.95; - - REQUIRE(result.size() == 1); - CHECK(result[0].as() == Approx(expected).epsilon(0.000001)); - } - - SECTION("max") - { - json result = json_query(store,"max($.store.book[*].price)"); - - double expected = 22.99; - - REQUIRE(result.size() == 1); - CHECK(result[0].as() == Approx(expected).epsilon(0.000001)); - } - - SECTION("max in filter") - { - std::string path = "$.store.book[?(@.price < max($.store.book[*].price))].title"; - - json expected = json::parse(R"( - ["Sayings of the Century","Sword of Honour","Moby Dick"] - )"); - - json result = json_query(store,path); - - CHECK(result == expected); - } -#if !(defined(__GNUC__) && (__GNUC__ == 4 && __GNUC_MINOR__ < 9)) -// GCC 4.8 has broken regex support: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53631 - SECTION("tokenize") - { - json j("The cat sat on the mat"); - json result = json_query(j,"tokenize($,'\\\\s+')[*]"); - - json expected = json::parse("[\"The\",\"cat\",\"sat\",\"on\",\"the\",\"mat\"]"); - - CHECK(result == expected); - } - SECTION("tokenize in filter") - { - json j = json::parse("[\"The cat sat on the mat\",\"The cat on the mat\"]"); - json result = json_query(j,"$[?(tokenize(@,'\\\\s+')[2]=='sat')]"); - - CHECK(result[0] == j[0]); - } - SECTION("tokenize in filter 2") - { - json result = json_query(store,"$.store.book[?(tokenize(@.author,'\\\\s+')[1] == 'Waugh')].title"); - std::string expected = "Sword of Honour"; - REQUIRE(result.size() == 1); - CHECK(result[0].as() == expected); - } -#endif -#endif -} - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_normalized_path_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_normalized_path_tests.cpp deleted file mode 100644 index fd90c3a4c9..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_normalized_path_tests.cpp +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; -using namespace jsoncons::jsonpath; - -// https://jsonpath.herokuapp.com/ - -TEST_CASE("store normalized path tests") -{ - const json store = json::parse(R"( - { - "store": { - "book": [ - { - "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { - "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - }, - { - "category": "fiction", - "author": "Herman Melville", - "title": "Moby Dick", - "isbn": "0-553-21311-3", - "price": 8.99 - }, - { - "category": "fiction", - "author": "J. R. R. Tolkien", - "title": "The Lord of the Rings", - "isbn": "0-395-19395-8", - "price": 22.99 - } - ], - "bicycle": { - "color": "red", - "price": 19.95 - } - }, - "expensive": 10 - } - )"); - - SECTION("$.store.book[0].category") - { - const json expected = json::parse(R"( - [ - "$['store']['book'][0]['category']" - ] - )"); - - std::string path = "$.store.book[0].category"; - - json result = json_query(store,"$.store.book[0].category",result_type::path); - CHECK(result == expected); - } - SECTION("test_string_index") - { - - const json expected = json::parse(R"( - [ - "$['store']['book'][0]['category'][0]", - "$['store']['book'][0]['category'][2]" - ] - )"); - - std::string path = "$.store.book.0.category[0,2]"; - - json result = json_query(store,path,result_type::path); - //CHECK(result == expected); // revisit - - //json result2 = json_query(store,path,result_type::value); - //std::cout << pretty_print(result2) << std::endl; - } - - SECTION("test_array_length") - { - - const json expected = json::parse(R"( - [ - "$['store']['book']['length']" - ] - )"); - - std::string path = "$.store.book.length"; - json result = json_query(store,path,result_type::path); - CHECK(result == expected); - - std::string path2 = "$.store.book['length']"; - json result2 = json_query(store, path, result_type::path); - CHECK(result2 == expected); - } - - SECTION("test_price_filter") - { - - const json expected = json::parse(R"( - [ - "$['store']['book'][0]['title']", - "$['store']['book'][2]['title']" - ] - )"); - - std::string path = "$.store.book[?(@.price < 10)].title"; - json result = json_query(store,path,result_type::path); - CHECK(result == expected); - } - - SECTION("test_length_expression") - { - - const json expected = json::parse(R"( - [ - "$['store']['book'][3]['title']" - ] - )"); - - std::string path = "$.store.book[(@.length-1)].title"; - json result = json_query(store,path,result_type::path); - CHECK(result == expected); - } -} - -TEST_CASE("normalized path test") -{ - json test = json::parse(R"( - { - "books": [ - {"title": "b1", "chapters":[{"subtitle":"b1c1"}]}, - {"title": "b2", "chapters":[{"subtitle":"b2c1"}]} - ] - } )"); - - json expected = json::parse(R"([{"subtitle":"b2c1"}])"); - std::string expected_path = "$['books'][1]['chapters'][0]"; - - SECTION("$.books[*].chapters[?(@.subtitle == 'b2c1')]") - { - std::string path = "$.books[*].chapters[?(@.subtitle == 'b2c1')]"; - json result = json_query(test, path); - - json path_result = json_query(test, path, result_type::path); - CHECK(result == expected); - - REQUIRE(path_result.size() == 1); - CHECK(path_result[0].as() == expected_path); - } - SECTION("$..books[*].chapters[?(@.subtitle == 'b2c1')]") - { - std::string path = "$..books[*].chapters[?(@.subtitle == 'b2c1')]"; - json result = json_query(test, path); - CHECK(result == expected); - - json path_result = json_query(test, path, result_type::path); - - REQUIRE(path_result.size() == 1); - CHECK(path_result[0].as() == expected_path); - } - SECTION("$..[?(@.subtitle == 'b2c1')]") - { - std::string path = "$..[?(@.subtitle == 'b2c1')]"; - json result = json_query(test, path); - CHECK(result == expected); - - json path_result = json_query(test, path, result_type::path); - - REQUIRE(path_result.size() == 1); - CHECK(path_result[0].as() == expected_path); - } -} - -TEST_CASE("jsonpath object union normalized path test") -{ - json test = json::parse(R"( - { - "books": [ - {"title": "b1", "chapters":[{"subtitle":"b1c1"}]}, - {"title": "b2", "chapters":[{"subtitle":"b2c1"}]} - ] - } )"); - - - SECTION("$..[chapters[?(@.subtitle == 'b2c1')],chapters[?(@.subtitle == 'b2c1')]]") - { - json expected = json::parse(R"([{"subtitle":"b2c1"}])"); - json expected_path = json::parse(R"(["$['books'][1]['chapters[?(@.subtitle == 'b2c1')]']"])"); - - std::string path = "$..[chapters[?(@.subtitle == 'b2c1')],chapters[?(@.subtitle == 'b2c1')]]"; - json result = json_query(test, path); - CHECK(result == expected); - - json path_result = json_query(test, path, result_type::path); - - CHECK(path_result == expected_path); - } -} - -TEST_CASE("jsonpath array union normalized path test") -{ - json root = json::parse(R"( -[[1,2,3,4,1,2,3,4],[0,1,2,3,4,5,6,7,8,9],[0,1,2,3,4,5,6,7,8,9]] -)"); - - SECTION("$[0,1,2]") - { - json expected = json::parse(R"([[0,1,2,3,4,5,6,7,8,9],[1,2,3,4,1,2,3,4]])"); - json expected_path = json::parse(R"( ["$[1]","$[0]"])"); - - std::string path = "$[0,1,2]"; - json result = json_query(root, path); - CHECK(result == expected); - - json path_result = json_query(root, path, result_type::path); - CHECK(path_result == expected_path); - } - - SECTION("$[0][0:4,2:8]") - { - json expected = json::parse(R"([1,2,3,4])"); - json expected_path = json::parse(R"(["$[0][0]","$[0][1]","$[0][2]","$[0][3]"])"); - - std::string path = "$[0][0:4,2:8]"; - json result = json_query(root, path); - CHECK(result == expected); - - json path_result = json_query(root, path, result_type::path); - CHECK(path_result == expected_path); - } -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_tests.cpp deleted file mode 100644 index 4ec8cdd003..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpath/jsonpath_tests.cpp +++ /dev/null @@ -1,1845 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include // std::unordered_set - -using namespace jsoncons; - -TEST_CASE("jsonpath name tests") -{ - ojson example = ojson::parse(R"( - { - "\\\\" : "1", - "*" : "2", - "\\\"" : "3" - } - )"); - - SECTION("test 1") - { - ojson expected = ojson::array(); - expected.push_back(example.at("\\\\")); - - ojson result1 = jsonpath::json_query(example,"$.\\\\"); - ojson result2 = jsonpath::json_query(example, "$. \\\\ "); - ojson result3 = jsonpath::json_query(example,"$[\\\\]"); - ojson result4 = jsonpath::json_query(example, "$['\\\\\\\\']"); - ojson result5 = jsonpath::json_query(example, "$[\"\\\\\\\\\"]"); - - CHECK(result1 == expected); - CHECK(result2 == expected); - CHECK(result3 == expected); - CHECK(result4 == expected); - CHECK(result5 == expected); - - REQUIRE_THROWS_AS(jsonpath::json_query(example,"$.'\\'"),jsonpath::jsonpath_error); - } - - SECTION("test 2") - { - ojson expected = ojson::array(); - expected.push_back(example.at("\\\\")); - expected.push_back(example.at("*")); - expected.push_back(example.at("\\\"")); - - ojson result1 = jsonpath::json_query(example,"$.*"); - ojson result2 = jsonpath::json_query(example, "$. * "); - ojson result3 = jsonpath::json_query(example, "$[*]"); - ojson result4 = jsonpath::json_query(example, "$ [*] "); - ojson result5 = jsonpath::json_query(example, "$ [ * ] "); - - CHECK(result1 == expected); - CHECK(result2 == expected); - CHECK(result3 == expected); - CHECK(result4 == expected); - CHECK(result5 == expected); - } - - SECTION("test 3") - { - ojson expected = ojson::array(); - expected.push_back(example.at("*")); - - ojson result1 = jsonpath::json_query(example, "$['*']"); - ojson result2 = jsonpath::json_query(example, "$ [\"*\"] "); - - CHECK(result1 == expected); - CHECK(result2 == expected); - } -} - -TEST_CASE("jsonpath store tests") -{ - json store = json::parse(R"( - { - "store": { - "book": [ - { - "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { - "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - }, - { - "category": "fiction", - "author": "Herman Melville", - "title": "Moby Dick", - "isbn": "0-553-21311-3", - "price": 8.99 - }, - { - "category": "fiction", - "author": "J. R. R. Tolkien", - "title": "The Lord of the Rings", - "isbn": "0-395-19395-8", - "price": 22.99 - } - ], - "bicycle": { - "color": "red", - "price": 19.95 - } - } - } - )"); - - - SECTION("Absolute path") - { - json result1 = jsonpath::json_query(store,"$"); - json result2 = jsonpath::json_query(store,"$.*"); - json result3 = jsonpath::json_query(store,"$..*"); - - REQUIRE(result1.size() == 1); - CHECK(result1[0] == store); - REQUIRE(result2.size() == 1); - CHECK(result2[0] == store["store"]); - REQUIRE(result3.size() == 1); - CHECK(result3[0] == store["store"]); - - json result4 = jsonpath::json_query(store,"$ "); - json result5 = jsonpath::json_query(store,"$ \n.\n * "); - json result6 = jsonpath::json_query(store,"$ .. * "); - - REQUIRE(result4.size() == 1); - CHECK(result4[0] == store); - REQUIRE(result5.size() == 1); - CHECK(result5[0] == store["store"]); - REQUIRE(result6.size() == 1); - CHECK(result6[0] == store["store"]); - } - - SECTION("test_path") - { - json book_list = store["store"]["book"]; - json bicycle = store["store"]["bicycle"]; - - json result1 = jsonpath::json_query(store,"$.store.book"); - json result2 = jsonpath::json_query(store,"$. store . book "); - json result3 = jsonpath::json_query(store,"store.book"); - json result4 = jsonpath::json_query(store," store . book "); - json result5 = jsonpath::json_query(store,"$.'store'.'book'"); - json result6 = jsonpath::json_query(store,"\"store\".\"book\""); - - json expected = json::array(); - expected.push_back(book_list); - - CHECK(result1 == expected); - CHECK(result2 == expected); - CHECK(result3 == expected); - CHECK(result4 == expected); - CHECK(result5 == expected); - CHECK(result6 == expected); - } - - //std::cout << pretty_print(result) << std::endl; - SECTION("test_jsonpath_store_book2") - { - json book_list = store["store"]["book"]; - - json result = jsonpath::json_query(store,"$['store']['book']"); - - json expected = json::array(); - expected.push_back(book_list); - - CHECK(result == expected); - // std::c/out << pretty_print(result) << std::endl; - } - - SECTION("$.store[book[0].title,book[1].title,book[3].title]") - { - json result = jsonpath::json_query(store,"$.store[book[0].title,book[1].title,book[3].title]"); - json expected = json::parse(R"(["Sayings of the Century","Sword of Honour","The Lord of the Rings"])"); - CHECK(result == expected); - } - - SECTION("$.store[book[3].title,book[?(@.price > 10)].title]") - { - json result = jsonpath::json_query(store,"$.store[book[3].title,book[?(@.price > 10)].title]"); - json expected = json::parse(R"(["Sword of Honour","The Lord of the Rings"])"); - CHECK(result == expected); - } - - SECTION("test_jsonpath_bracket_with_double_quotes") - { - json book_list = store["store"]["book"]; - - json result = jsonpath::json_query(store,"$[\"store\"][\"book\"]"); - - json expected = json::array(); - expected.push_back(book_list); - - CHECK(result == expected); - // std::c/out << pretty_print(result) << std::endl; - } - SECTION("test_jsonpath_store_book_bicycle") - { - /* SECTION("one") - { - - json result = jsonpath::json_query(store,"$['store']['book']"); - json expected = json::array(); - expected.push_back(book_list); - CHECK(result == expected); - }*/ - - SECTION("two") - { - json book_list = store["store"]["book"]; - json bicycle = store["store"]["bicycle"]; - - json result = jsonpath::json_query(store,"$['store']['book','bicycle']"); - json expected1 = json::array(); - expected1.push_back(book_list); - expected1.push_back(bicycle); - json expected2 = json::array(); - expected2.push_back(bicycle); - expected2.push_back(book_list); - CHECK((result == expected1 || result == expected2)); - } - - //std::cout << pretty_print(result) << std::endl; - } - - SECTION("test_jsonpath_store_book_bicycle_unquoted") - { - json book_list = store["store"]["book"]; - json bicycle = store["store"]["bicycle"]; - - - json result = jsonpath::json_query(store,"$[store][book,bicycle]"); - - json expected1 = json::array(); - expected1.push_back(book_list); - expected1.push_back(bicycle); - json expected2 = json::array(); - expected2.push_back(bicycle); - expected2.push_back(book_list); - CHECK((result == expected1 || result == expected2)); - - //std::cout << pretty_print(result) << std::endl; - } - - SECTION("test_jsonpath_store_book_union") - { - json result = jsonpath::json_query(store,"$['store']..['author','title']"); - - json expected = json::parse(R"( - [ - "Nigel Rees", - "Sayings of the Century", - "Evelyn Waugh", - "Sword of Honour", - "Herman Melville", - "Moby Dick", - "J. R. R. Tolkien", - "The Lord of the Rings" - ] - )"); - } - - SECTION("test_jsonpath_store_book_star") - { - json book_list = store["store"]["book"]; - - json result = jsonpath::json_query(store,"$['store']['book'][*]"); - json expected = book_list; - - //std::cout << pretty_print(result) << std::endl; - CHECK(result == expected); - } - - SECTION("test_store_dotdot_price") - { - json book_list = store["store"]["book"]; - json bicycle = store["store"]["bicycle"]; - - json result = jsonpath::json_query(store,"$.store..price"); - - json expected = json::array(); - expected.push_back(bicycle["price"]); - for (size_t i = 0; i < book_list.size(); ++i) - { - expected.push_back(book_list[i]["price"]); - } - - //std::cout << pretty_print(result) << std::endl; - - CHECK(result == expected); - } - - SECTION("test_jsonpath_recursive_descent") - { - json result1 = jsonpath::json_query(store,"$..book[2]"); - //std::cout << pretty_print(result1) << std::endl; - CHECK(result1.size() == 1); - - CHECK(result1[0] == store["store"]["book"][2]); - - json result1a = jsonpath::json_query(store,"$..book.2"); - //std::cout << pretty_print(result1a) << std::endl; - CHECK(result1a.size() == 1); - CHECK(result1a[0] == store["store"]["book"][2]); - - json result2 = jsonpath::json_query(store,"$..book[-1:]"); - //std::cout << pretty_print(result2) << std::endl; - CHECK(result2.size() == 1); - CHECK(result2[0] == store["store"]["book"][3]); - - SECTION("$..book[0,1]") - { - json result = jsonpath::json_query(store,"$..book[0,1]"); - - json expected1 = json::array{store["store"]["book"][0],store["store"]["book"][1]}; - json expected2 = json::array{store["store"]["book"][1],store["store"]["book"][0]}; - //std::cout << pretty_print(result) << std::endl; - CHECK(result.size() == 2); - CHECK((result == expected1 || result == expected2)); - } - - json result4 = jsonpath::json_query(store,"$..book[:2]"); - //std::cout << pretty_print(result4) << std::endl; - CHECK(result4.size() == 2); - CHECK(result4[0] == store["store"]["book"][0]); - CHECK(result4[1] == store["store"]["book"][1]); - - json result5 = jsonpath::json_query(store,"$..book[1:2]"); - //std::cout << pretty_print(result5) << std::endl; - CHECK(result5.size() == 1); - CHECK(result5[0] == store["store"]["book"][1]); - - json result6 = jsonpath::json_query(store,"$..book[-2:]"); - //std::cout << pretty_print(result6) << std::endl; - CHECK(result6.size() == 2); - CHECK(result6[0] == store["store"]["book"][2]); - CHECK(result6[1] == store["store"]["book"][3]); - - json result7 = jsonpath::json_query(store,"$..book[2:]"); - //std::cout << pretty_print(result7) << std::endl; - CHECK(result7.size() == 2); - CHECK(result7[0] == store["store"]["book"][2]); - CHECK(result7[1] == store["store"]["book"][3]); - } - - SECTION("test_jsonpath_filter1") - { - json book_list = store["store"]["book"]; - - json result = jsonpath::json_query(store,"$..book[?(@.price<10)]"); - //std::cout << pretty_print(result) << std::endl; - json books = book_list; - json expected = json::array(); - for (size_t i = 0; i < books.size(); ++i) - { - double price = books[i]["price"].as(); - if (price < 10) - { - expected.push_back(books[i]); - } - } - CHECK(result == expected); - } - - SECTION("test_jsonpath_filter2") - { - json book_list = store["store"]["book"]; - - json result = jsonpath::json_query(store,"$..book[?(10 > @.price)]"); - - //std::cout << pretty_print(result) << std::endl; - json books = book_list; - json expected = json::array(); - for (size_t i = 0; i < books.size(); ++i) - { - double price = books[i]["price"].as(); - if (10 > price) - { - expected.push_back(books[i]); - } - } - CHECK(result == expected); - } - - SECTION("test_jsonpath_filter_category_eq_reference") - { - json book_list = store["store"]["book"]; - - json result = jsonpath::json_query(store,"$..book[?(@.category == 'reference')]"); - - //std::cout << pretty_print(result) << std::endl; - json books = book_list; - json expected = json::array(); - for (size_t i = 0; i < books.size(); ++i) - { - double price = books[i]["price"].as(); - (void)price; - - if (books[i]["category"].as() == "reference") - { - expected.push_back(books[i]); - } - } - CHECK(result == expected); - } - - SECTION("test_jsonpath_filter3") - { - json book_list = store["store"]["book"]; - - json result = jsonpath::json_query(store,"$..book[?((@.price > 8) && (@.price < 12))]"); - - json books = book_list; - - json expected = json::array(); - for (size_t i = 0; i < books.size(); ++i) - { - double price = books[i]["price"].as(); - if (price > 8 && price < 12) - { - expected.push_back(books[i]); - } - } - //std::cout << pretty_print(result) << std::endl; - //std::cout << pretty_print(expected) << std::endl; - - CHECK(result == expected); - } - - SECTION("test_jsonpath_book_isbn") - { - json book_list = store["store"]["book"]; - - json books = book_list; - for (size_t i = 0; i < books.size(); ++i) - { - bool has_isbn = books[i].contains("isbn"); - if (has_isbn) - { - json result = jsonpath::json_query(books[i],"$.isbn"); - json expected = json::array(); - expected.push_back(books[i]["isbn"]); - CHECK(result == expected); - //std::cout << pretty_print(result) << std::endl; - } - } - } - - SECTION("test_jsonpath_book_empty_isbn") - { - json book_list = store["store"]["book"]; - - const char* empty_isbn = "{ \"store\": {\"book\": [ { \"category\": \"reference\",\"author\": \"Nigel Rees\",\"title\": \"Sayings of the Century\",\"price\": 8.95},{ \"category\": \"fiction\",\"author\": \"Evelyn Waugh\",\"title\": \"Sword of Honour\",\"price\": 12.99},{ \"category\": \"fiction\",\"author\": \"Herman Melville\",\"title\": \"Moby Dick\",\"isbn\": \"0-553-21311-3\",\"price\": 8.99},{ \"category\": \"fiction\",\"author\": \"J. R. R. Tolkien\",\"title\": \"The Lord of the Rings\",\"isbn\": \"\",\"price\": 22.99}],\"bicycle\": {\"color\": \"red\",\"price\": 19.95}}}"; - json root2 = json::parse(empty_isbn); - - json books = book_list; - for (size_t i = 0; i < books.size(); ++i) - { - bool has_isbn = books[i].contains("isbn"); - if (has_isbn) - { - json result = jsonpath::json_query(books[i],"$.isbn"); - json expected = json::array(); - expected.push_back(books[i]["isbn"]); - CHECK(result == expected); - //std::cout << pretty_print(result) << std::endl; - } - } - } - - SECTION("test_jsonpath_filter4") - { - json book_list = store["store"]["book"]; - - json result = jsonpath::json_query(store,"$..book[?(@.isbn)]"); - - json books = book_list; - - json expected = json::array(); - for (size_t i = 0; i < books.size(); ++i) - { - if (books[i].contains("isbn")) - { - expected.push_back(books[i]); - } - } - //std::cout << pretty_print(result) << std::endl; - //std::cout << pretty_print(expected) << std::endl; - - CHECK(result == expected); - } - SECTION("test_jsonpath_array_length") - { - json result = jsonpath::json_query(store,"$..book.length"); - - //std::cout << pretty_print(result) << std::endl; - - CHECK(1 == result.size()); - CHECK(store["store"]["book"].size() == result[0].as()); - } - - SECTION("test_jsonpath_book_category") - { - json book_list = store["store"]["book"]; - - json book = book_list[0]; - - json result = jsonpath::json_query(book,"$.category"); - - CHECK(1 == result.size()); - CHECK("reference" == result[0].as()); - } - - SECTION("test_jsonpath_book_filter_false") - { - json result = jsonpath::json_query(store,"$..book[?(false)]"); - //std::cout << pretty_print(result) << std::endl; - - json expected = json::array(); - - CHECK(result == expected); - } - - SECTION("test_jsonpath_book_filter_false_and_false") - { - json result = jsonpath::json_query(store,"$..book[?(false && false)]"); - //std::cout << pretty_print(result) << std::endl; - - json expected = json::array(); - - CHECK(result == expected); - } - - SECTION("test_jsonpath_book_filter_false_or_false") - { - json result = jsonpath::json_query(store,"$..book[?(false || false)]"); - //std::cout << pretty_print(result) << std::endl; - - json expected = json::array(); - - CHECK(result == expected); - } - - SECTION("test_jsonpath_book_filter_false_or_true") - { - json book_list = store["store"]["book"]; - - json result = jsonpath::json_query(store,"$..book[?(false || true)]"); - //std::cout << pretty_print(result) << std::endl; - - CHECK(result == book_list); - } - - SECTION("test_jsonpath_store_book_authors") - { - json book_list = store["store"]["book"]; - - json result = jsonpath::json_query(store,"$.store.book[?(@.price < 10)].author"); - - json expected = json::array(); - for (size_t i = 0; i < book_list.size(); ++i) - { - json book = book_list[i]; - if (book["price"].as() < 10) - { - expected.push_back(book["author"]); - } - } - - //json expected = book_list; - - //std::cout << pretty_print(result) << std::endl; - - CHECK(result == expected); - } - - SECTION("test_jsonpath_store_book_tests") - { - json book_list = store["store"]["book"]; - - json result1 = jsonpath::json_query(store,"$.store.book[ ?(@.category == @.category) ]"); - CHECK(book_list == result1); - - json result2 = jsonpath::json_query(store,"$.store.book[ ?(@.category == @['category']) ]"); - CHECK(book_list == result2); - - json result3 = jsonpath::json_query(store,"$.store.book[ ?(@ == @) ]"); - CHECK(book_list == result3); - - json result4 = jsonpath::json_query(store,"$.store.book[ ?(@.category != @.category) ]"); - json expected4 = json::array(); - CHECK(result4 == expected4); - } - - SECTION("test_jsonpath_store_book_tests2") - { - json result1 = jsonpath::json_query(store,"$.store.book[ ?((@.author == 'Nigel Rees') || (@.author == 'Evelyn Waugh')) ].author"); - json expected1 = json::array(); - expected1.push_back("Nigel Rees"); - expected1.push_back("Evelyn Waugh"); - CHECK(result1 == expected1); - - json result1b = jsonpath::json_query(store,"$.store.book[ ?((@.author == 'Nigel Rees') || (@.author == 'Evelyn Waugh')) ].title"); - json expected1b = json::array(); - expected1b.push_back("Sayings of the Century"); - expected1b.push_back("Sword of Honour"); - //std::cout << result1b << std::endl; - CHECK(expected1b == result1b); - - json result2 = jsonpath::json_query(store,"$.store.book[ ?(((@.author == 'Nigel Rees') || (@.author == 'Evelyn Waugh')) && (@.price < 15)) ].author"); - json expected2 = json::array(); - expected2.push_back("Nigel Rees"); - expected2.push_back("Evelyn Waugh"); - CHECK(result2 == expected2); - - json result3 = jsonpath::json_query(store,"$.store.book[ ?(((@.author == 'Nigel Rees') || (@.author == 'Evelyn Waugh')) && (@.category == 'reference')) ].author"); - json expected3 = json::array(); - expected3.push_back("Nigel Rees"); - CHECK(result3 == expected3); - - json result4 = jsonpath::json_query(store,"$.store.book[ ?(((@.author == 'Nigel Rees') || (@.author == 'Evelyn Waugh')) && (@.category != 'fiction')) ].author"); - json expected4 = json::array(); - expected4.push_back("Nigel Rees"); - CHECK(result4 == expected4); - - json result5 = jsonpath::json_query(store,"$.store.book[?('a' == 'a')].author"); - json expected5 = json::array(); - expected5.push_back("Nigel Rees"); - expected5.push_back("Evelyn Waugh"); - expected5.push_back("Herman Melville"); - expected5.push_back("J. R. R. Tolkien"); - CHECK(result5 == expected5); - - json result6 = jsonpath::json_query(store,"$.store.book[?('a' == 'b')].author"); - json expected6 = json::array(); - CHECK(result6 == expected6); - } - - #if !(defined(__GNUC__) && (__GNUC__ == 4 && __GNUC_MINOR__ < 9)) - // GCC 4.8 has broken regex support: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53631 - SECTION("test_jsonpath_store_book_regex") - { - json result3 = jsonpath::json_query(store,"$.store.book[ ?(@.category =~ /fic.*?/)].author"); - json expected3 = json::array(); - expected3.push_back("Evelyn Waugh"); - expected3.push_back("Herman Melville"); - expected3.push_back("J. R. R. Tolkien"); - CHECK(result3 == expected3); - - json result4 = jsonpath::json_query(store,"$.store.book[ ?(@.author =~ /Evelyn.*?/)].author"); - json expected4 = json::array(); - expected4.push_back("Evelyn Waugh"); - CHECK(result4 == expected4); - - json result5 = jsonpath::json_query(store,"$.store.book[ ?(!(@.author =~ /Evelyn.*?/))].author"); - json expected5 = json::array(); - expected5.push_back("Nigel Rees"); - expected5.push_back("Herman Melville"); - expected5.push_back("J. R. R. Tolkien"); - CHECK(result5 == expected5); - } - #endif - - SECTION("test_jsonpath_everything") - { - json book_list = store["store"]["book"]; - json bicycle = store["store"]["bicycle"]; - - json result = jsonpath::json_query(store,"$.store.*"); - //std::cout << result << std::endl; - - json expected = json::array(); - expected.push_back(bicycle); - expected.push_back(book_list); - - CHECK(result == expected); - } - - SECTION("test_jsonpath_everything_in_store") - { - json result = jsonpath::json_query(store,"$..*"); - //std::cout << result << std::endl; - - json expected = json::array(); - expected.push_back(store["store"]); - - CHECK(result == expected); - } - - SECTION("test_array_slice_operator") - { - SECTION("Array slice") - { - - json result1 = jsonpath::json_query(store,"$..book[1:2].author"); - json expected1 = json::parse(R"( - [ - "Evelyn Waugh" - ] - )"); - CHECK(result1 == expected1); - - json result2 = jsonpath::json_query(store,"$..book[1:3:2].author"); - json expected2 = expected1; - CHECK(result2 == expected2); - - json result3 = jsonpath::json_query(store,"$..book[1:4:2].author"); - json expected3 = json::parse(R"( - [ - "Evelyn Waugh", - "J. R. R. Tolkien" - ] - )"); - CHECK(result3 == expected3); - } - - SECTION("Union") - { - std::unordered_set expected = {"Evelyn Waugh","J. R. R. Tolkien","Nigel Rees"}; - - json result1 = jsonpath::json_query(store,"$..book[1:4:2,0].author"); - CHECK(result1.size() == expected.size()); - for (const auto& item : result1.array_range()) - { - CHECK(expected.count(item.as()) == 1); - } - - json result2 = jsonpath::json_query(store,"$..book[1::2,0].author"); - CHECK(result2.size() == expected.size()); - for (const auto& item : result2.array_range()) - { - CHECK(expected.count(item.as()) == 1); - } - } - } - - SECTION("test_max_pre") - { - std::string path = "$.store.book[*].price"; - json result = jsonpath::json_query(store,path); - - //std::cout << result << std::endl; - } - - SECTION("test_max") - { - std::string path = "$.store.book[?(@.price < max($.store.book[*].price))].title"; - - json expected = json::parse(R"( - ["Sayings of the Century","Sword of Honour","Moby Dick"] - )"); - - json result = jsonpath::json_query(store,path); - CHECK(result == expected); - - //std::cout << result << std::endl; - } - - SECTION("test_min") - { - std::string path = "$.store.book[?(@.price > min($.store.book[*].price))].title"; - - json expected = json::parse(R"( - ["Sword of Honour","Moby Dick","The Lord of the Rings"] - )"); - - json result = jsonpath::json_query(store,path); - CHECK(result == expected); - } - - SECTION("test_sum_filter_func") - { - std::string path = "$.store.book[?(@.price > sum($.store.book[*].price) / 4)].title"; // unexpected exception if '$.store.book.length' instead '4' - - json expected = json::parse(R"( - ["The Lord of the Rings"] - )"); - - json result = jsonpath::json_query(store,path); - CHECK(result == expected); - } - - SECTION("test_prod_func") - { - std::string path = "$.store.bicycle[?(479373 < prod($..price) && prod($..price) < 479374)].color"; - - json expected = json::parse(R"( - ["red"] - )"); - - json result = jsonpath::json_query(store,path); - CHECK(result == expected); - } - - SECTION("test_ws1") - { - json result = jsonpath::json_query(store,"$..book[ ?(( @.price > 8 ) && (@.price < 12)) ].author"); - - json expected = json::parse(R"( - [ - "Nigel Rees", - "Herman Melville" - ] - )"); - - CHECK(result == expected); - } - -} - -// store tests - -TEST_CASE("test_jsonpath_last_of_two_arrays") -{ - json val = json::parse(R"( -{ "store": { - "book": [ - { "author": "Nigel Rees" - }, - { "author": "Evelyn Waugh" - }, - { "author": "Herman Melville" - } - ] - }, - "Roman": { - "book": [ - { "author": "Tolstoy L" - }, - { "author": "Tretyakovskiy R" - }, - { "author": "Kulik M" - } - ] - } -} - )"); - - json expected = json::parse(R"( -[ - { "author": "Kulik M"}, - { "author": "Herman Melville"} -] - )"); - - json result = jsonpath::json_query(val, "$..book[(@.length - 1)]"); - - CHECK(result == expected); -} - -TEST_CASE("test_jsonpath_next_to_last_of_two_arrays") -{ - json val = json::parse(R"( -{ "store": { - "book": [ - { "author": "Nigel Rees" - }, - { "author": "Evelyn Waugh" - }, - { "author": "Herman Melville" - } - ] - }, - "Roman": { - "book": [ - { "author": "Tolstoy L" - }, - { "author": "Tretyakovskiy R" - }, - { "author": "Kulik M" - } - ] - } -} - )"); - - json expected = json::parse(R"( -[ - { "author": "Tretyakovskiy R"}, - { "author": "Evelyn Waugh"} -] - )"); - - json result = jsonpath::json_query(val, "$..book[(@.length - 2)]"); - - CHECK(result == expected); - - json expected2 = json::parse(R"( -[ - "Tolstoy L", - "Nigel Rees" -] - )"); - std::string path2 = "$..[0].author"; - json result2 = jsonpath::json_query(val, path2); - CHECK(result2 == expected2); - -} - -TEST_CASE("test_jsonpath_aggregation") -{ - json val = json::parse(R"( -{ - "firstName": "John", - "lastName" : "doe", - "age" : 26, - "address" : { - "streetAddress": "naist street", - "city" : "Nara", - "postalCode" : "630-0192" - }, - "phoneNumbers": [ - { - "type" : "iPhone", - "number": "0123-4567-8888" - }, - { - "type" : "home", - "number": "0123-4567-8910" - } - ] -} - )"); - - SECTION("$['firstName','lastName']") - { - json expected1 = json::parse(R"( - ["John","doe"] - )"); - json expected2 = json::parse(R"( - ["doe","John"] - )"); - - json result2 = jsonpath::json_query(val, "$['firstName','lastName']"); - CHECK((result2 == expected1 || result2 == expected2)); - - json result3 = jsonpath::json_query(val, "$[\"firstName\",\"lastName\"]"); - CHECK((result3 == expected1 || result3 == expected2)); - } - - SECTION("$..['firstName','city']") - { - json expected1 = json::parse(R"(["John","Nara"])"); - json expected2 = json::parse(R"(["Nara","John"])"); - std::string path = "$..['firstName','city']"; - - json result = jsonpath::json_query(val, path); - CHECK((result == expected1 || result == expected2)); - } -} - -TEST_CASE("test_jsonpath_aggregation2") -{ - json val = json::parse(R"( -{ "store": { - "book": [ - { "author": "Nigel Rees" - }, - { "author": "Evelyn Waugh" - }, - { "author": "Herman Melville" - } - ] - } -} - )"); - - json result = jsonpath::json_query(val, "$..book[(@.length - 1),(@.length - 2)]"); - - json expected1 = json::parse(R"([{"author": "Herman Melville"},{"author": "Evelyn Waugh"}])"); - json expected2 = json::parse(R"([{"author": "Evelyn Waugh"},{"author": "Herman Melville"}])"); - CHECK((result == expected1 || result == expected2)); -} - -TEST_CASE("test_jsonpath_aggregation3") -{ - json val = json::parse(R"( -{ - "firstName": "John", - "lastName" : "doe", - "age" : 26, - "address" : { - "streetAddress": "naist street", - "city" : "Nara", - "postalCode" : "630-0192" - }, - "phoneNumbers": [ - { - "type" : "iPhone", - "number": "0123-4567-8888" - }, - { - "type" : "home", - "number": "0123-4567-8910" - } - ] -} - )"); - - std::unordered_set expected = {"iPhone","0123-4567-8888","home","0123-4567-8910"}; - - json result = jsonpath::json_query(val, "$..['type','number']"); - CHECK(result.size() == expected.size()); - for (const auto& item : result.array_range()) - { - CHECK(expected.count(item.as()) == 1); - } -} - -TEST_CASE("test_jsonpath_aggregation4") -{ - json val = json::parse(R"( -{ - "firstName": "John", - "lastName" : "doe", - "age" : 26, - "address" : { - "streetAddress": "naist street", - "city" : "Nara", - "postalCode" : "630-0192" - }, - "phoneNumbers": [ - { - "type" : "iPhone", - "number": "0123-4567-8888" - }, - { - "type" : "home", - "number": "0123-4567-8910" - } - ] -} - )"); - - json test1 = jsonpath::json_query(val, "$.phoneNumbers"); - //std::cout << test1 << std::endl; - json test2 = jsonpath::json_query(val, "$[phoneNumbers]"); - //std::cout << test2 << std::endl; - json test3 = jsonpath::json_query(val, "$..['type']"); - //std::cout << test3 << std::endl; - - std::unordered_set expected = {"iPhone","0123-4567-8888","home","0123-4567-8910"}; - - json result2 = jsonpath::json_query(val, "$.phoneNumbers..['type','number']"); - CHECK(result2.size() == expected.size()); - for (const auto& item : result2.array_range()) - { - CHECK(expected.count(item.as()) == 1); - } -} - -TEST_CASE("test_jsonpath_string_indexation") -{ - json val; - val["about"] = "I\xe2\x82\xacJ"; - - json expected1 = json::array(1,"I"); - json result1 = jsonpath::json_query(val, "$..about[0]"); - CHECK(result1 == expected1); - - json expected2 = json::array(1,"\xe2\x82\xac"); - json result2 = jsonpath::json_query(val, "$..about[1]"); - CHECK(result2 == expected2); - - json expected3 = json::array(1,"J"); - json result3 = jsonpath::json_query(val, "$..about[2]"); - CHECK(result3 == expected3); - - json expected4 = json::array(1,3); - json result4 = jsonpath::json_query(val, "$..about.length"); - CHECK(result4 == expected4); -} - -TEST_CASE("test_union_array_elements") -{ - json val = json::parse(R"( -{ "store": { - "book": [ - { "author": "Nigel Rees" - }, - { "author": "Evelyn Waugh" - }, - { "author": "Herman Melville" - } - ] - }, - "Roman": { - "book": [ - { "author": "Tolstoy L" - }, - { "author": "Tretyakovskiy R" - }, - { "author": "Kulik M" - } - ] - } -} - )"); - - json expected1 = json::parse(R"( -[ - { "author": "Kulik M"}, - { "author": "Herman Melville"} -] - )"); - json result1 = jsonpath::json_query(val, "$..book[-1]"); - CHECK(result1 == expected1); - - std::unordered_set expected = {"Kulik M","Tolstoy L","Herman Melville","Nigel Rees"}; - - json expected2 = json::parse(R"( -[ - { - "author": "Kulik M" - }, - { - "author": "Tolstoy L" - }, - { - "author": "Herman Melville" - }, - { - "author": "Nigel Rees" - } -] - )"); - json result2 = jsonpath::json_query(val, "$..book[-1,-3]"); - - CHECK(result2.size() == expected.size()); - for (const auto& item : result2.array_range()) - { - CHECK(expected.count(item["author"].as()) == 1); - } - - //CHECK(result2 == expected2); - //json expected3 = expected2; - json result3 = jsonpath::json_query(val, "$..book[-1,(@.length - 3)]"); - CHECK(result3.size() == expected.size()); - for (const auto& item : result3.array_range()) - { - CHECK(expected.count(item["author"].as()) == 1); - } - //CHECK(result3 == expected3); - //json expected4 = expected2; - json result4 = jsonpath::json_query(val, "$..book[(@.length - 1),-3]"); - CHECK(result4.size() == expected.size()); - for (const auto& item : result4.array_range()) - { - CHECK(expected.count(item["author"].as()) == 1); - } - //CHECK(result4 == expected4); -} - -TEST_CASE("test_replace") -{ - json j; - try - { - j = json::parse(R"( -{"store": -{"book": [ -{"category": "reference", -"author": "Margaret Weis", -"title": "Dragonlance Series", -"price": 31.96}, {"category": "reference", -"author": "Brent Weeks", -"title": "Night Angel Trilogy", -"price": 14.70 -}]}} -)"); - } - catch (const ser_error& e) - { - std::cout << e.what() << std::endl; - } - - //std::cout << "!!!test_replace" << std::endl; - //std::cout << ("1\n") << pretty_print(j) << std::endl; - - CHECK(31.96 == Approx(j["store"]["book"][0]["price"].as()).epsilon(0.001)); - - jsonpath::json_replace(j,"$..book[?(@.price==31.96)].price", 30.9); - - CHECK(30.9 == Approx(j["store"]["book"][0]["price"].as()).epsilon(0.001)); - - //std::cout << ("2\n") << pretty_print(j) << std::endl; -} - - -TEST_CASE("test_select_two") -{ - json j = json::parse(R"( -[ - { - "a": 5, - "b": 500, - "c": 5000 - }, - { - "a": 6, - "b": 600, - "c": 6000 - }, - { - "a": 7, - "b": 700, - "c": 7000 - } -] -)"); - - json result = jsonpath::json_query(j,"$..*[?((@.a == 5 && @.b == 500) || (@.a == 6 && @.b == 600))]"); - - json expected = json::parse(R"( -[ - { - "a": 5, - "b": 500, - "c": 5000 - }, - { - "a": 6, - "b": 600, - "c": 6000 - } -] -)"); - - CHECK(result == expected); -} - -TEST_CASE("test_select_length_4") -{ - json j = json::parse(R"( -[ - { - "result": [ - 1, - 2, - 3, - 4 - ] - } -] - -)"); - - json result = jsonpath::json_query(j,"$..[?(@.result.length == 4)]"); - - json expected = json::parse(R"( -[{"result":[1,2,3,4]}] - )"); - - CHECK(result == expected); -} - -TEST_CASE("complex json") -{ - const json complex_json = json::parse(R"( - [ - { - "root": { - "id" : 10, - "second": [ - { - "names": [ - 2 - ], - "complex": [ - { - "names": [ - 1 - ], - "panels": [ - { - "result": [ - 1 - ] - }, - { - "result": [ - 1, - 2, - 3, - 4 - ] - }, - { - "result": [ - 1 - ] - } - ] - } - ] - } - ] - } - }, - { - "root": { - "id" : 20, - "second": [ - { - "names": [ - 2 - ], - "complex": [ - { - "names": [ - 1 - ], - "panels": [ - { - "result": [ - 1 - ] - }, - { - "result": [ - 1, - 2, - 3, - 4 - ] - }, - { - "result": [ - 1 - ] - } - ] - } - ] - } - ] - } - } - ] - )"); - - SECTION("test_select_length_4_2") - { - json result = jsonpath::json_query(complex_json,"$..[?(@.result.length == 4)]"); - json expected = json::parse(R"( - [{"result":[1,2,3,4]},{"result":[1,2,3,4]}] - )"); - - CHECK(result == expected); - } - SECTION("test_select_length_4_2_plus") - { - - json result = jsonpath::json_query(complex_json,"$..[?(@.id == 10)]..[?(@.result.length == 4)]"); - //std::cout << result << std::endl; - - json expected = json::parse(R"( - [{"result":[1,2,3,4]}] - )"); - - CHECK(result == expected); - } - - SECTION("test_select_length_4_2_plus_plus") - { - - json result = jsonpath::json_query(complex_json,"$..[?(@.result.length == 4)][?(@.result[0] == 3 || @.result[1] == 3 || @.result[2] == 3 || @.result[3] == 3)]"); - //std::cout << result << std::endl; - - json expected = json::parse(R"( - [{"result":[1,2,3,4]},{"result":[1,2,3,4]}] - )"); - - CHECK(result == expected); - } -} - - -TEST_CASE("test_nested") -{ - json j = json::parse(R"( -{ - "id" : 10, - "b": {"id" : 10} -} -)"); - - json result = jsonpath::json_query(j,"$..[?(@.id == 10)]"); - - json expected = json::parse(R"( -[ - { - "id" : 10, - "b" : { - "id" : 10 - } - }, - { - "id" : 10 - } -] -)"); - - CHECK(result == expected); -} - -TEST_CASE("test_array_nested") -{ - json j = json::parse(R"( -{ - "a" : [ - { - "id" : 10, - "b": {"id" : 10} - } - ] -} -)"); - - json result = jsonpath::json_query(j,"$..[?(@.id == 10)]"); - - json expected = json::parse(R"( -[ - { - "id" : 10, - "b" : { - "id" : 10 - } - }, - { - "id" : 10 - } -] -)"); - - CHECK(result == expected); -} - -TEST_CASE("test_array_array_nested") -{ - json j = json::parse(R"( -{ - "a" : [[ - { - "id" : 10, - "b": {"id" : 10} - } - ]] -} -)"); - - json result = jsonpath::json_query(j,"$..[?(@.id == 10)]"); - - json expected = json::parse(R"( -[ - { - "id" : 10, - "b" : { - "id" : 10 - } - }, - { - "id" : 10 - } -] -)"); - - CHECK(result == expected); -} - -TEST_CASE("jsonpath test 1") -{ - json j = json::parse(R"( -[ - { - "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { - "category": "fiction", - "author": "Evelyn Waugh", - "title": "Sword of Honour", - "price": 12.99 - }, - { - "category": "fiction", - "author": "Herman Melville", - "title": "Moby Dick", - "isbn": "0-553-21311-3", - "price": 8.99 - }, - { - "category": "fiction", - "author": "J. R. R. Tolkien", - "title": "The Lord of the Rings", - "isbn": "0-395-19395-8", - "price": 22.99 - } -] -)"); - - SECTION("$.0.category") - { - json result = jsonpath::json_query(j,"$.0.category"); - REQUIRE(result.size() == 1); - CHECK(result[0].as() == std::string("reference")); - } - SECTION("$.0.'category'") - { - json result = jsonpath::json_query(j,"$.0.'category'"); - REQUIRE(result.size() == 1); - CHECK(result[0].as() == std::string("reference")); - } - SECTION("$[0].category") - { - json result = jsonpath::json_query(j,"$[0].category"); - REQUIRE(result.size() == 1); - CHECK(result[0].as() == std::string("reference")); - } - SECTION("0.category") - { - json result = jsonpath::json_query(j,"0.category"); - REQUIRE(result.size() == 1); - CHECK(result[0].as() == std::string("reference")); - } - - SECTION("0['category']") - { - json result = jsonpath::json_query(j,"0['category']"); - REQUIRE(result.size() == 1); - CHECK(result[0].as() == std::string("reference")); - } - SECTION("0[\"category\"]") - { - json result = jsonpath::json_query(j,"0[\"category\"]"); - REQUIRE(result.size() == 1); - CHECK(result[0].as() == std::string("reference")); - } - SECTION("count($.*)") - { - json result = jsonpath::json_query(j,"count($.*)"); - REQUIRE(result.size() == 1); - CHECK(result[0].as() == 4); - } - - SECTION("$.*") - { - json result = jsonpath::json_query(j,"$.*"); - REQUIRE(result.size() == 4); - CHECK(result == j); - } - - SECTION("$[-3].category") - { - json result = jsonpath::json_query(j,"$[-3].category"); - REQUIRE(result.size() == 1); - CHECK(result[0].as() == std::string("fiction")); - } - - SECTION("$[-2:].category") - { - json expected = json::parse(R"([ "Moby Dick", "The Lord of the Rings"])"); - json result = jsonpath::json_query(j,"$[-2:].title"); - REQUIRE(result.size() == 2); - CHECK(result == expected); - } - - SECTION("$[-1,-3,-4].category") - { - std::unordered_set expected{"The Lord of the Rings", "Sword of Honour", "Sayings of the Century"}; - json result = jsonpath::json_query(j,"$[-1,-3,-4].title"); - - CHECK(result.size() == expected.size()); - for (const auto& item : result.array_range()) - { - CHECK(expected.count(item.as()) == 1); - } - } - - SECTION("count($.*)") - { - json result = jsonpath::json_query(j,"count($.*)"); - REQUIRE(result.size() == 1); - CHECK(result[0].as() == 4); - } - - SECTION("count($[*])") - { - json result = jsonpath::json_query(j,"count($[*])"); - REQUIRE(result.size() == 1); - CHECK(result[0].as() == 4); - } - SECTION("keys($[1])") - { - json expected = json::array{"author","category","price","title"}; - - json result = jsonpath::json_query(j,"keys($[1])[*]"); - CHECK(result == expected); - } -#if !(defined(__GNUC__) && (__GNUC__ == 4 && __GNUC_MINOR__ < 9)) -// GCC 4.8 has broken regex support: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53631 - SECTION("$[?(tokenize(@.author,'\\\\s+')[1] == 'Waugh')].title") - { - json expected = json::array{"Sword of Honour"}; - - json result = jsonpath::json_query(j,"$[?(tokenize(@.author,'\\\\s+')[1] == 'Waugh')].title"); - - CHECK(result == expected); - } - - SECTION("tokenize($[0].author,'\\\\s+')") - { - json expected = json::parse("[[\"Nigel\",\"Rees\"]]"); - - json result = jsonpath::json_query(j,"tokenize($[0].author,'\\\\s+')"); - - CHECK(result == expected); - } -#endif -} -TEST_CASE("jsonpath array union test") -{ - json root = json::parse(R"( -[[1,2,3,4,1,2,3,4],[0,1,2,3,4,5,6,7,8,9],[0,1,2,3,4,5,6,7,8,9]] -)"); - SECTION("Test 1") - { - json expected = json::parse(R"( -[[0,1,2,3,4,5,6,7,8,9],[1,2,3,4,1,2,3,4]] -)"); - json result = jsonpath::json_query(root,"$[0,1,2]"); - CHECK(result == expected); - } - - SECTION("Test 2") - { - json expected = json::parse(R"( -[1,2,3,4] -)"); - json result = jsonpath::json_query(root, "$[0][0:4,2:8]"); - CHECK(result == expected); - } - - SECTION("Test 3") - { - json expected = json::parse(R"( -[1,4] -)"); - json result = jsonpath::json_query(root, "$[0][0,0,0,3]"); - CHECK(result == expected); - } - - SECTION("Test 4") - { - json expected = json::parse(R"( -[1,2] -)"); - //json result1 = jsonpath::json_query(root,"$[0.0,1.1,2.2]"); - json result2 = jsonpath::json_query(root,"$[0[0],1[1],2[2]]"); - - //CHECK(result1 == expected); - CHECK(result2 == expected); - } -} - -TEST_CASE("jsonpath object union test 1") -{ - const json root = json::parse(R"( -[{ - "firstName": "John", - "lastName" : "Smith", - "age" : 26, - "address" : { - "streetAddress": "naist street", - "city" : "Nara", - "postalCode" : "630-0192" - }, - "phoneNumbers": [ - { - "type" : "iPhone", - "number": "0123-4567-8888" - }, - { - "type" : "home", - "number": "0123-4567-8910" - } - ] -}, -{ - "firstName": "John", - "lastName" : "Doe", - "age" : 30, - "address" : { - "streetAddress": "naist street", - "city" : "Nara", - "postalCode" : "630-0192" - }, - "phoneNumbers": [ - { - "type" : "iPhone", - "number": "0123-4567-8888" - }, - { - "type" : "home", - "number": "0123-4567-8910" - } - ] -}] - -)"); - - SECTION("$..[firstName,address.city]") - { - json expected = json::parse(R"(["John","Nara"])"); - std::string path = "$..[firstName,address.city]"; - json result = jsonpath::json_query(root,path); - CHECK(result == expected); - } - SECTION("$..[firstName,*.city]") - { - json expected = json::parse(R"(["John","Nara"])"); - std::string path = "$..[firstName,*.city]"; - json result = jsonpath::json_query(root,path); - CHECK(result == expected); - } - SECTION("$..[firstName,address[city]]") - { - json expected = json::parse(R"(["John","Nara"])"); - std::string path = "$..[firstName,address[city]]"; - json result = jsonpath::json_query(root,path); - CHECK(result == expected); - } - SECTION("$..[firstName,address['city']]") - { - json expected = json::parse(R"(["John","Nara"])"); - std::string path = "$..[firstName,address['city']]"; - json result = jsonpath::json_query(root,path); - CHECK(result == expected); - } - SECTION("$..[firstName,address.'city']") - { - json expected = json::parse(R"(["John","Nara"])"); - std::string path = "$..[firstName,address.'city']"; - json result = jsonpath::json_query(root,path); - CHECK(result == expected); - } - SECTION(R"($..[firstName,address."city"])") - { - json expected = json::parse(R"(["John","Nara"])"); - std::string path = R"($..[firstName,address."city"])"; - json result = jsonpath::json_query(root,path); - CHECK(result == expected); - } - SECTION(R"($..[firstName,address["city"]])") - { - json expected = json::parse(R"(["John","Nara"])"); - std::string path = R"($..[firstName,address["city"]])"; - json result = jsonpath::json_query(root,path); - CHECK(result == expected); - } - SECTION(R"($..['firstName','address'["city"]])") - { - json expected = json::parse(R"(["John","Nara"])"); - std::string path = R"($..['firstName','address'["city"]])"; - json result = jsonpath::json_query(root,path); - CHECK(result == expected); - } - SECTION(R"($..["firstName","address"["city"]])") - { - json expected = json::parse(R"(["John","Nara"])"); - std::string path = R"($..["firstName","address"["city"]])"; - json result = jsonpath::json_query(root,path); - CHECK(result == expected); - } - SECTION(R"($..["firstName","address"["city"]])") - { - json expected = json::parse(R"(["John","Nara"])"); - std::string path = R"($..[?(@.firstName == 'John')])"; - json result = jsonpath::json_query(root,path); - //std::cout << pretty_print(result) << "\n\n"; - //CHECK(result == expected); - } - SECTION(R"($..[?(@.firstName == 'John')][?(@.lastName == 'Doe')])") - { - json expected = json::parse(R"(["John","Nara"])"); - std::string path = R"($..[?(@.firstName == 'John')][?(@.lastName == 'Doe')])"; - json result = jsonpath::json_query(root,path); - REQUIRE(result.size() == 1); - CHECK(result[0] == root[1]); - } -} - -TEST_CASE("jsonpath object union test") -{ - const json root = json::parse(R"( -[{ - "firstName": "John", - "lastName" : "Smith" -}, -{ - "firstName": "John", - "lastName" : "Doe" -} -] - -)"); - - SECTION(R"($..["firstName","address"["city"]])") - { - json expected = json::parse(R"([{"firstName":"John","lastName":"Smith"},{"firstName":"John","lastName":"Doe"}])"); - std::string path = R"($..[?(@.firstName == 'John')])"; - json result = jsonpath::json_query(root,path); - //std::cout << result << "\n\n"; - CHECK(result == expected); - } - - SECTION(R"($..["firstName","address"["city"]])") - { - json expected = json::parse(R"([{"firstName":"John","lastName":"Doe"}])"); - std::string path = R"($[?(@.firstName == 'John')][?(@.lastName == 'Doe')])"; - json result = jsonpath::json_query(root,path); - //std::cout << result << "\n\n"; - CHECK(result == expected); - } -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpointer/jsonpointer_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpointer/jsonpointer_tests.cpp deleted file mode 100644 index dfb6575a29..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/jsonpointer/jsonpointer_tests.cpp +++ /dev/null @@ -1,316 +0,0 @@ -// Copyright 2017 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; -using namespace jsoncons::literals; - -void check_get_with_const_ref(const json& example, const std::string& pointer, const json& expected) -{ - - std::error_code ec; - const json& result = jsonpointer::get(example,pointer,ec); - CHECK_FALSE(ec); - CHECK(result == expected); -} - -void check_contains(const json& example, const std::string& pointer, bool expected) -{ - bool result = jsonpointer::contains(example,pointer); - CHECK(result == expected); -} - -void check_insert_or_assign(json& example, const std::string& path, const json& value, const json& expected) -{ - std::error_code ec; - jsonpointer::insert_or_assign(example, path, value, ec); - CHECK_FALSE(ec); - CHECK(example == expected); -} - -void check_replace(json& example, const std::string& path, const json& value, const json& expected) -{ - std::error_code ec; - jsonpointer::replace(example, path, value, ec); - CHECK_FALSE(ec); - CHECK(example == expected); -} - -void check_remove(json& example, const std::string& path, const json& expected) -{ - std::error_code ec; - jsonpointer::remove(example, path, ec); - CHECK_FALSE(ec); - CHECK(example == expected); -} - -TEST_CASE("get_with_const_ref_test") -{ -// Example from RFC 6901 -const json example = json::parse(R"( - { - "foo": ["bar", "baz"], - "": 0, - "a/b": 1, - "c%d": 2, - "e^f": 3, - "g|h": 4, - "i\\j": 5, - "k\"l": 6, - " ": 7, - "m~n": 8 - } -)"); - - check_contains(example,"",true); - check_contains(example,"/foo",true); - check_contains(example,"/foo/0",true); - check_contains(example,"/",true); - check_contains(example,"/a~1b",true); - check_contains(example,"/c%d",true); - check_contains(example,"/e^f",true); - check_contains(example,"/g|h",true); - check_contains(example,"/i\\j",true); - check_contains(example,"/k\"l",true); - check_contains(example,"/ ",true); - check_contains(example,"/m~0n",true); - - check_get_with_const_ref(example,"",example); - check_get_with_const_ref(example,"/foo",json::parse("[\"bar\", \"baz\"]")); - check_get_with_const_ref(example,"/foo/0",json("bar")); - check_get_with_const_ref(example,"/",json(0)); - check_get_with_const_ref(example,"/a~1b",json(1)); - check_get_with_const_ref(example,"/c%d",json(2)); - check_get_with_const_ref(example,"/e^f",json(3)); - check_get_with_const_ref(example,"/g|h",json(4)); - check_get_with_const_ref(example,"/i\\j",json(5)); - check_get_with_const_ref(example,"/k\"l",json(6)); - check_get_with_const_ref(example,"/ ",json(7)); - check_get_with_const_ref(example,"/m~0n",json(8)); -} - -TEST_CASE("get_with_ref_test") -{ -// Example from RFC 6901 -json example = json::parse(R"( - { - "foo": ["bar", "baz"] - } -)"); - - std::error_code ec; - json& result = jsonpointer::get(example,"/foo/0",ec); - CHECK_FALSE(ec); - - result = "bat"; - - //std::cout << example << std::endl; -} - -TEST_CASE("get_with_nonexistent_target") -{ - json example = R"( - { "foo": "bar" } - )"_json; - - check_contains(example,"/baz",false); -} - -// insert_or_assign - -TEST_CASE("test_add_object_member") -{ - json example = json::parse(R"( - { "foo": "bar"} - )"); - - const json expected = json::parse(R"( - { "foo": "bar", "baz" : "qux"} - )"); - - check_insert_or_assign(example,"/baz", json("qux"), expected); -} - -TEST_CASE("test_add_array_element") -{ - json example = json::parse(R"( - { "foo": [ "bar", "baz" ] } - )"); - - const json expected = json::parse(R"( - { "foo": [ "bar", "qux", "baz" ] } - )"); - - check_insert_or_assign(example,"/foo/1", json("qux"), expected); -} - -TEST_CASE("test_add_array_value") -{ - json example = json::parse(R"( - { "foo": ["bar"] } - )"); - - const json expected = json::parse(R"( - { "foo": ["bar", ["abc", "def"]] } - )"); - - check_insert_or_assign(example,"/foo/-", json::array({"abc", "def"}), expected); -} - -// remove - -TEST_CASE("test_remove_object_member") -{ - json example = json::parse(R"( - { "foo": "bar", "baz" : "qux"} - )"); - - const json expected = json::parse(R"( - { "foo": "bar"} - )"); - - check_remove(example,"/baz", expected); -} - -TEST_CASE("test_remove_array_element") -{ - json example = json::parse(R"( - { "foo": [ "bar", "qux", "baz" ] } - )"); - - const json expected = json::parse(R"( - { "foo": [ "bar", "baz" ] } - )"); - - check_remove(example,"/foo/1", expected); -} - -// replace - -TEST_CASE("test_replace_object_value") -{ - json example = json::parse(R"( - { - "baz": "qux", - "foo": "bar" - } - )"); - - const json expected = json::parse(R"( - { - "baz": "boo", - "foo": "bar" - } - )"); - - check_replace(example,"/baz", json("boo"), expected); -} -TEST_CASE("test_replace_array_value") -{ - json example = json::parse(R"( - { "foo": [ "bar", "baz" ] } - )"); - - const json expected = json::parse(R"( - { "foo": [ "bar", "qux" ] } - )"); - - check_replace(example,"/foo/1", json("qux"), expected); -} - -TEST_CASE("jsonpointer path tests") -{ - SECTION("/a~1b") - { - jsonpointer::address p("/a~1b"); - - auto it = p.begin(); - auto end = p.end(); - - CHECK((*it++ == "a/b")); - CHECK(it == end); - } - SECTION("/a~1b") - { - jsonpointer::address p("/m~0n"); - - auto it = p.begin(); - auto end = p.end(); - - CHECK((*it++ == "m~n")); - CHECK(it == end); - } - SECTION("/0/1") - { - jsonpointer::address p("/0/1"); - - auto it = p.begin(); - auto end = p.end(); - - CHECK((*it++ == "0")); - CHECK((*it++ == "1")); - CHECK(it == end); - } -} - -TEST_CASE("jsonpointer concatenation") -{ - // Example from RFC 6901 - json example = json::parse(R"( - { - "a/b": ["bar", "baz"], - "m~n": ["foo", "qux"] - } - )"); - - SECTION("path append a/b") - { - jsonpointer::address p; - p /= "a/b"; - p /= "0"; - - auto it = p.begin(); - auto end = p.end(); - - CHECK((*it++ == "a/b")); - CHECK((*it++ == "0")); - CHECK(it == end); - - std::error_code ec; - json j = jsonpointer::get(example, p, ec); - std::cout << j << "\n"; - } - - SECTION("concatenate two paths") - { - jsonpointer::address p1; - p1 /= "m~n"; - jsonpointer::address p2; - p2 /= "1"; - jsonpointer::address p = p1 + p2; - - auto it = p.begin(); - auto end = p.end(); - - CHECK((*it++ == "m~n")); - CHECK((*it++ == "1")); - CHECK(it == end); - - json j = jsonpointer::get(example, p); - std::cout << j << "\n"; - } -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/msgpack/decode_msgpack_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/msgpack/decode_msgpack_tests.cpp deleted file mode 100644 index 843050aa28..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/msgpack/decode_msgpack_tests.cpp +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2016 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; -using namespace jsoncons::msgpack; - -void check_decode_msgpack(const std::vector& v, const json& expected) -{ - json result = decode_msgpack(v); - REQUIRE(result == expected); -} - -TEST_CASE("decode_number_msgpack_test") -{ - // positive fixint 0x00 - 0x7f - check_decode_msgpack({0x00},json(0U)); - check_decode_msgpack({0x01},json(1U)); - check_decode_msgpack({0x0a},json(10U)); - check_decode_msgpack({0x17},json(23U)); - check_decode_msgpack({0x18},json(24U)); - check_decode_msgpack({0x7f},json(127U)); - - check_decode_msgpack({0xcc,0xff},json(255U)); - check_decode_msgpack({0xcd,0x01,0x00},json(256U)); - check_decode_msgpack({0xcd,0xff,0xff},json(65535U)); - check_decode_msgpack({0xce,0,1,0x00,0x00},json(65536U)); - check_decode_msgpack({0xce,0xff,0xff,0xff,0xff},json(4294967295U)); - check_decode_msgpack({0xcf,0,0,0,1,0,0,0,0},json(4294967296U)); - check_decode_msgpack({0xcf,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff},json((std::numeric_limits::max)())); - - check_decode_msgpack({0x01},json(1)); - check_decode_msgpack({0x0a},json(10)); - check_decode_msgpack({0x17},json(23)); - check_decode_msgpack({0x18},json(24)); - check_decode_msgpack({0x7f},json(127)); - - check_decode_msgpack({0xcc,0xff},json(255)); - check_decode_msgpack({0xcd,0x01,0x00},json(256)); - check_decode_msgpack({0xcd,0xff,0xff},json(65535)); - check_decode_msgpack({0xce,0,1,0x00,0x00},json(65536)); - check_decode_msgpack({0xce,0xff,0xff,0xff,0xff},json(4294967295)); - check_decode_msgpack({0xd3,0,0,0,1,0,0,0,0},json(4294967296)); - check_decode_msgpack({0xd3,0x7f,0xff,0xff,0xff,0xff,0xff,0xff,0xff},json((std::numeric_limits::max)())); - - // negative fixint 0xe0 - 0xff - check_decode_msgpack({0xe0},json(-32)); - check_decode_msgpack({0xff},json(-1)); // - - // negative integers - check_decode_msgpack({0xd1,0xff,0},json(-256)); - check_decode_msgpack({0xd1,0xfe,0xff},json(-257)); - check_decode_msgpack({0xd2,0xff,0xff,0,0},json(-65536)); - check_decode_msgpack({0xd2,0xff,0xfe,0xff,0xff},json(-65537)); - check_decode_msgpack({0xd3,0xff,0xff,0xff,0xff,0,0,0,0},json(-4294967296)); - check_decode_msgpack({0xd3,0xff,0xff,0xff,0xfe,0xff,0xff,0xff,0xff},json(-4294967297)); - - // null, true, false - check_decode_msgpack({0xc0},json::null()); // - check_decode_msgpack({0xc3},json(true)); // - check_decode_msgpack({0xc2},json(false)); // - - // floating point - check_decode_msgpack({0xcb,0,0,0,0,0,0,0,0},json(0.0)); - check_decode_msgpack({0xcb,0xbf,0xf0,0,0,0,0,0,0},json(-1.0)); - check_decode_msgpack({0xcb,0xc1,0x6f,0xff,0xff,0xe0,0,0,0},json(-16777215.0)); - - // string - check_decode_msgpack({0xa0},json("")); - check_decode_msgpack({0xa1,' '},json(" ")); - check_decode_msgpack({0xbf,'1','2','3','4','5','6','7','8','9','0', - '1','2','3','4','5','6','7','8','9','0', - '1','2','3','4','5','6','7','8','9','0', - '1'}, - json("1234567890123456789012345678901")); - check_decode_msgpack({0xd9,0x20,'1','2','3','4','5','6','7','8','9','0', - '1','2','3','4','5','6','7','8','9','0', - '1','2','3','4','5','6','7','8','9','0', - '1','2'}, - json("12345678901234567890123456789012")); - -} - -TEST_CASE("decode_msgpack_arrays_and_maps") -{ - // fixarray - check_decode_msgpack({0x90},json::array()); - check_decode_msgpack({0x80},json::object()); - - check_decode_msgpack({0x91,'\0'},json::parse("[0]")); - check_decode_msgpack({0x92,'\0','\0'},json::array({0,0})); - check_decode_msgpack({0x92,0x91,'\0','\0'}, json::parse("[[0],0]")); - check_decode_msgpack({0x91,0xa5,'H','e','l','l','o'},json::parse("[\"Hello\"]")); - - check_decode_msgpack({0x81,0xa2,'o','c',0x91,'\0'}, json::parse("{\"oc\": [0]}")); - check_decode_msgpack({0x81,0xa2,'o','c',0x94,'\0','\1','\2','\3'}, json::parse("{\"oc\": [0, 1, 2, 3]}")); -} - -TEST_CASE("Compare msgpack packed item and jsoncons item") -{ - std::vector bytes; - msgpack::msgpack_bytes_encoder encoder(bytes); - encoder.begin_array(2); // Must be definite length array - encoder.string_value("foo"); - encoder.byte_string_value(byte_string{'b','a','r'}); - encoder.end_array(); - encoder.flush(); - - json expected = json::array(); - - expected.emplace_back("foo"); - expected.emplace_back(byte_string{ 'b','a','r' }); - - json j = msgpack::decode_msgpack(bytes); - - REQUIRE(j == expected); -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/msgpack/encode_msgpack_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/msgpack/encode_msgpack_tests.cpp deleted file mode 100644 index 97071eb1c6..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/msgpack/encode_msgpack_tests.cpp +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2016 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -void check_encode_msgpack(const std::vector& expected, - const json& j) -{ - std::vector result; - msgpack::encode_msgpack(j, result); - if (result.size() != expected.size()) - { - std::cout << std::hex << (int)expected[0] << " " << std::hex << (int)result[0] << std::endl; - } - REQUIRE(result.size() == expected.size()); - for (size_t i = 0; i < expected.size(); ++i) - { - if (expected[i] != result[i]) - { - std::cout << "Different " << i << "\n"; - for (size_t k = 0; k < expected.size(); ++k) - { - std::cout << std::hex << (int)expected[k] << " " << std::hex << (int)result[k] << std::endl; - } - } - REQUIRE(result[i] == expected[i]); - } -} - -TEST_CASE("encode_msgpack_test") -{ - // positive fixint 0x00 - 0x7f - check_encode_msgpack({0x00},json(0U)); - check_encode_msgpack({0x01},json(1U)); - check_encode_msgpack({0x0a},json(10U)); - check_encode_msgpack({0x17},json(23U)); - check_encode_msgpack({0x18},json(24U)); - check_encode_msgpack({0x7f},json(127U)); - - check_encode_msgpack({0xcc,0xff},json(255U)); - check_encode_msgpack({0xcd,0x01,0x00},json(256U)); - check_encode_msgpack({0xcd,0xff,0xff},json(65535U)); - check_encode_msgpack({0xce,0,1,0x00,0x00},json(65536U)); - check_encode_msgpack({0xce,0xff,0xff,0xff,0xff},json(4294967295U)); - check_encode_msgpack({0xcf,0,0,0,1,0,0,0,0},json(4294967296U)); - check_encode_msgpack({0xcf,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff},json((std::numeric_limits::max)())); - - check_encode_msgpack({0x01},json(1)); - check_encode_msgpack({0x0a},json(10)); - check_encode_msgpack({0x17},json(23)); - check_encode_msgpack({0x18},json(24)); - check_encode_msgpack({0x7f},json(127)); - - check_encode_msgpack({0xcc,0xff},json(255)); - check_encode_msgpack({0xcd,0x01,0x00},json(256)); - check_encode_msgpack({0xcd,0xff,0xff},json(65535)); - check_encode_msgpack({0xce,0,1,0x00,0x00},json(65536)); - check_encode_msgpack({0xce,0xff,0xff,0xff,0xff},json(4294967295)); - check_encode_msgpack({0xcf,0,0,0,1,0,0,0,0},json(4294967296)); - check_encode_msgpack({0xcf,0x7f,0xff,0xff,0xff,0xff,0xff,0xff,0xff},json((std::numeric_limits::max)())); - - // negative fixint 0xe0 - 0xff - check_encode_msgpack({0xe0},json(-32)); - check_encode_msgpack({0xff},json(-1)); // - - // negative integers - check_encode_msgpack({0xd1,0xff,0},json(-256)); - check_encode_msgpack({0xd1,0xfe,0xff},json(-257)); - check_encode_msgpack({0xd2,0xff,0xff,0,0},json(-65536)); - check_encode_msgpack({0xd2,0xff,0xfe,0xff,0xff},json(-65537)); - check_encode_msgpack({0xd3,0xff,0xff,0xff,0xff,0,0,0,0},json(-4294967296)); - check_encode_msgpack({0xd3,0xff,0xff,0xff,0xfe,0xff,0xff,0xff,0xff},json(-4294967297)); - - // null, true, false - check_encode_msgpack({0xc0},json::null()); // - check_encode_msgpack({0xc3},json(true)); // - check_encode_msgpack({0xc2},json(false)); // - - // floating point - check_encode_msgpack({0xca,0,0,0,0},json(0.0)); - check_encode_msgpack({0xca,0xbf,0x80,0,0},json(-1.0)); - check_encode_msgpack({0xca,0xcb,0x7f,0xff,0xff},json(-16777215.0)); - - // string - check_encode_msgpack({0xa0},json("")); - check_encode_msgpack({0xa1,' '},json(" ")); - check_encode_msgpack({0xbf,'1','2','3','4','5','6','7','8','9','0', - '1','2','3','4','5','6','7','8','9','0', - '1','2','3','4','5','6','7','8','9','0', - '1'}, - json("1234567890123456789012345678901")); - check_encode_msgpack({0xd9,0x20,'1','2','3','4','5','6','7','8','9','0', - '1','2','3','4','5','6','7','8','9','0', - '1','2','3','4','5','6','7','8','9','0', - '1','2'}, - json("12345678901234567890123456789012")); - - -} - -TEST_CASE("encode_msgpack_arrays_and_maps") -{ - // fixarray - check_encode_msgpack({0x90},json::array()); - check_encode_msgpack({0x80},json::object()); - - check_encode_msgpack({0x91,'\0'},json::parse("[0]")); - check_encode_msgpack({0x92,'\0','\0'},json::array({0,0})); - check_encode_msgpack({0x92,0x91,'\0','\0'}, json::parse("[[0],0]")); - check_encode_msgpack({0x91,0xa5,'H','e','l','l','o'},json::parse("[\"Hello\"]")); - - check_encode_msgpack({0x81,0xa2,'o','c',0x91,'\0'}, json::parse("{\"oc\": [0]}")); - check_encode_msgpack({0x81,0xa2,'o','c',0x94,'\0','\1','\2','\3'}, json::parse("{\"oc\": [0, 1, 2, 3]}")); -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/msgpack/msgpack_encoder_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/msgpack/msgpack_encoder_tests.cpp deleted file mode 100644 index aee9765ca5..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/msgpack/msgpack_encoder_tests.cpp +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2016 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("serialize array to msgpack") -{ - std::vector v; - msgpack::msgpack_bytes_encoder encoder(v); - //encoder.begin_object(1); - encoder.begin_array(3); - encoder.bool_value(true); - encoder.bool_value(false); - encoder.null_value(); - encoder.end_array(); - //encoder.end_object(); - encoder.flush(); - - try - { - json result = msgpack::decode_msgpack(v); - std::cout << result << std::endl; - } - catch (const std::exception& e) - { - std::cout << e.what() << std::endl; - } -} - -TEST_CASE("Too many and too few items in MessagePack object or array") -{ - std::error_code ec{}; - std::vector v; - msgpack::msgpack_bytes_encoder encoder(v); - - SECTION("Too many items in array") - { - CHECK(encoder.begin_array(3)); - CHECK(encoder.bool_value(true)); - CHECK(encoder.bool_value(false)); - CHECK(encoder.null_value()); - CHECK(encoder.begin_array(2)); - CHECK(encoder.string_value("cat")); - CHECK(encoder.string_value("feline")); - CHECK(encoder.end_array()); - REQUIRE_THROWS_WITH(encoder.end_array(), msgpack::msgpack_error_category_impl().message((int)msgpack::msgpack_errc::too_many_items).c_str()); - encoder.flush(); - } - SECTION("Too few items in array") - { - CHECK(encoder.begin_array(5)); - CHECK(encoder.bool_value(true)); - CHECK(encoder.bool_value(false)); - CHECK(encoder.null_value()); - CHECK(encoder.begin_array(2)); - CHECK(encoder.string_value("cat")); - CHECK(encoder.string_value("feline")); - CHECK(encoder.end_array()); - REQUIRE_THROWS_WITH(encoder.end_array(), msgpack::msgpack_error_category_impl().message((int)msgpack::msgpack_errc::too_few_items).c_str()); - encoder.flush(); - } - SECTION("Too many items in object") - { - CHECK(encoder.begin_object(3)); - CHECK(encoder.name("a")); - CHECK(encoder.bool_value(true)); - CHECK(encoder.name("b")); - CHECK(encoder.bool_value(false)); - CHECK(encoder.name("c")); - CHECK(encoder.null_value()); - CHECK(encoder.name("d")); - CHECK(encoder.begin_array(2)); - CHECK(encoder.string_value("cat")); - CHECK(encoder.string_value("feline")); - CHECK(encoder.end_array()); - REQUIRE_THROWS_WITH(encoder.end_object(), msgpack::msgpack_error_category_impl().message((int)msgpack::msgpack_errc::too_many_items).c_str()); - encoder.flush(); - } - SECTION("Too few items in object") - { - CHECK(encoder.begin_object(5)); - CHECK(encoder.name("a")); - CHECK(encoder.bool_value(true)); - CHECK(encoder.name("b")); - CHECK(encoder.bool_value(false)); - CHECK(encoder.name("c")); - CHECK(encoder.null_value()); - CHECK(encoder.name("d")); - CHECK(encoder.begin_array(2)); - CHECK(encoder.string_value("cat")); - CHECK(encoder.string_value("feline")); - CHECK(encoder.end_array()); - REQUIRE_THROWS_WITH(encoder.end_object(), msgpack::msgpack_error_category_impl().message((int)msgpack::msgpack_errc::too_few_items).c_str()); - encoder.flush(); - } -} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/msgpack/msgpack_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/msgpack/msgpack_tests.cpp deleted file mode 100644 index 99a0ae9dcd..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/msgpack/msgpack_tests.cpp +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2016 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; -using namespace jsoncons::msgpack; - -TEST_CASE("msgpack_test") -{ - json j1; - j1["zero"] = 0; - j1["one"] = 1; - j1["two"] = 2; - j1["null"] = null_type(); - j1["true"] = true; - j1["false"] = false; - j1["max int64_t"] = (std::numeric_limits::max)(); - j1["max uint64_t"] = (std::numeric_limits::max)(); - j1["min int64_t"] = (std::numeric_limits::lowest)(); - j1["max int32_t"] = (std::numeric_limits::max)(); - j1["max uint32_t"] = (std::numeric_limits::max)(); - j1["min int32_t"] = (std::numeric_limits::lowest)(); - j1["max int16_t"] = (std::numeric_limits::max)(); - j1["max uint16_t"] = (std::numeric_limits::max)(); - j1["min int16_t"] = (std::numeric_limits::lowest)(); - j1["max int8_t"] = (std::numeric_limits::max)(); - j1["max uint8_t"] = (std::numeric_limits::max)(); - j1["min int8_t"] = (std::numeric_limits::lowest)(); - j1["max double"] = (std::numeric_limits::max)(); - j1["min double"] = (std::numeric_limits::lowest)(); - j1["max float"] = (std::numeric_limits::max)(); - j1["zero float"] = 0.0; - j1["min float"] = (std::numeric_limits::lowest)(); - j1["String too long for small string optimization"] = "String too long for small string optimization"; - - json ja = json::array(); - ja.push_back(0); - ja.push_back(1); - ja.push_back(2); - ja.push_back(null_type()); - ja.push_back(true); - ja.push_back(false); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back(0.0); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back("String too long for small string optimization"); - - j1["An array"] = ja; - - std::vector v; - encode_msgpack(j1, v); - - json j2 = decode_msgpack(v); - - CHECK(j1 == j2); -} - -TEST_CASE("msgpack_test2") -{ - wjson j1; - j1[L"zero"] = 0; - j1[L"one"] = 1; - j1[L"two"] = 2; - j1[L"null"] = null_type(); - j1[L"true"] = true; - j1[L"false"] = false; - j1[L"max int64_t"] = (std::numeric_limits::max)(); - j1[L"max uint64_t"] = (std::numeric_limits::max)(); - j1[L"min int64_t"] = (std::numeric_limits::lowest)(); - j1[L"max int32_t"] = (std::numeric_limits::max)(); - j1[L"max uint32_t"] = (std::numeric_limits::max)(); - j1[L"min int32_t"] = (std::numeric_limits::lowest)(); - j1[L"max int16_t"] = (std::numeric_limits::max)(); - j1[L"max uint16_t"] = (std::numeric_limits::max)(); - j1[L"min int16_t"] = (std::numeric_limits::lowest)(); - j1[L"max int8_t"] = (std::numeric_limits::max)(); - j1[L"max uint8_t"] = (std::numeric_limits::max)(); - j1[L"min int8_t"] = (std::numeric_limits::lowest)(); - j1[L"max double"] = (std::numeric_limits::max)(); - j1[L"min double"] = (std::numeric_limits::lowest)(); - j1[L"max float"] = (std::numeric_limits::max)(); - j1[L"zero float"] = 0.0; - j1[L"min float"] = (std::numeric_limits::lowest)(); - j1[L"S"] = L"S"; - j1[L"String too long for small string optimization"] = L"String too long for small string optimization"; - - wjson ja = wjson::array(); - ja.push_back(0); - ja.push_back(1); - ja.push_back(2); - ja.push_back(null_type()); - ja.push_back(true); - ja.push_back(false); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back((std::numeric_limits::max)()); - ja.push_back(0.0); - ja.push_back((std::numeric_limits::lowest)()); - ja.push_back(L"S"); - ja.push_back(L"String too long for small string optimization"); - - j1[L"An array"] = ja; - - std::vector v; - encode_msgpack(j1, v); - - //wjson j2 = decode_msgpack(v); - - //CHECK(j1 == j2); -} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/ojson_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/ojson_tests.cpp deleted file mode 100644 index 5d1fdc11bd..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/ojson_tests.cpp +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("test_index") -{ - ojson o = ojson::parse(R"( - { - "street_number" : "100", - "street_name" : "Queen St W", - "city" : "Toronto", - "country" : "Canada" - } - )"); - - CHECK("100" == o[0].as()); - CHECK("Queen St W" == o[1].as()); - CHECK("Toronto" ==o[2].as()); - CHECK("Canada" ==o[3].as()); - - CHECK("100" == o.at(0).as()); - CHECK("Queen St W" == o.at(1).as()); - CHECK("Toronto" ==o.at(2).as()); - CHECK("Canada" ==o.at(3).as()); -} - -TEST_CASE("test_object") -{ - ojson o = ojson::parse(R"( - { - "street_number" : "100", - "street_name" : "Queen St W", - "city" : "Toronto", - "country" : "Canada" - } - )"); - - o.insert_or_assign("postal_code", "M5H 2N2"); - - ojson o2 = o; - CHECK(o == o2); - - ojson o3 = o; - o3["street_name"] = "Queen St W"; - //CHECK(o == o3); - - //BOOST_CHECK_EQUAL("Queen St W",o["street_name"].as()); - //CHECK(2 == o["city"].as()); - //CHECK(4 == o["street_number"].as()); - - auto it = o.find("country"); - CHECK_FALSE((it == o.object_range().end())); - o.insert_or_assign(it,"province","Ontario"); - - o.insert_or_assign("unit_type","O"); - - o.erase("unit_type"); -} - -TEST_CASE("test_object_emplace") -{ - ojson o = ojson::parse(R"( - { - "street_number" : "100", - "street_name" : "Queen St W", - "city" : "Toronto", - "country" : "Canada" - } - )"); - - o.try_emplace("postal_code", "M5H 2N2"); - - ojson o2 = o; - CHECK(o == o2); - - ojson o3 = o; - o3["street_name"] = "Queen St W"; - //CHECK(o == o3); - - //BOOST_CHECK_EQUAL("Queen St W",o["street_name"].as()); - //CHECK(2 == o["city"].as()); - //CHECK(4 == o["street_number"].as()); - - auto it = o.find("country"); - CHECK_FALSE((it == o.object_range().end())); - o.try_emplace(it,"province","Ontario"); - - o.try_emplace("unit_type","O"); - - o.erase("unit_type"); -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/order_preserving_json_object_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/order_preserving_json_object_tests.cpp deleted file mode 100644 index 6dd362666f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/order_preserving_json_object_tests.cpp +++ /dev/null @@ -1,290 +0,0 @@ -// Copyright 2019 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("json sorted_unique_range_tag") -{ - typedef std::pair item_type; - std::vector items; - items.emplace_back("a", 1); - items.emplace_back("b", 2); - items.emplace_back("c", 3); - items.emplace_back("d", 4); - items.emplace_back("e", 5); - items.emplace_back("f", 6); - items.emplace_back("g", 7); - - json j; - j.insert(sorted_unique_range_tag(), items.begin(), items.end()); - - SECTION("iterate") - { - REQUIRE(j.size() == 7); - - auto it = j.object_range().begin(); - CHECK(it->key() == std::string("a")); - CHECK(it->value().as() == 1); - CHECK((++it)->key() == std::string("b")); - CHECK(it->value().as() == 2); - CHECK((++it)->key() == std::string("c")); - CHECK(it->value().as() == 3); - CHECK((++it)->key() == std::string("d")); - CHECK(it->value().as() == 4); - CHECK((++it)->key() == std::string("e")); - CHECK(it->value().as() == 5); - CHECK((++it)->key() == std::string("f")); - CHECK(it->value().as() == 6); - CHECK((++it)->key() == std::string("g")); - CHECK(it->value().as() == 7); - } - - SECTION("find") - { - auto it1 = j.find("a"); - REQUIRE(bool(it1 != j.object_range().end())); - CHECK(it1->value().as() == 1); - - auto it2 = j.find("b"); - REQUIRE(bool(it2 != j.object_range().end())); - CHECK(it2->value().as() == 2); - - auto it3 = j.find("c"); - REQUIRE(bool(it3 != j.object_range().end())); - CHECK(it3->value().as() == 3); - - auto it4 = j.find("d"); - REQUIRE(bool(it4 != j.object_range().end())); - CHECK(it4->value().as() == 4); - } -} - -TEST_CASE("ojson sorted_unique_range_tag") -{ - typedef std::pair item_type; - std::vector items; - items.emplace_back("a", 1); - items.emplace_back("b", 2); - items.emplace_back("c", 3); - items.emplace_back("d", 4); - items.emplace_back("e", 5); - items.emplace_back("f", 6); - items.emplace_back("g", 7); - - ojson j; - j.insert(sorted_unique_range_tag(), items.begin(), items.end()); - - SECTION("iterate") - { - REQUIRE(j.size() == 7); - - auto it = j.object_range().begin(); - CHECK(it->key() == std::string("a")); - CHECK(it->value().as() == 1); - CHECK((++it)->key() == std::string("b")); - CHECK(it->value().as() == 2); - CHECK((++it)->key() == std::string("c")); - CHECK(it->value().as() == 3); - CHECK((++it)->key() == std::string("d")); - CHECK(it->value().as() == 4); - CHECK((++it)->key() == std::string("e")); - CHECK(it->value().as() == 5); - CHECK((++it)->key() == std::string("f")); - CHECK(it->value().as() == 6); - CHECK((++it)->key() == std::string("g")); - CHECK(it->value().as() == 7); - } - - SECTION("find") - { - auto it1 = j.find("a"); - REQUIRE(bool(it1 != j.object_range().end())); - CHECK(it1->value().as() == 1); - - auto it2 = j.find("b"); - REQUIRE(bool(it2 != j.object_range().end())); - CHECK(it2->value().as() == 2); - - auto it3 = j.find("c"); - REQUIRE(bool(it3 != j.object_range().end())); - CHECK(it3->value().as() == 3); - - auto it4 = j.find("d"); - REQUIRE(bool(it4 != j.object_range().end())); - CHECK(it4->value().as() == 4); - } -} - -TEST_CASE("order preserving insert") -{ - json_object o; - - typedef std::pair item_type; - std::vector items; - items.emplace_back("b", 1); - items.emplace_back("a", 2); - items.emplace_back("c", 3); - items.emplace_back("a", 4); - items.emplace_back("a", 5); - items.emplace_back("d", 6); - items.emplace_back("a", 7); - - o.insert(std::make_move_iterator(items.begin()), std::make_move_iterator(items.end()), - [](item_type&& item){return ojson::key_value_type(std::forward(item.first),std::forward(item.second));}); - - SECTION("iterate") - { - REQUIRE(o.size() == 4); - - auto it = o.begin(); - CHECK(it->key() == std::string("b")); - CHECK(it->value().as() == 1); - CHECK((++it)->key() == std::string("a")); - CHECK(it->value().as() == 2); - CHECK((++it)->key() == std::string("c")); - CHECK(it->value().as() == 3); - CHECK((++it)->key() == std::string("d")); - CHECK(it->value().as() == 6); - } - - SECTION("find") - { - auto it1 = o.find("a"); - REQUIRE(bool(it1 != o.end())); - CHECK(it1->value().as() == 2); - - auto it2 = o.find("b"); - REQUIRE(bool(it2 != o.end())); - CHECK(it2->value().as() == 1); - - auto it3 = o.find("c"); - REQUIRE(bool(it3 != o.end())); - CHECK(it3->value().as() == 3); - - auto it4 = o.find("d"); - REQUIRE(bool(it4 != o.end())); - CHECK(it4->value().as() == 6); - } -} - -TEST_CASE("order preserving insert_or_assign") -{ - json_object o; - - o.insert_or_assign("b", 1); - o.insert_or_assign("a", 2); - o.insert_or_assign("c", 3); - o.insert_or_assign("a", 4); - o.insert_or_assign("a", 5); - - SECTION("insert_or_assign") - { - REQUIRE(o.size() == 3); - - auto it = o.find("a"); - REQUIRE(bool(it != o.end())); - CHECK(it->value().as() == 5); - - auto it2 = o.begin(); - CHECK(it2->key() == std::string("b")); - CHECK(it2->value().as() == 1); - CHECK((++it2)->key() == std::string("a")); - CHECK(it2->value().as() == 5); - CHECK((++it2)->key() == std::string("c")); - CHECK(it2->value().as() == 3); - } - - SECTION("insert_or_assign at pos") - { - auto it = o.find("a"); - auto it2 = o.insert_or_assign(it,"d",3); - CHECK_FALSE((it2 == o.end())); - - auto it3 = o.begin(); - CHECK(it3->key() == std::string("b")); - CHECK(it3->value().as() == 1); - CHECK((++it3)->key() == std::string("d")); - CHECK(it3->value().as() == 3); - CHECK((++it3)->key() == std::string("a")); - CHECK(it3->value().as() == 5); - CHECK((++it3)->key() == std::string("c")); - CHECK(it3->value().as() == 3); - - //for (auto kv : o) - //{ - // std::cout << kv.key() << ": " << kv.value() << "\n"; - //} - } - - SECTION("try_emplace") - { - REQUIRE(o.size() == 3); - - o.try_emplace("d",7); - o.try_emplace("d",8); - - auto it3 = o.begin(); - CHECK(it3->key() == std::string("b")); - CHECK(it3->value().as() == 1); - CHECK((++it3)->key() == std::string("a")); - CHECK(it3->value().as() == 5); - CHECK((++it3)->key() == std::string("c")); - CHECK(it3->value().as() == 3); - CHECK((++it3)->key() == std::string("d")); - CHECK(it3->value().as() == 7); - } - - SECTION("try_emplace at pos") - { - auto it = o.find("a"); - auto it2 = o.try_emplace(it,"d",7); - o.try_emplace(it2, "d", 8); - - auto it3 = o.begin(); - CHECK(it3->key() == std::string("b")); - CHECK(it3->value().as() == 1); - CHECK((++it3)->key() == std::string("d")); - CHECK(it3->value().as() == 7); - CHECK((++it3)->key() == std::string("a")); - CHECK(it3->value().as() == 5); - CHECK((++it3)->key() == std::string("c")); - CHECK(it3->value().as() == 3); - } - - SECTION("erase") - { - REQUIRE(o.size() == 3); - - o.erase("a"); - REQUIRE(o.size() == 2); - - auto it2 = o.begin(); - CHECK(it2->key() == std::string("b")); - CHECK(it2->value().as() == 1); - CHECK((++it2)->key() == std::string("c")); - CHECK(it2->value().as() == 3); - } - - SECTION("erase range") - { - REQUIRE(o.size() == 3); - - o.erase(o.begin(),o.begin()+2); - REQUIRE(o.size() == 1); - - auto it2 = o.begin(); - CHECK(it2->key() == std::string("c")); - CHECK(it2->value().as() == 3); - } -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/parse_string_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/parse_string_tests.cpp deleted file mode 100644 index 6e5191d1e3..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/parse_string_tests.cpp +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2016 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -struct jsonpath_filter_fixture -{ -}; - -class lenient_error_handler : public parse_error_handler -{ -public: - lenient_error_handler(std::error_code value) - : value_(value) - { - } -private: - std::error_code value_; - - bool do_error(std::error_code ec, const ser_context&) noexcept - { - return ec == value_; // if returns true, use default processing - } -}; - -TEST_CASE("test_parse_small_string1") -{ - std::string input = "\"String\""; - - json_decoder decoder; - try - { - json_reader reader(input,decoder); - reader.read_next(); - } - catch (const std::exception&) - { - } - CHECK(decoder.is_valid()); -} - -TEST_CASE("test_parse_small_string2") -{ - std::string input = "\"Str\\\"ing\""; - - json_decoder decoder; - try - { - json_reader reader(input, decoder); - reader.read_next(); - } - catch (const std::exception&) - { - } - CHECK(decoder.is_valid()); -} - -TEST_CASE("test_parse_small_string4") -{ - std::string input = "\"Str\\\"ing\""; - - for (size_t i = 2; i < input.length(); ++i) - { - std::istringstream is(input); - json_decoder decoder; - try - { - json_reader reader(is, decoder); - reader.buffer_length(i); - reader.read_next(); - } - catch (const std::exception&) - { - } - CHECK(decoder.is_valid()); - CHECK(std::string("Str\"ing") == decoder.get_result().as()); - } -} -TEST_CASE("test_parse_big_string1") -{ - std::string input = "\"Big Str\\\"ing\""; - - for (size_t i = 2; i < input.length(); ++i) - { - std::istringstream is(input); - json_decoder decoder; - try - { - json_reader reader(is, decoder); - reader.buffer_length(i); - reader.read_next(); - } - catch (const std::exception&) - { - } - CHECK(decoder.is_valid()); - CHECK(std::string("Big Str\"ing") == decoder.get_result().as()); - } -} - -TEST_CASE("test_parse_big_string2") -{ - std::string input = "\"Big\t Str\\\"ing\""; - - //for (size_t i = 2; i < input.length(); ++i) - //{ - std::istringstream is(input); - json_decoder decoder; - lenient_error_handler err_handler(json_errc::illegal_character_in_string); - try - { - json_reader reader(is, decoder, err_handler); - //reader.buffer_length(i); - reader.read_next(); - } - catch (const std::exception&) - { - } - CHECK(decoder.is_valid()); - CHECK(std::string("Big\t Str\"ing") == decoder.get_result().as()); - //} -} - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/short_string_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/short_string_tests.cpp deleted file mode 100644 index 151c50ad2f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/short_string_tests.cpp +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("test_small_string") -{ - json s("ABCD"); - CHECK(s.get_storage_type() == jsoncons::storage_type::short_string_val); - CHECK(s.as() == std::string("ABCD")); - - json t(s); - CHECK(t.get_storage_type() == jsoncons::storage_type::short_string_val); - CHECK(t.as() == std::string("ABCD")); - - json q; - q = s; - CHECK(q.get_storage_type() == jsoncons::storage_type::short_string_val); - CHECK(q.as() == std::string("ABCD")); -} - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/staj_iterator_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/staj_iterator_tests.cpp deleted file mode 100644 index 818967be0a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/staj_iterator_tests.cpp +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2018 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("array_iterator test") -{ - std::string s = R"( - [ - { - "enrollmentNo" : 100, - "firstName" : "Tom", - "lastName" : "Cochrane", - "mark" : 55 - }, - { - "enrollmentNo" : 101, - "firstName" : "Catherine", - "lastName" : "Smith", - "mark" : 95 - }, - { - "enrollmentNo" : 102, - "firstName" : "William", - "lastName" : "Skeleton", - "mark" : 60 - } - ] - )"; - - std::istringstream is(s); - - json_cursor reader(is); - - staj_array_iterator it(reader); - - for (const auto& val : it) - { - std::cout << val << "\n"; - } - -} - -TEST_CASE("object_iterator test") -{ - std::string s = R"( - { - "enrollmentNo" : 100, - "firstName" : "Tom", - "lastName" : "Cochrane", - "mark" : 55 - } - )"; - - std::istringstream is(s); - - json_cursor reader(is); - - staj_object_iterator it(reader); - - for (const auto& item : it) - { - std::cout << item.first << ": " << item.second << "\n"; - } -} - - - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/stateful_allocator_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/stateful_allocator_tests.cpp deleted file mode 100644 index 42aa80f39d..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/stateful_allocator_tests.cpp +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -class pool -{ -private: - struct node_type - { - char* memory_ptr_; - node_type* next_ptr_; - }; - struct node_typeA - { - node_type data; - char c[1]; - }; - typedef std::aligned_storage::type storage_type; - - size_t offset_, size_; - node_type* head_; - node_type* curr_; - - node_type* add_node() - { - size_t mem_size = sizeof(storage_type) + size_ - 1; - node_type* storage = reinterpret_cast(alloc(mem_size)); - node_type* p = new(storage)node_type(); - auto pa = reinterpret_cast(storage); - p->memory_ptr_ = new(&pa->c)char[size_]; - p->next_ptr_ = nullptr; - return storage; - } - - void* alloc(size_t n) {return malloc(n);} - void dealloc(void* storage) {free(storage);} -public: - pool(size_t size) - : offset_(0), size_(size), allocate_count_(0), deallocate_count_(0), construct_count_(0), destroy_count_(0) - { - head_ = curr_ = add_node(); - } - ~pool() - { - while (head_) - { - node_type* curr2_ = head_->next_ptr_; - dealloc(head_); - head_ = curr2_; - } - } - void* allocate(size_t n) - { - void *pv; - if (n > (size_ - offset_)) - { - if (size_ < n) - { - size_ = n; - } - curr_->next_ptr_ = add_node(); - curr_ = curr_->next_ptr_; - offset_ = 0; - } - pv = reinterpret_cast(curr_->memory_ptr_ + offset_); - - size_t mem_size = sizeof(storage_type) + n - 1; - - offset_ += mem_size; - return pv; - } - - size_t allocate_count_; - size_t deallocate_count_; - size_t construct_count_; - size_t destroy_count_; -}; - -template -class pool_allocator -{ -public: - typedef size_t size_type; - typedef ptrdiff_t difference_type; - typedef T* pointer; - typedef const T* const_pointer; - typedef T& reference; - typedef const T& const_reference; - typedef T value_type; - template - struct rebind - { - typedef pool_allocator other; - }; - pool_allocator(pool* pp) throw() - : pool_ptr_(pp) - { - } - pool_allocator(const pool_allocator& s) throw() - : pool_ptr_(s.pool_ptr_) - { - } - template - pool_allocator(const pool_allocator &s) throw() - : pool_ptr_(s.pool_ptr_) - { - } - ~pool_allocator() throw() - { - } - pointer address(reference x) const - { - return &x; - } - const_pointer address(const_reference x) const - { - return &x; - } - pointer allocate(size_type n, const void* = 0) - { - ++pool_ptr_->allocate_count_; - return static_cast(pool_ptr_->allocate(n * sizeof(T))); - } - void deallocate(pointer, size_type) - { - ++pool_ptr_->deallocate_count_; - } - size_type max_size() const throw() - { - return size_t(-1) / sizeof(T); - } - template - void construct(pointer p, Args&&... args) - { - ::new(p) T(std::forward(args)...); - ++pool_ptr_->construct_count_; - } - void destroy(pointer p) - { - (void)p; - p->~T(); - ++pool_ptr_->destroy_count_; - } - pool* pool_ptr_; -}; - -template -bool operator==(const pool_allocator &s0, const pool_allocator &s1) -{ - return s0.pool_ptr_ == s1.pool_ptr_; -} -template -bool operator!=(const pool_allocator &s0, const pool_allocator &s1) -{ - return s0.pool_ptr_ != s1.pool_ptr_; -} -#if !defined(__GNUC__) -// basic_string doesn't satisfy C++11 allocator requirements -TEST_CASE("test_string_allocation") -{ - - pool a_pool(1024); - pool_allocator allocator(&a_pool); - - typedef basic_json> myjson; - - { - myjson j("String too long for short string", allocator); - } - std::cout << "Allocate count = " << a_pool.allocate_count_ - << ", construct count = " << a_pool.construct_count_ - << ", destroy count = " << a_pool.destroy_count_ - << ", deallocate count = " << a_pool.deallocate_count_ << std::endl; - CHECK(a_pool.allocate_count_ == a_pool.deallocate_count_); - CHECK(a_pool.construct_count_ == a_pool.destroy_count_); - -} -#endif - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/string_to_double_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/string_to_double_tests.cpp deleted file mode 100644 index c632e3d10e..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/string_to_double_tests.cpp +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("test_string_to_double") -{ - std::cout << "sizeof(json): " << sizeof(json) << std::endl; - - const char* s1 = "0.0"; - json j1 = json::parse(s1); - double expected1 = 0.0; - CHECK( j1.as() == expected1); - - const char* s2 = "0.123456789"; - json j2 = json::parse(s2); - double expected2 = 0.123456789; - CHECK( j2.as() == expected2); - - const char* s3 = "123456789.123456789"; - json j3 = json::parse(s3); - char* end3 = nullptr; - double expected3 = strtod(s3,&end3); - CHECK( j3.as() == expected3); -} - -TEST_CASE("test_exponent") -{ - jsoncons::detail::string_to_double reader; - const char* begin = "1.15507e-173"; - char* endptr = nullptr; - const double value1 = 1.15507e-173; - const double value2 = strtod(begin, &endptr ); - const double value3 = reader(begin,endptr-begin); - - CHECK(value1 == value2); - CHECK(value2 == value3); - - const char* s1 = "1.15507e+173"; - json j1 = json::parse(s1); - double expected1 = 1.15507e+173; - CHECK( j1.as() == expected1); - -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/string_view_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/string_view_tests.cpp deleted file mode 100644 index d7515798bd..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/string_view_tests.cpp +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2019 Daniel Parker -// Distributed under Boost license - -#include -#include - -#if defined(JSONCONS_HAS_STRING_VIEW) - -#include - -using namespace jsoncons; - -TEST_CASE("string_view tests") -{ - std::cout << "string_view tests\n"; - json j = json::parse(R"( - { - "a" : "2", - "c" : [4,5,6] - } - )"); - - auto s = j["a"].as(); - CHECK(bool(s == "2")); -} - -#endif - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/tests_main.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/tests_main.cpp deleted file mode 100644 index 1d646dc52f..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/tests_main.cpp +++ /dev/null @@ -1,3 +0,0 @@ -#define CATCH_CONFIG_MAIN -#include - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/ubjson/decode_ubjson_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/ubjson/decode_ubjson_tests.cpp deleted file mode 100644 index 3b0d070e0a..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/ubjson/decode_ubjson_tests.cpp +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright 2016 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -void check_decode_ubjson(const std::vector& v, const json& expected) -{ - json j1 = ubjson::decode_ubjson(v); - REQUIRE(j1 == expected); - - std::string s; - for (auto c : v) - { - s.push_back(c); - } - std::istringstream is(s); - json j2 = ubjson::decode_ubjson(is); - REQUIRE(j2 == expected); -} - -void check_decode_ubjson(const std::vector& expected, const std::vector& result) -{ - if (result.size() != expected.size()) - { - std::cout << std::hex << (int)expected[0] << " " << std::hex << (int)result[0] << std::endl; - } - REQUIRE(result.size() == expected.size()); - for (size_t i = 0; i < expected.size(); ++i) - { - if (expected[i] != result[i]) - { - std::cout << "Different " << i << "\n"; - for (size_t k = 0; k < expected.size(); ++k) - { - std::cout << std::hex << std::setprecision(2) << std::setw(2) - << std::noshowbase << std::setfill('0') << static_cast(result[k]); - } - } - REQUIRE(result[i] == expected[i]); - } -} - -TEST_CASE("decode_number_ubjson_test") -{ - SECTION("null, true, false") - { - check_decode_ubjson({'Z'},json::null()); - check_decode_ubjson({'T'},json(true)); - check_decode_ubjson({'F'},json(false)); - } - SECTION("uint8") - { - check_decode_ubjson({'U',0x00},json(0U)); - check_decode_ubjson({'U',0x01},json(1U)); - check_decode_ubjson({'U',0x0a},json(10U)); - check_decode_ubjson({'U',0x17},json(23U)); - check_decode_ubjson({'U',0x18},json(24U)); - check_decode_ubjson({'U',0x7f},json(127U)); - check_decode_ubjson({'U',0xff},json(255U)); - } - SECTION("int8,int16,int32,int64") - { - check_decode_ubjson({'i',0xff},json(-1)); - check_decode_ubjson({'I',0x01,0x00},json(256)); - check_decode_ubjson({'l',0,0,0xff,0xff},json(65535)); - check_decode_ubjson({'l',0,1,0x00,0x00},json(65536)); - check_decode_ubjson({'L',0,0,0,0,0xff,0xff,0xff,0xff},json(4294967295)); - check_decode_ubjson({'L',0,0,0,1,0,0,0,0},json(4294967296)); - check_decode_ubjson({'L',0x7f,0xff,0xff,0xff,0xff,0xff,0xff,0xff},json((std::numeric_limits::max)())); - // negative integers - check_decode_ubjson({'I',0xff,0},json(-256)); - check_decode_ubjson({'I',0xfe,0xff},json(-257)); - check_decode_ubjson({'l',0xff,0xff,0,0},json(-65536)); - check_decode_ubjson({'l',0xff,0xfe,0xff,0xff},json(-65537)); - check_decode_ubjson({'L',0xff,0xff,0xff,0xff,0,0,0,0},json(-4294967296)); - check_decode_ubjson({'L',0xff,0xff,0xff,0xfe,0xff,0xff,0xff,0xff},json(-4294967297)); - } - - SECTION("float32,float64") - { - check_decode_ubjson({'D',0,0,0,0,0,0,0,0},json(0.0)); - check_decode_ubjson({'D',0xbf,0xf0,0,0,0,0,0,0},json(-1.0)); - check_decode_ubjson({'D',0xc1,0x6f,0xff,0xff,0xe0,0,0,0},json(-16777215.0)); - } - - SECTION("array") - { - check_decode_ubjson({'[',']'},json::parse("[]")); - check_decode_ubjson({'[', 'Z', 'T', 'F', ']'},json::parse("[null,true,false]")); - check_decode_ubjson({'[','#','i',0},json::parse("[]")); - check_decode_ubjson({'[','#','i',1,'I',0xff,0},json::parse("[-256]")); - } - SECTION("ubjson array optimized with type and count") - { - check_decode_ubjson({'[','$','I','#','i',2, - 0x01,0x00, // 256 - 0xff,0}, // -256 - json::parse("[256,-256]")); - } - SECTION("ubjson object optimized with type and count") - { - check_decode_ubjson({'{','$','I','#','i',2, - 'i',5,'f','i','r','s','t', - 0x01,0x00, // 256 - 'i',6,'s','e','c','o','n','d', - 0xff,0}, // -256 - json::parse("{\"first\":256,\"second\":-256}")); - } -} - -TEST_CASE("decode_ubjson_arrays_and_maps") -{ - check_decode_ubjson({'[','#','U',0x00},json::array()); - check_decode_ubjson({ '{','#','U',0x00 }, json::object()); - - check_decode_ubjson({'[','#','U',0x01,'U',0x00},json::parse("[0]")); - check_decode_ubjson({'[','#','U',0x02,'U',0x00,'U',0x00},json::parse("[0,0]")); - check_decode_ubjson({'[','#','U',0x02, - '[','#','U',0x01,'U',0x00, - 'U',0x00},json::parse("[[0],0]")); - check_decode_ubjson({'[','#','U',0x01,'S','U',0x05,'H','e','l','l','o'},json::parse("[\"Hello\"]")); - check_decode_ubjson({'{','#','U',0x01,'U',0x02,'o','c','[','#','U',0x01,'U',0x00}, json::parse("{\"oc\": [0]}")); - check_decode_ubjson({'{','#','U',0x01,'U',0x02,'o','c','[','#','U',0x04,'U',0x00,'U',0x01,'U',0x02,'U',0x03}, json::parse("{\"oc\": [0,1,2,3]}")); -} - -TEST_CASE("decode indefinite length ubjson arrays and maps") -{ - std::vector v; - ubjson::ubjson_bytes_encoder encoder(v); - - SECTION("[\"Hello\"]") - { - encoder.begin_array(); - encoder.string_value("Hello"); - encoder.end_array(); - - check_decode_ubjson({'[','S','U',0x05,'H','e','l','l','o',']'}, v); - } - - SECTION("{\"oc\": [0]}") - { - encoder.begin_object(); - encoder.name("oc"); - encoder.begin_array(); - encoder.uint64_value(0); - encoder.end_array(); - encoder.end_object(); - - check_decode_ubjson({'{','U',0x02,'o','c','[','U',0x00,']','}'}, v); - } - - SECTION("{\"oc\": [0,1,2,3]}") - { - encoder.begin_object(); - encoder.name("oc"); - encoder.begin_array(); - encoder.uint64_value(0); - encoder.uint64_value(1); - encoder.uint64_value(2); - encoder.uint64_value(3); - encoder.end_array(); - encoder.end_object(); - - check_decode_ubjson({'{','U',0x02,'o','c','[','U',0x00,'U',0x01,'U',0x02,'U',0x03,']','}'}, v); - } -} - - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/ubjson/encode_ubjson_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/ubjson/encode_ubjson_tests.cpp deleted file mode 100644 index 9fdcc178a2..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/ubjson/encode_ubjson_tests.cpp +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2016 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -void check_encode_ubjson(const std::vector& expected, const json& j) -{ - std::vector result; - ubjson::encode_ubjson(j, result); - if (result.size() != expected.size()) - { - std::cout << std::hex << (int)expected[0] << " " << std::hex << (int)result[0] << std::endl; - } - REQUIRE(result.size() == expected.size()); - for (size_t i = 0; i < expected.size(); ++i) - { - if (expected[i] != result[i]) - { - std::cout << "Different " << i << "\n"; - for (size_t k = 0; k < expected.size(); ++k) - { - std::cout << std::hex << (int)expected[k] << " " << std::hex << (int)result[k] << std::endl; - } - } - REQUIRE(result[i] == expected[i]); - } -} - -void check_encode_ubjson(const std::vector& expected, const std::vector& result) -{ - if (result.size() != expected.size()) - { - std::cout << std::hex << (int)expected[0] << " " << std::hex << (int)result[0] << std::endl; - } - REQUIRE(result.size() == expected.size()); - for (size_t i = 0; i < expected.size(); ++i) - { - if (expected[i] != result[i]) - { - std::cout << "Different " << i << "\n"; - for (size_t k = 0; k < expected.size(); ++k) - { - std::cout << std::hex << (int)expected[k] << " " << std::hex << (int)result[k] << std::endl; - } - } - REQUIRE(result[i] == expected[i]); - } -} - -TEST_CASE("encode_ubjson_test") -{ - check_encode_ubjson({'U',0x00},json(0U)); - check_encode_ubjson({'U',0x01},json(1U)); - check_encode_ubjson({'U',0x0a},json(10U)); - check_encode_ubjson({'U',0x17},json(23U)); - check_encode_ubjson({'U',0x18},json(24U)); - check_encode_ubjson({'U',0x7f},json(127U)); - check_encode_ubjson({'U',0xff},json(255U)); - check_encode_ubjson({'I',0x01,0x00},json(256U)); - check_encode_ubjson({'l',0,1,0x00,0x00},json(65536U)); - check_encode_ubjson({'L',0,0,0,1,0,0,0,0},json(4294967296U)); - - check_encode_ubjson({'U',0x01},json(1)); - check_encode_ubjson({'U',0x0a},json(10)); - check_encode_ubjson({'U',0x17},json(23)); - check_encode_ubjson({'U',0x18},json(24)); - check_encode_ubjson({'U',0x7f},json(127)); - - check_encode_ubjson({'U',0xff},json(255)); - check_encode_ubjson({'I',0x01,0x00},json(256)); - check_encode_ubjson({'l',0,1,0x00,0x00},json(65536)); - check_encode_ubjson({'L',0,0,0,1,0,0,0,0},json(4294967296)); - check_encode_ubjson({'L',0x7f,0xff,0xff,0xff,0xff,0xff,0xff,0xff},json((std::numeric_limits::max)())); - - check_encode_ubjson({'i',0xe0},json(-32)); - check_encode_ubjson({'i',0xff},json(-1)); // - - // negative integers - check_encode_ubjson({'I',0xff,0},json(-256)); - check_encode_ubjson({'I',0xfe,0xff},json(-257)); - check_encode_ubjson({'l',0xff,0xff,0,0},json(-65536)); - check_encode_ubjson({'l',0xff,0xfe,0xff,0xff},json(-65537)); - check_encode_ubjson({'L',0xff,0xff,0xff,0xff,0,0,0,0},json(-4294967296)); - check_encode_ubjson({'L',0xff,0xff,0xff,0xfe,0xff,0xff,0xff,0xff},json(-4294967297)); - - // null, true, false - check_encode_ubjson({'Z'},json::null()); // - check_encode_ubjson({'T'},json(true)); // - check_encode_ubjson({'F'},json(false)); // - - // floating point - check_encode_ubjson({'d',0,0,0,0},json(0.0)); - check_encode_ubjson({'d',0xbf,0x80,0,0},json(-1.0)); - check_encode_ubjson({'d',0xcb,0x7f,0xff,0xff},json(-16777215.0)); - - // string - check_encode_ubjson({'S','U',0x00},json("")); - check_encode_ubjson({'S','U',0x01,' '},json(" ")); - check_encode_ubjson({'S','U',0x1f,'1','2','3','4','5','6','7','8','9','0', - '1','2','3','4','5','6','7','8','9','0', - '1','2','3','4','5','6','7','8','9','0', - '1'}, - json("1234567890123456789012345678901")); - check_encode_ubjson({'S','U',0x20,'1','2','3','4','5','6','7','8','9','0', - '1','2','3','4','5','6','7','8','9','0', - '1','2','3','4','5','6','7','8','9','0', - '1','2'}, - json("12345678901234567890123456789012")); -} -TEST_CASE("encode_ubjson_arrays_and_maps") -{ - check_encode_ubjson({'[','#','U',0x00},json::array()); - check_encode_ubjson({'{','#','U',0x00},json::object()); - check_encode_ubjson({'[','#','U',0x01,'U',0x00},json::parse("[0]")); - check_encode_ubjson({'[','#','U',0x02,'U',0x00,'U',0x00},json::parse("[0,0]")); - check_encode_ubjson({'[','#','U',0x02, - '[','#','U',0x01,'U',0x00, - 'U',0x00},json::parse("[[0],0]")); - check_encode_ubjson({'[','#','U',0x01,'S','U',0x05,'H','e','l','l','o'},json::parse("[\"Hello\"]")); - check_encode_ubjson({'{','#','U',0x01,'U',0x02,'o','c','[','#','U',0x01,'U',0x00}, json::parse("{\"oc\": [0]}")); - check_encode_ubjson({'{','#','U',0x01,'U',0x02,'o','c','[','#','U',0x04,'U',0x00,'U',0x01,'U',0x02,'U',0x03}, json::parse("{\"oc\": [0,1,2,3]}")); -} - -TEST_CASE("encode indefinite length ubjson arrays and maps") -{ - std::vector v; - ubjson::ubjson_bytes_encoder encoder(v); - - SECTION("[\"Hello\"]") - { - encoder.begin_array(); - encoder.string_value("Hello"); - encoder.end_array(); - - check_encode_ubjson({'[','S','U',0x05,'H','e','l','l','o',']'}, v); - } - - SECTION("{\"oc\": [0]}") - { - encoder.begin_object(); - encoder.name("oc"); - encoder.begin_array(); - encoder.uint64_value(0); - encoder.end_array(); - encoder.end_object(); - - check_encode_ubjson({'{','U',0x02,'o','c','[','U',0x00,']','}'}, v); - } - - SECTION("{\"oc\": [0,1,2,3]}") - { - encoder.begin_object(); - encoder.name("oc"); - encoder.begin_array(); - encoder.uint64_value(0); - encoder.uint64_value(1); - encoder.uint64_value(2); - encoder.uint64_value(3); - encoder.end_array(); - encoder.end_object(); - - check_encode_ubjson({'{','U',0x02,'o','c','[','U',0x00,'U',0x01,'U',0x02,'U',0x03,']','}'}, v); - } -} - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/ubjson/ubjson_encoder_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/ubjson/ubjson_encoder_tests.cpp deleted file mode 100644 index 713008e528..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/ubjson/ubjson_encoder_tests.cpp +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2016 Daniel Parker -// Distributed under Boost license - -#if defined(_MSC_VER) -#include "windows.h" // test no inadvertant macro expansions -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; -using namespace jsoncons::ubjson; - -TEST_CASE("serialize array to ubjson") -{ - std::vector v; - ubjson::ubjson_bytes_encoder encoder(v); - encoder.begin_array(3); - encoder.bool_value(true); - encoder.bool_value(false); - encoder.null_value(); - encoder.end_array(); - encoder.flush(); - - try - { - json result = decode_ubjson(v); - std::cout << result << std::endl; - } - catch (const std::exception& e) - { - std::cout << e.what() << std::endl; - } -} - -TEST_CASE("Too many and too few items in UBJSON object or array") -{ - std::error_code ec{}; - std::vector v; - ubjson::ubjson_bytes_encoder encoder(v); - - SECTION("Too many items in array") - { - CHECK(encoder.begin_array(3)); - CHECK(encoder.bool_value(true)); - CHECK(encoder.bool_value(false)); - CHECK(encoder.null_value()); - CHECK(encoder.begin_array(2)); - CHECK(encoder.string_value("cat")); - CHECK(encoder.string_value("feline")); - CHECK(encoder.end_array()); - REQUIRE_THROWS_WITH(encoder.end_array(), ubjson_error_category_impl().message((int)ubjson_errc::too_many_items).c_str()); - encoder.flush(); - } - SECTION("Too few items in array") - { - CHECK(encoder.begin_array(5)); - CHECK(encoder.bool_value(true)); - CHECK(encoder.bool_value(false)); - CHECK(encoder.null_value()); - CHECK(encoder.begin_array(2)); - CHECK(encoder.string_value("cat")); - CHECK(encoder.string_value("feline")); - CHECK(encoder.end_array()); - REQUIRE_THROWS_WITH(encoder.end_array(), ubjson_error_category_impl().message((int)ubjson_errc::too_few_items).c_str()); - encoder.flush(); - } - SECTION("Too many items in object") - { - CHECK(encoder.begin_object(3)); - CHECK(encoder.name("a")); - CHECK(encoder.bool_value(true)); - CHECK(encoder.name("b")); - CHECK(encoder.bool_value(false)); - CHECK(encoder.name("c")); - CHECK(encoder.null_value()); - CHECK(encoder.name("d")); - CHECK(encoder.begin_array(2)); - CHECK(encoder.string_value("cat")); - CHECK(encoder.string_value("feline")); - CHECK(encoder.end_array()); - REQUIRE_THROWS_WITH(encoder.end_object(), ubjson_error_category_impl().message((int)ubjson_errc::too_many_items).c_str()); - encoder.flush(); - } - SECTION("Too few items in object") - { - CHECK(encoder.begin_object(5)); - CHECK(encoder.name("a")); - CHECK(encoder.bool_value(true)); - CHECK(encoder.name("b")); - CHECK(encoder.bool_value(false)); - CHECK(encoder.name("c")); - CHECK(encoder.null_value()); - CHECK(encoder.name("d")); - CHECK(encoder.begin_array(2)); - CHECK(encoder.string_value("cat")); - CHECK(encoder.string_value("feline")); - CHECK(encoder.end_array()); - REQUIRE_THROWS_WITH(encoder.end_object(), ubjson_error_category_impl().message((int)ubjson_errc::too_few_items).c_str()); - encoder.flush(); - } -} diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/unicode_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/unicode_tests.cpp deleted file mode 100644 index 252db14489..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/unicode_tests.cpp +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -#if 0 - -BOOST_AUTO_TEST_CASE( test_surrogate_pair ) -{ - std::string input = "[\"\\u8A73\\u7D30\\u95B2\\u89A7\\uD800\\uDC01\\u4E00\"]"; - json value = json::parse(input); - json_options options; - options.escape_all_non_ascii(true); - std::string output; - value.dump(output,options); - - CHECK(input == output); -} - -TEST_CASE("test_wide_surrogate_pair") -{ - std::wstring input = L"[\"\\u8A73\\u7D30\\u95B2\\u89A7\\uD800\\uDC01\\u4E00\"]"; - wjson value = wjson::parse(input); - wjson_options options; - options.escape_all_non_ascii(true); - std::wstring output; - value.dump(output,options); - - CHECK(input == output); -} - -BOOST_AUTO_TEST_CASE( test1 ) -{ - std::istringstream is("{\"unicode_string_1\":\"\\uD800\\uDC00\"}"); - - json root = json::parse(is); - CHECK(root.is_object()); - CHECK(root.is()); - - root["double_1"] = 10.0; - - json double_1 = root["double_1"]; - - BOOST_CHECK_CLOSE(double_1.as(), 10.0, 0.000001); - - BOOST_CHECK_CLOSE(double_1.as(), 10.0, 0.000001); - - json copy(root); -} -#endif - diff --git a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/wjson_tests.cpp b/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/wjson_tests.cpp deleted file mode 100644 index 0a4d24dcd2..0000000000 --- a/cpp/src/core/thirdparty/jsoncons-0.126.0/tests/src/wjson_tests.cpp +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2013 Daniel Parker -// Distributed under Boost license - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace jsoncons; - -TEST_CASE("test_wjson") -{ - wjson root; - root[L"field1"] = L"test"; - root[L"field2"] = 3.9; - root[L"field3"] = true; - - CHECK(root[L"field1"].as() == L"test"); - CHECK(root[L"field2"].as() == 3.9); - CHECK(root[L"field3"].as() == true); - - std::wstring s1 = root[L"field1"].as(); - CHECK(s1 == L"test"); -} - -TEST_CASE("test_wjson_escape_u") -{ - std::wstring input = L"[\"\\uABCD\"]"; - std::wistringstream is(input); - - wjson root = wjson::parse(is); - - std::wstring s = root[0].as(); - CHECK( s.length() == 1 ); - CHECK( s[0] == 0xABCD ); -} - -TEST_CASE("wjson serialization tests") -{ - jsoncons::wjson testBlock; - testBlock[L"foo"] = true; - testBlock[L"bar"] = false; - testBlock[L"baz"] = true; - std::wstring testStr; - testBlock.dump(testStr); - - CHECK(testStr == L"{\"bar\":false,\"baz\":true,\"foo\":true}"); -} - From 073a987d79d83248cd323cbc406fe22405810e42 Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 26 Sep 2019 17:08:18 +0800 Subject: [PATCH 117/217] MS-587 Count get wrong result after adding vectors and index built immediately Former-commit-id: 8e46b545f10a230f1f74c9529c23aabf2af56b40 --- cpp/CHANGELOG.md | 1 + cpp/src/db/DBImpl.cpp | 66 ++++++++++++++++++++--------- cpp/src/db/DBImpl.h | 4 ++ cpp/src/db/engine/EngineFactory.cpp | 2 +- cpp/src/db/meta/MySQLMetaImpl.cpp | 56 +++++++++++++++++++++++- cpp/src/db/meta/SqliteMetaImpl.cpp | 63 ++++++++++++++++++++++++--- 6 files changed, 163 insertions(+), 29 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 2612062263..477685cbc4 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -8,6 +8,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-568 - Fix gpuresource free error - MS-572 - Milvus crash when get SIGINT - MS-577 - Unittest Query randomly hung +- MS-587 - Count get wrong result after adding vectors and index built immediately ## Improvement - MS-552 - Add and change the easylogging library diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index a763c9199c..a4945ca02e 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -275,7 +275,11 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) } } - //step 3: wait and build index + //step 3: let merge file thread finish + //to avoid duplicate data bug + WaitMergeFileFinish(); + + //step 4: wait and build index //for IDMAP type, only wait all NEW file converted to RAW file //for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files std::vector file_types; @@ -470,12 +474,8 @@ void DBImpl::BackgroundTimerTask() { server::SystemInfo::GetInstance().Init(); while (true) { if (shutting_down_.load(std::memory_order_acquire)){ - for(auto& iter : compact_thread_results_) { - iter.wait(); - } - for(auto& iter : index_thread_results_) { - iter.wait(); - } + WaitMergeFileFinish(); + WaitBuildIndexFinish(); ENGINE_LOG_DEBUG << "DB background thread exit"; break; @@ -489,6 +489,20 @@ void DBImpl::BackgroundTimerTask() { } } +void DBImpl::WaitMergeFileFinish() { + std::lock_guard lck(compact_result_mutex_); + for(auto& iter : compact_thread_results_) { + iter.wait(); + } +} + +void DBImpl::WaitBuildIndexFinish() { + std::lock_guard lck(index_result_mutex_); + for(auto& iter : index_thread_results_) { + iter.wait(); + } +} + void DBImpl::StartMetricTask() { static uint64_t metric_clock_tick = 0; metric_clock_tick++; @@ -545,18 +559,24 @@ void DBImpl::StartCompactionTask() { MemSerialize(); //compactiong has been finished? - if(!compact_thread_results_.empty()) { - std::chrono::milliseconds span(10); - if (compact_thread_results_.back().wait_for(span) == std::future_status::ready) { - compact_thread_results_.pop_back(); + { + std::lock_guard lck(compact_result_mutex_); + if (!compact_thread_results_.empty()) { + std::chrono::milliseconds span(10); + if (compact_thread_results_.back().wait_for(span) == std::future_status::ready) { + compact_thread_results_.pop_back(); + } } } //add new compaction task - if(compact_thread_results_.empty()) { - compact_thread_results_.push_back( + { + std::lock_guard lck(compact_result_mutex_); + if (compact_thread_results_.empty()) { + compact_thread_results_.push_back( compact_thread_pool_.enqueue(&DBImpl::BackgroundCompaction, this, compact_table_ids_)); - compact_table_ids_.clear(); + compact_table_ids_.clear(); + } } } @@ -700,17 +720,23 @@ void DBImpl::StartBuildIndexTask(bool force) { } //build index has been finished? - if(!index_thread_results_.empty()) { - std::chrono::milliseconds span(10); - if (index_thread_results_.back().wait_for(span) == std::future_status::ready) { - index_thread_results_.pop_back(); + { + std::lock_guard lck(index_result_mutex_); + if (!index_thread_results_.empty()) { + std::chrono::milliseconds span(10); + if (index_thread_results_.back().wait_for(span) == std::future_status::ready) { + index_thread_results_.pop_back(); + } } } //add new build index task - if(index_thread_results_.empty()) { - index_thread_results_.push_back( + { + std::lock_guard lck(index_result_mutex_); + if (index_thread_results_.empty()) { + index_thread_results_.push_back( index_thread_pool_.enqueue(&DBImpl::BackgroundBuildIndex, this)); + } } } diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 24b7e47a02..bc9e453e5c 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -111,6 +111,8 @@ class DBImpl : public DB { QueryResults &results); void BackgroundTimerTask(); + void WaitMergeFileFinish(); + void WaitBuildIndexFinish(); void StartMetricTask(); @@ -140,10 +142,12 @@ class DBImpl : public DB { std::mutex mem_serialize_mutex_; ThreadPool compact_thread_pool_; + std::mutex compact_result_mutex_; std::list> compact_thread_results_; std::set compact_table_ids_; ThreadPool index_thread_pool_; + std::mutex index_result_mutex_; std::list> index_thread_results_; std::mutex build_index_mutex_; diff --git a/cpp/src/db/engine/EngineFactory.cpp b/cpp/src/db/engine/EngineFactory.cpp index d50cc8ad3e..28cfa97511 100644 --- a/cpp/src/db/engine/EngineFactory.cpp +++ b/cpp/src/db/engine/EngineFactory.cpp @@ -35,7 +35,7 @@ EngineFactory::Build(uint16_t dimension, return nullptr; } - ENGINE_LOG_DEBUG << "EngineFactory EngineTypee: " << (int)index_type; + ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int)index_type; ExecutionEnginePtr execution_engine_ptr = std::make_shared(dimension, location, index_type, metric_type, nlist); diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 3e54686ed0..48302fa099 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -373,6 +373,9 @@ Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, return HandleException("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", dropPartitionsByDatesQuery.error()); } } //Scoped Connection + + ENGINE_LOG_DEBUG << "Successfully drop partitions, table id = " << table_schema.table_id_; + } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES", e.what()); } @@ -443,6 +446,7 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { } } //Scoped Connection + ENGINE_LOG_DEBUG << "Successfully create table: " << table_schema.table_id_; return utils::CreateTablePath(options_, table_schema.table_id_); } catch (std::exception &e) { @@ -589,6 +593,8 @@ Status MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableI } //Scoped Connection + ENGINE_LOG_DEBUG << "Successfully update table index, table id = " << table_id; + } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE INDEX PARAM", e.what()); } @@ -621,6 +627,8 @@ Status MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) } //Scoped Connection + ENGINE_LOG_DEBUG << "Successfully update table flag, table id = " << table_id; + } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FLAG", e.what()); } @@ -725,6 +733,8 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { } //Scoped Connection + ENGINE_LOG_DEBUG << "Successfully drop table index, table id = " << table_id; + } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN DROPPING TABLE INDEX", e.what()); } @@ -762,6 +772,8 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { DeleteTableFiles(table_id); } + ENGINE_LOG_DEBUG << "Successfully delete table, table id = " << table_id; + } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN DELETING TABLE", e.what()); } @@ -795,6 +807,9 @@ Status MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { return HandleException("QUERY ERROR WHEN DELETING TABLE FILES", deleteTableFilesQuery.error()); } } //Scoped Connection + + ENGINE_LOG_DEBUG << "Successfully delete table files, table id = " << table_id; + } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN DELETING TABLE FILES", e.what()); } @@ -1001,6 +1016,7 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { } } // Scoped Connection + ENGINE_LOG_DEBUG << "Successfully create table file, file id = " << file_schema.file_id_; return utils::CreateTableFilePath(options_, file_schema); } catch (std::exception &e) { @@ -1082,6 +1098,9 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { files.push_back(table_file); } + if(res.size() > 0) { + ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-index files"; + } return ret; } catch (std::exception &e) { @@ -1195,6 +1214,9 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, files[table_file.date_].push_back(table_file); } + if(res.size() > 0) { + ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-search files"; + } return ret; } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH", e.what()); @@ -1285,6 +1307,9 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, files[table_file.date_].push_back(table_file); } + if(res.size() > 0) { + ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-merge files"; + } return ret; } catch (std::exception &e) { @@ -1369,6 +1394,7 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, table_files.emplace_back(file_schema); } + ENGINE_LOG_DEBUG << "Get table files by id"; return ret; } catch (std::exception &e) { @@ -1386,7 +1412,7 @@ Status MySQLMetaImpl::Archive() { for (auto &kv : criterias) { auto &criteria = kv.first; auto &limit = kv.second; - if (criteria == "days") { + if (criteria == engine::ARCHIVE_CONF_DAYS) { size_t usecs = limit * D_SEC * US_PS; long now = utils::GetMicroSecTimeStamp(); @@ -1410,16 +1436,20 @@ Status MySQLMetaImpl::Archive() { return HandleException("QUERY ERROR DURING ARCHIVE", archiveQuery.error()); } + ENGINE_LOG_DEBUG << "Archive old files"; + } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN DURING ARCHIVE", e.what()); } } - if (criteria == "disk") { + if (criteria == engine::ARCHIVE_CONF_DISK) { uint64_t sum = 0; Size(sum); auto to_delete = (sum - limit * G); DiscardFiles(to_delete); + + ENGINE_LOG_DEBUG << "Archive files to free disk"; } } @@ -1596,6 +1626,8 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { } } //Scoped Connection + ENGINE_LOG_DEBUG << "Update single table file, file id = " << file_schema.file_id_; + } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILE", e.what()); } @@ -1625,6 +1657,8 @@ Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { return HandleException("QUERY ERROR WHEN UPDATING TABLE FILE TO INDEX", updateTableFilesToIndexQuery.error()); } + ENGINE_LOG_DEBUG << "Update files to to_index, table id = " << table_id; + } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES TO INDEX", e.what()); } @@ -1705,6 +1739,8 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { } } //Scoped Connection + ENGINE_LOG_DEBUG << "Update " << files.size() << " table files"; + } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES", e.what()); } @@ -1782,6 +1818,11 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return HandleException("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", cleanUpFilesWithTTLQuery.error()); } } + + if(res.size() > 0) { + ENGINE_LOG_DEBUG << "Clean " << res.size() << " files deleted in " << seconds << " seconds"; + } + } //Scoped Connection } catch (std::exception &e) { @@ -1832,6 +1873,10 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return HandleException("QUERY ERROR WHEN CLEANING UP TABLES WITH TTL", cleanUpFilesWithTTLQuery.error()); } } + + if(res.size() > 0) { + ENGINE_LOG_DEBUG << "Remove " << res.size() << " tables from meta"; + } } //Scoped Connection } catch (std::exception &e) { @@ -1864,6 +1909,10 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { utils::DeleteTablePath(options_, table_id); } } + + if(table_ids.size() > 0) { + ENGINE_LOG_DEBUG << "Remove " << table_ids.size() << " tables folder"; + } } } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what()); @@ -1904,6 +1953,9 @@ Status MySQLMetaImpl::CleanUp() { } } + if(res.size() > 0) { + ENGINE_LOG_DEBUG << "Clean " << res.size() << " files"; + } } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN CLEANING UP FILES", e.what()); } diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index aa8f16f5a8..1fe7e3d76c 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -177,6 +177,9 @@ Status SqliteMetaImpl::DropPartitionsByDates(const std::string &table_id, c(&TableFileSchema::table_id_) == table_id and in(&TableFileSchema::date_, dates) )); + + ENGINE_LOG_DEBUG << "Successfully drop partitions, table id = " << table_schema.table_id_; + } catch (std::exception &e) { return HandleException("Encounter exception when drop partition", e.what()); } @@ -217,6 +220,8 @@ Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) { return HandleException("Encounter exception when create table", e.what()); } + ENGINE_LOG_DEBUG << "Successfully create table: " << table_schema.table_id_; + return utils::CreateTablePath(options_, table_schema.table_id_); } catch (std::exception &e) { @@ -241,6 +246,8 @@ Status SqliteMetaImpl::DeleteTable(const std::string& table_id) { c(&TableSchema::state_) != (int) TableSchema::TO_DELETE )); + ENGINE_LOG_DEBUG << "Successfully delete table, table id = " << table_id; + } catch (std::exception &e) { return HandleException("Encounter exception when delete table", e.what()); } @@ -266,6 +273,8 @@ Status SqliteMetaImpl::DeleteTableFiles(const std::string& table_id) { c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE )); + ENGINE_LOG_DEBUG << "Successfully delete table files, table id = " << table_id; + } catch (std::exception &e) { return HandleException("Encounter exception when delete table files", e.what()); } @@ -414,6 +423,8 @@ Status SqliteMetaImpl::UpdateTableIndex(const std::string &table_id, const Table c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP )); + ENGINE_LOG_DEBUG << "Successfully update table index, table id = " << table_id; + } catch (std::exception &e) { std::string msg = "Encounter exception when update table index: table_id = " + table_id; return HandleException(msg, e.what()); @@ -434,6 +445,7 @@ Status SqliteMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag where( c(&TableSchema::table_id_) == table_id )); + ENGINE_LOG_DEBUG << "Successfully update table flag, table id = " << table_id; } catch (std::exception &e) { std::string msg = "Encounter exception when update table flag: table_id = " + table_id; @@ -508,6 +520,8 @@ Status SqliteMetaImpl::DropTableIndex(const std::string &table_id) { c(&TableSchema::table_id_) == table_id )); + ENGINE_LOG_DEBUG << "Successfully drop table index, table id = " << table_id; + } catch (std::exception &e) { return HandleException("Encounter exception when delete table index files", e.what()); } @@ -603,6 +617,7 @@ Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { auto id = ConnectorPtr->insert(file_schema); file_schema.id_ = id; + ENGINE_LOG_DEBUG << "Successfully create table file, file id = " << file_schema.file_id_; return utils::CreateTableFilePath(options_, file_schema); } catch (std::exception& e) { @@ -666,6 +681,9 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { files.push_back(table_file); } + if(selected.size() > 0) { + ENGINE_LOG_DEBUG << "Collect " << selected.size() << " to-index files"; + } return ret; } catch (std::exception &e) { @@ -704,31 +722,31 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, auto status = DescribeTable(table_schema); if (!status.ok()) { return status; } - decltype(ConnectorPtr->select(select_columns)) result; + decltype(ConnectorPtr->select(select_columns)) selected; if (partition.empty() && ids.empty()) { auto filter = where(match_tableid and match_type); - result = ConnectorPtr->select(select_columns, filter); + selected = ConnectorPtr->select(select_columns, filter); } else if (partition.empty() && !ids.empty()) { auto match_fileid = in(&TableFileSchema::id_, ids); auto filter = where(match_tableid and match_fileid and match_type); - result = ConnectorPtr->select(select_columns, filter); + selected = ConnectorPtr->select(select_columns, filter); } else if (!partition.empty() && ids.empty()) { auto match_date = in(&TableFileSchema::date_, partition); auto filter = where(match_tableid and match_date and match_type); - result = ConnectorPtr->select(select_columns, filter); + selected = ConnectorPtr->select(select_columns, filter); } else if (!partition.empty() && !ids.empty()) { auto match_fileid = in(&TableFileSchema::id_, ids); auto match_date = in(&TableFileSchema::date_, partition); auto filter = where(match_tableid and match_fileid and match_date and match_type); - result = ConnectorPtr->select(select_columns, filter); + selected = ConnectorPtr->select(select_columns, filter); } Status ret; TableFileSchema table_file; - for (auto &file : result) { + for (auto &file : selected) { table_file.id_ = std::get<0>(file); table_file.table_id_ = std::get<1>(file); table_file.file_id_ = std::get<2>(file); @@ -757,6 +775,9 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, ENGINE_LOG_ERROR << "No file to search for table: " << table_id; } + if(selected.size() > 0) { + ENGINE_LOG_DEBUG << "Collect " << selected.size() << " to-search files"; + } return ret; } catch (std::exception &e) { @@ -824,6 +845,9 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, files[table_file.date_].push_back(table_file); } + if(selected.size() > 0) { + ENGINE_LOG_DEBUG << "Collect " << selected.size() << " to-merge files"; + } return result; } catch (std::exception &e) { @@ -878,6 +902,7 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, table_files.emplace_back(file_schema); } + ENGINE_LOG_DEBUG << "Get table files by id"; return result; } catch (std::exception &e) { return HandleException("Encounter exception when lookup table files", e.what()); @@ -912,6 +937,8 @@ Status SqliteMetaImpl::Archive() { } catch (std::exception &e) { return HandleException("Encounter exception when update table files", e.what()); } + + ENGINE_LOG_DEBUG << "Archive old files"; } if (criteria == engine::ARCHIVE_CONF_DISK) { uint64_t sum = 0; @@ -919,6 +946,8 @@ Status SqliteMetaImpl::Archive() { int64_t to_delete = (int64_t)sum - limit * G; DiscardFiles(to_delete); + + ENGINE_LOG_DEBUG << "Archive files to free disk"; } } @@ -1026,6 +1055,8 @@ Status SqliteMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { ConnectorPtr->update(file_schema); + ENGINE_LOG_DEBUG << "Update single table file, file id = " << file_schema.file_id_; + } catch (std::exception &e) { std::string msg = "Exception update table file: table_id = " + file_schema.table_id_ + " file_id = " + file_schema.file_id_; @@ -1049,6 +1080,9 @@ Status SqliteMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { c(&TableFileSchema::table_id_) == table_id and c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW )); + + ENGINE_LOG_DEBUG << "Update files to to_index, table id = " << table_id; + } catch (std::exception &e) { return HandleException("Encounter exception when update table files to to_index", e.what()); } @@ -1094,6 +1128,7 @@ Status SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { return HandleException("UpdateTableFiles error: sqlite transaction failed"); } + ENGINE_LOG_DEBUG << "Update " << files.size() << " table files"; } catch (std::exception &e) { return HandleException("Encounter exception when update table files", e.what()); } @@ -1143,6 +1178,10 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed"); } + if(files.size() > 0) { + ENGINE_LOG_DEBUG << "Clean " << files.size() << " files deleted in " << seconds << " seconds"; + } + } catch (std::exception &e) { return HandleException("Encounter exception when clean table files", e.what()); } @@ -1171,6 +1210,10 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed"); } + if(tables.size() > 0) { + ENGINE_LOG_DEBUG << "Remove " << tables.size() << " tables from meta"; + } + } catch (std::exception &e) { return HandleException("Encounter exception when clean table files", e.what()); } @@ -1188,6 +1231,10 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } } + if(table_ids.size() > 0) { + ENGINE_LOG_DEBUG << "Remove " << table_ids.size() << " tables folder"; + } + } catch (std::exception &e) { return HandleException("Encounter exception when delete table folder", e.what()); } @@ -1221,6 +1268,10 @@ Status SqliteMetaImpl::CleanUp() { return HandleException("CleanUp error: sqlite transaction failed"); } + if(files.size() > 0) { + ENGINE_LOG_DEBUG << "Clean " << files.size() << " files"; + } + } catch (std::exception &e) { return HandleException("Encounter exception when clean table file", e.what()); } From d0b888c01239fcb183b227d20845fec9340e981c Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Thu, 26 Sep 2019 18:49:32 +0800 Subject: [PATCH 118/217] MS-574 refine configuration names Former-commit-id: c8f29009c5dc6712a0fd67185e24d4f9ce1286d1 --- cpp/conf/server_config.template | 40 +++-- cpp/src/metrics/PrometheusMetrics.cpp | 2 +- cpp/src/server/Config.cpp | 218 +++++++++++++++++++------- cpp/src/server/Config.h | 51 +++--- cpp/src/server/DBWrapper.cpp | 4 +- cpp/src/server/Server.cpp | 6 + 6 files changed, 217 insertions(+), 104 deletions(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 6364698100..d80dfda958 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -1,39 +1,37 @@ +# All the following configurations are default values. + server_config: - address: 0.0.0.0 # milvus server ip address (IPv4) - port: 19530 # port range: 1025 ~ 65534 - mode: single # deployment type: single, cluster, read_only + address: 0.0.0.0 # milvus server ip address (IPv4) + port: 19530 # port range: 1025 ~ 65534 + deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable time_zone: UTC+8 db_config: - path: @MILVUS_DB_PATH@ # milvus database path - slave_path: # secondary database path, split by semicolon + primary_path: @MILVUS_DB_PATH@ # path used to store data and meta + secondary_path: # path used to store data only, split by semicolon - # URI format: dialect://username:password@host:port/database - # All parts except dialect are optional, but you MUST include the delimiters - # Currently dialect supports mysql or sqlite - backend_url: sqlite://:@:/ + backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database + # Keep 'dialect://:@:/', and replace other texts with real values. + # Replace 'dialect' with 'mysql' or 'sqlite' - archive_disk_threshold: 0 # GB, file will be archived when disk usage exceed, 0 for no limit - archive_days_threshold: 0 # DAYS, older files will be archived, 0 for no limit - buffer_size: 4 # GB, maximum insert buffer size allowed - build_index_gpu: 0 # gpu id used for building index + insert_buffer_size: 4 # GB, maximum insert buffer size allowed + build_index_gpu: 0 # gpu id used for building index metric_config: - auto_bootup: off # whether enable monitoring when bootup - collector: prometheus # prometheus + enable_monitor: false # enable monitoring or not + collector: prometheus # prometheus prometheus_config: - port: 8080 # port prometheus used to fetch metrics + port: 8080 # port prometheus used to fetch metrics cache_config: - cpu_mem_capacity: 16 # GB, CPU memory size used for cache - cpu_mem_threshold: 0.85 # percent of data kept when cache cleanup triggered - cache_insert_data: false # whether load data into cache when insert + cpu_mem_capacity: 16 # GB, CPU memory used for cache + cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered + cache_insert_data: false # whether load inserted data into cache engine_config: blas_threshold: 20 resource_config: - mode: simple - pool: + resource_pool: - cpu - gpu0 diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index b607dbb020..6f3513072a 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -31,7 +31,7 @@ ErrorCode PrometheusMetrics::Init() { try { Config &config = Config::GetInstance(); - Status s = config.GetMetricConfigAutoBootup(startup_); + Status s = config.GetMetricConfigEnableMonitor(startup_); if (!s.ok()) return s.code(); if (!startup_) return SERVER_SUCCESS; diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 8ee2f7c62b..3730ff6f68 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -70,6 +70,111 @@ Config::LoadConfigFile(const std::string &filename) { return Status::OK(); } +Status +Config::ValidateConfig() { + Status s; + + /* server config */ + std::string server_addr; + s = GetServerConfigAddress(server_addr); + if (!s.ok()) return s; + + std::string server_port; + s = GetServerConfigPort(server_port); + if (!s.ok()) return s; + + std::string server_mode; + s = GetServerConfigMode(server_mode); + if (!s.ok()) return s; + + std::string server_time_zone; + s = GetServerConfigTimeZone(server_time_zone); + if (!s.ok()) return s; + + /* db config */ + std::string db_primary_path; + s = GetDBConfigPrimaryPath(db_primary_path); + if (!s.ok()) return s; + + std::string db_secondary_path; + s = GetDBConfigSecondaryPath(db_secondary_path); + if (!s.ok()) return s; + + std::string db_backend_url; + s = GetDBConfigBackendUrl(db_backend_url); + if (!s.ok()) return s; + + int32_t db_archive_disk_threshold; + s = GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold); + if (!s.ok()) return s; + + int32_t db_archive_days_threshold; + s = GetDBConfigArchiveDaysThreshold(db_archive_days_threshold); + if (!s.ok()) return s; + + int32_t db_insert_buffer_size; + s = GetDBConfigInsertBufferSize(db_insert_buffer_size); + if (!s.ok()) return s; + + int32_t db_build_index_gpu; + s = GetDBConfigBuildIndexGPU(db_build_index_gpu); + if (!s.ok()) return s; + + /* metric config */ + bool metric_enable_monitor; + s = GetMetricConfigEnableMonitor(metric_enable_monitor); + if (!s.ok()) return s; + + std::string metric_collector; + s = GetMetricConfigCollector(metric_collector); + if (!s.ok()) return s; + + std::string metric_prometheus_port; + s = GetMetricConfigPrometheusPort(metric_prometheus_port); + if (!s.ok()) return s; + + /* cache config */ + int32_t cache_cpu_mem_capacity; + s = GetCacheConfigCpuMemCapacity(cache_cpu_mem_capacity); + if (!s.ok()) return s; + + float cache_cpu_mem_threshold; + s = GetCacheConfigCpuMemThreshold(cache_cpu_mem_threshold); + if (!s.ok()) return s; + + int32_t cache_gpu_mem_capacity; + s = GetCacheConfigGpuMemCapacity(cache_gpu_mem_capacity); + if (!s.ok()) return s; + + float cache_gpu_mem_threshold; + s = GetCacheConfigGpuMemThreshold(cache_gpu_mem_threshold); + if (!s.ok()) return s; + + bool cache_insert_data; + s = GetCacheConfigCacheInsertData(cache_insert_data); + if (!s.ok()) return s; + + /* engine config */ + int32_t engine_blas_threshold; + s = GetEngineConfigBlasThreshold(engine_blas_threshold); + if (!s.ok()) return s; + + int32_t engine_omp_thread_num; + s = GetEngineConfigOmpThreadNum(engine_omp_thread_num); + if (!s.ok()) return s; + + /* resource config */ + std::string resource_mode; + s = GetResourceConfigMode(resource_mode); + if (!s.ok()) return s; + + std::vector resource_pool; + s = GetResourceConfigPool(resource_pool); + if (!s.ok()) return s; + + return Status::OK(); +} + void Config::PrintConfigSection(const std::string& config_node_name) { std::cout << std::endl; @@ -115,8 +220,11 @@ Config::CheckServerConfigPort(const std::string &value) { Status Config::CheckServerConfigMode(const std::string &value) { - if (value != "single" && value != "cluster" && value != "read_only") { - return Status(SERVER_INVALID_ARGUMENT, "Invalid server config mode [single, cluster, read_only]: " + value); + if (value != "single" && + value != "cluster_readonly" && + value != "cluster_writable") { + return Status(SERVER_INVALID_ARGUMENT, + "Invalid server config mode [single, cluster_readonly, cluster_writable]: " + value); } return Status::OK(); } @@ -140,15 +248,15 @@ Config::CheckServerConfigTimeZone(const std::string &value) { } Status -Config::CheckDBConfigPath(const std::string &value) { +Config::CheckDBConfigPrimaryPath(const std::string &value) { if (value.empty()) { - return Status(SERVER_INVALID_ARGUMENT, "DB config path empty"); + return Status(SERVER_INVALID_ARGUMENT, "DB config primary_path empty"); } return Status::OK(); } Status -Config::CheckDBConfigSlavePath(const std::string &value) { +Config::CheckDBConfigSecondaryPath(const std::string &value) { return Status::OK(); } @@ -177,15 +285,15 @@ Config::CheckDBConfigArchiveDaysThreshold(const std::string &value) { } Status -Config::CheckDBConfigBufferSize(const std::string &value) { +Config::CheckDBConfigInsertBufferSize(const std::string &value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config buffer_size: " + value); + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config insert_buffer_size: " + value); } else { int64_t buffer_size = std::stoi(value) * GB; unsigned long total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (buffer_size >= total_mem) { - return Status(SERVER_INVALID_ARGUMENT, "DB config buffer_size exceed system memory: " + value); + return Status(SERVER_INVALID_ARGUMENT, "DB config insert_buffer_size exceed system memory: " + value); } } return Status::OK(); @@ -205,7 +313,7 @@ Config::CheckDBConfigBuildIndexGPU(const std::string &value) { } Status -Config::CheckMetricConfigAutoBootup(const std::string& value) { +Config::CheckMetricConfigEnableMonitor(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config auto_bootup: " + value); } @@ -242,10 +350,10 @@ Config::CheckCacheConfigCpuMemCapacity(const std::string& value) { std::cerr << "Warning: cpu_mem_capacity value is too big" << std::endl; } - int32_t buffer_size; - Status s = GetDBConfigBufferSize(buffer_size); + int32_t buffer_value; + Status s = GetDBConfigInsertBufferSize(buffer_value); if (!s.ok()) return s; - int64_t insert_buffer_size = buffer_size * GB; + int64_t insert_buffer_size = buffer_value * GB; if (insert_buffer_size + cpu_cache_capacity >= total_mem) { return Status(SERVER_INVALID_ARGUMENT, "Sum of cpu_mem_capacity and buffer_size exceed system memory"); } @@ -427,23 +535,23 @@ Config::GetServerConfigStrTimeZone() { //////////////////////////////////////////////////////////////////////////////// /* db config */ std::string -Config::GetDBConfigStrPath() { +Config::GetDBConfigStrPrimaryPath() { std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PATH, - CONFIG_DB_PATH_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value).ok()) { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PRIMARY_PATH, + CONFIG_DB_PRIMARY_PATH_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); } return value; } std::string -Config::GetDBConfigStrSlavePath() { +Config::GetDBConfigStrSecondaryPath() { std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SLAVE_PATH, - CONFIG_DB_SLAVE_PATH_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value).ok()) { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SECONDARY_PATH, + CONFIG_DB_SECONDARY_PATH_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); } return value; } @@ -482,12 +590,12 @@ Config::GetDBConfigStrArchiveDaysThreshold() { } std::string -Config::GetDBConfigStrBufferSize() { +Config::GetDBConfigStrInsertBufferSize() { std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUFFER_SIZE, - CONFIG_DB_BUFFER_SIZE_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value).ok()) { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_INSERT_BUFFER_SIZE, + CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); } return value; } @@ -506,12 +614,12 @@ Config::GetDBConfigStrBuildIndexGPU() { //////////////////////////////////////////////////////////////////////////////// /* metric config */ std::string -Config::GetMetricConfigStrAutoBootup() { +Config::GetMetricConfigStrEnableMonitor() { std::string value; - if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value).ok()) { - value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_AUTO_BOOTUP, - CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value); + if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value).ok()) { + value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_ENABLE_MONITOR, + CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value); } return value; } @@ -659,14 +767,14 @@ Config::GetServerConfigTimeZone(std::string& value) { } Status -Config::GetDBConfigPath(std::string& value) { - value = GetDBConfigStrPath(); - return CheckDBConfigPath(value); +Config::GetDBConfigPrimaryPath(std::string& value) { + value = GetDBConfigStrPrimaryPath(); + return CheckDBConfigPrimaryPath(value); } Status -Config::GetDBConfigSlavePath(std::string& value) { - value = GetDBConfigStrSlavePath(); +Config::GetDBConfigSecondaryPath(std::string& value) { + value = GetDBConfigStrSecondaryPath(); return Status::OK(); } @@ -695,9 +803,9 @@ Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { } Status -Config::GetDBConfigBufferSize(int32_t& value) { - std::string str = GetDBConfigStrBufferSize(); - Status s = CheckDBConfigBufferSize(str); +Config::GetDBConfigInsertBufferSize(int32_t& value) { + std::string str = GetDBConfigStrInsertBufferSize(); + Status s = CheckDBConfigInsertBufferSize(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -713,9 +821,9 @@ Config::GetDBConfigBuildIndexGPU(int32_t& value) { } Status -Config::GetMetricConfigAutoBootup(bool& value) { - std::string str = GetMetricConfigStrAutoBootup(); - Status s = CheckMetricConfigPrometheusPort(str); +Config::GetMetricConfigEnableMonitor(bool& value) { + std::string str = GetMetricConfigStrEnableMonitor(); + Status s = CheckMetricConfigEnableMonitor(str); if (!s.ok()) return s; std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); @@ -847,18 +955,18 @@ Config::SetServerConfigTimeZone(const std::string& value) { /* db config */ Status -Config::SetDBConfigPath(const std::string& value) { - Status s = CheckDBConfigPath(value); +Config::SetDBConfigPrimaryPath(const std::string& value) { + Status s = CheckDBConfigPrimaryPath(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); return Status::OK(); } Status -Config::SetDBConfigSlavePath(const std::string& value) { - Status s = CheckDBConfigSlavePath(value); +Config::SetDBConfigSecondaryPath(const std::string& value) { + Status s = CheckDBConfigSecondaryPath(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); return Status::OK(); } @@ -887,10 +995,10 @@ Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { } Status -Config::SetDBConfigBufferSize(const std::string& value) { - Status s = CheckDBConfigBufferSize(value); +Config::SetDBConfigInsertBufferSize(const std::string& value) { + Status s = CheckDBConfigInsertBufferSize(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); return Status::OK(); } @@ -904,10 +1012,10 @@ Config::SetDBConfigBuildIndexGPU(const std::string& value) { /* metric config */ Status -Config::SetMetricConfigAutoBootup(const std::string& value) { - Status s = CheckMetricConfigAutoBootup(value); +Config::SetMetricConfigEnableMonitor(const std::string& value) { + Status s = CheckMetricConfigEnableMonitor(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_AUTO_BOOTUP, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_ENABLE_MONITOR, value); return Status::OK(); } diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index 956c08100f..710230eb0a 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -41,18 +41,18 @@ static const char* CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; /* db config */ static const char* CONFIG_DB = "db_config"; -static const char* CONFIG_DB_PATH = "path"; -static const char* CONFIG_DB_PATH_DEFAULT = "/tmp/milvus"; -static const char* CONFIG_DB_SLAVE_PATH = "slave_path"; -static const char* CONFIG_DB_SLAVE_PATH_DEFAULT = ""; +static const char* CONFIG_DB_PRIMARY_PATH = "primary_path"; +static const char* CONFIG_DB_PRIMARY_PATH_DEFAULT = "/tmp/milvus"; +static const char* CONFIG_DB_SECONDARY_PATH = "secondary_path"; +static const char* CONFIG_DB_SECONDARY_PATH_DEFAULT = ""; static const char* CONFIG_DB_BACKEND_URL = "backend_url"; static const char* CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/"; static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold"; static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0"; static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold"; static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; -static const char* CONFIG_DB_BUFFER_SIZE = "buffer_size"; -static const char* CONFIG_DB_BUFFER_SIZE_DEFAULT = "4"; +static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; +static const char* CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4"; static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu"; static const char* CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0"; @@ -71,8 +71,8 @@ static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; /* metric config */ static const char* CONFIG_METRIC = "metric_config"; -static const char* CONFIG_METRIC_AUTO_BOOTUP = "auto_bootup"; -static const char* CONFIG_METRIC_AUTO_BOOTUP_DEFAULT = "false"; +static const char* CONFIG_METRIC_ENABLE_MONITOR = "enable_monitor"; +static const char* CONFIG_METRIC_ENABLE_MONITOR_DEFAULT = "false"; static const char* CONFIG_METRIC_COLLECTOR = "collector"; static const char* CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus"; static const char* CONFIG_METRIC_PROMETHEUS = "prometheus_config"; @@ -90,13 +90,14 @@ static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; static const char* CONFIG_RESOURCE = "resource_config"; static const char* CONFIG_RESOURCE_MODE = "mode"; static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; -static const char* CONFIG_RESOURCE_POOL = "pool"; +static const char* CONFIG_RESOURCE_POOL = "resource_pool"; class Config { public: static Config& GetInstance(); Status LoadConfigFile(const std::string& filename); + Status ValidateConfig(); void PrintAll(); private: @@ -120,16 +121,16 @@ class Config { Status CheckServerConfigTimeZone(const std::string& value); /* db config */ - Status CheckDBConfigPath(const std::string& value); - Status CheckDBConfigSlavePath(const std::string& value); + Status CheckDBConfigPrimaryPath(const std::string& value); + Status CheckDBConfigSecondaryPath(const std::string& value); Status CheckDBConfigBackendUrl(const std::string& value); Status CheckDBConfigArchiveDiskThreshold(const std::string& value); Status CheckDBConfigArchiveDaysThreshold(const std::string& value); - Status CheckDBConfigBufferSize(const std::string& value); + Status CheckDBConfigInsertBufferSize(const std::string& value); Status CheckDBConfigBuildIndexGPU(const std::string& value); /* metric config */ - Status CheckMetricConfigAutoBootup(const std::string& value); + Status CheckMetricConfigEnableMonitor(const std::string& value); Status CheckMetricConfigCollector(const std::string& value); Status CheckMetricConfigPrometheusPort(const std::string& value); @@ -156,16 +157,16 @@ class Config { std::string GetServerConfigStrTimeZone(); /* db config */ - std::string GetDBConfigStrPath(); - std::string GetDBConfigStrSlavePath(); + std::string GetDBConfigStrPrimaryPath(); + std::string GetDBConfigStrSecondaryPath(); std::string GetDBConfigStrBackendUrl(); std::string GetDBConfigStrArchiveDiskThreshold(); std::string GetDBConfigStrArchiveDaysThreshold(); - std::string GetDBConfigStrBufferSize(); + std::string GetDBConfigStrInsertBufferSize(); std::string GetDBConfigStrBuildIndexGPU(); /* metric config */ - std::string GetMetricConfigStrAutoBootup(); + std::string GetMetricConfigStrEnableMonitor(); std::string GetMetricConfigStrCollector(); std::string GetMetricConfigStrPrometheusPort(); @@ -191,16 +192,16 @@ class Config { Status GetServerConfigTimeZone(std::string& value); /* db config */ - Status GetDBConfigPath(std::string& value); - Status GetDBConfigSlavePath(std::string& value); + Status GetDBConfigPrimaryPath(std::string& value); + Status GetDBConfigSecondaryPath(std::string& value); Status GetDBConfigBackendUrl(std::string& value); Status GetDBConfigArchiveDiskThreshold(int32_t& value); Status GetDBConfigArchiveDaysThreshold(int32_t& value); - Status GetDBConfigBufferSize(int32_t& value); + Status GetDBConfigInsertBufferSize(int32_t& value); Status GetDBConfigBuildIndexGPU(int32_t& value); /* metric config */ - Status GetMetricConfigAutoBootup(bool& value); + Status GetMetricConfigEnableMonitor(bool& value); Status GetMetricConfigCollector(std::string& value); Status GetMetricConfigPrometheusPort(std::string& value); @@ -227,16 +228,16 @@ class Config { Status SetServerConfigTimeZone(const std::string& value); /* db config */ - Status SetDBConfigPath(const std::string& value); - Status SetDBConfigSlavePath(const std::string& value); + Status SetDBConfigPrimaryPath(const std::string& value); + Status SetDBConfigSecondaryPath(const std::string& value); Status SetDBConfigBackendUrl(const std::string& value); Status SetDBConfigArchiveDiskThreshold(const std::string& value); Status SetDBConfigArchiveDaysThreshold(const std::string& value); - Status SetDBConfigBufferSize(const std::string& value); + Status SetDBConfigInsertBufferSize(const std::string& value); Status SetDBConfigBuildIndexGPU(const std::string& value); /* metric config */ - Status SetMetricConfigAutoBootup(const std::string& value); + Status SetMetricConfigEnableMonitor(const std::string& value); Status SetMetricConfigCollector(const std::string& value); Status SetMetricConfigPrometheusPort(const std::string& value); diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 16b13a1249..745965dc6a 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -44,13 +44,13 @@ Status DBWrapper::StartService() { if (!s.ok()) return s; std::string path; - s = config.GetDBConfigPath(path); + s = config.GetDBConfigPrimaryPath(path); if (!s.ok()) return s; opt.meta_.path_ = path + "/db"; std::string db_slave_path; - s = config.GetDBConfigSlavePath(db_slave_path); + s = config.GetDBConfigSecondaryPath(db_slave_path); if (!s.ok()) return s; StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta_.slave_paths_); diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 57081845a5..87d3b374b2 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -241,6 +241,12 @@ Server::LoadConfig() { std::cerr << "Failed to load config file: " << config_filename_ << std::endl; exit(0); } + + s = config.ValidateConfig(); + if (!s.ok()) { + std::cerr << "Config check fail: " << s.message() << std::endl; + exit(0); + } return SERVER_SUCCESS; } From 4c91d328b1c2b0d4bc2c2ac0f8c3be8dc4818bbf Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 26 Sep 2019 18:53:50 +0800 Subject: [PATCH 119/217] format wrapper code Former-commit-id: 04af0f6d6e66d1fdcd01fc8de0cf93000793d5db --- cpp/src/main.cpp | 5 +-- cpp/src/wrapper/DataTransfer.cpp | 47 +++++++++++----------- cpp/src/wrapper/DataTransfer.h | 9 ++--- cpp/src/wrapper/KnowhereResource.cpp | 24 +++++++----- cpp/src/wrapper/KnowhereResource.h | 9 ++--- cpp/src/wrapper/VecImpl.cpp | 58 +++++++++++++--------------- cpp/src/wrapper/VecImpl.h | 39 +++++++++++-------- cpp/src/wrapper/VecIndex.cpp | 24 +++++------- cpp/src/wrapper/VecIndex.h | 23 ++++++----- 9 files changed, 117 insertions(+), 121 deletions(-) diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index dc17733a8b..6d28a3bd67 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -27,15 +27,13 @@ #include "utils/CommonUtil.h" #include "metrics/Metrics.h" #include "server/Server.h" -#include "version.h" +#include "../version.h" INITIALIZE_EASYLOGGINGPP void print_help(const std::string &app_name); -using namespace zilliz::milvus; - int main(int argc, char *argv[]) { std::cout << std::endl << "Welcome to use Milvus by Zilliz!" << std::endl; @@ -103,6 +101,7 @@ main(int argc, char *argv[]) { } /* Handle Signal */ + using ::zilliz::milvus; signal(SIGHUP, server::SignalUtil::HandleSignal); signal(SIGINT, server::SignalUtil::HandleSignal); signal(SIGUSR1, server::SignalUtil::HandleSignal); diff --git a/cpp/src/wrapper/DataTransfer.cpp b/cpp/src/wrapper/DataTransfer.cpp index 294a3c20ab..c6603b1991 100644 --- a/cpp/src/wrapper/DataTransfer.cpp +++ b/cpp/src/wrapper/DataTransfer.cpp @@ -16,45 +16,46 @@ // under the License. -#include "DataTransfer.h" +#include "wrapper/DataTransfer.h" +#include +#include +#include namespace zilliz { namespace milvus { namespace engine { -using namespace zilliz::knowhere; - -DatasetPtr -GenDatasetWithIds(const int64_t &nb, const int64_t &dim, const float *xb, const long *ids) { +knowhere::DatasetPtr +GenDatasetWithIds(const int64_t &nb, const int64_t &dim, const float *xb, const int64_t *ids) { std::vector shape{nb, dim}; - auto tensor = ConstructFloatTensor((uint8_t *) xb, nb * dim * sizeof(float), shape); - std::vector tensors{tensor}; - std::vector tensor_fields{ConstructFloatField("data")}; - auto tensor_schema = std::make_shared(tensor_fields); + auto tensor = knowhere::ConstructFloatTensor((uint8_t *) xb, nb * dim * sizeof(float), shape); + std::vector tensors{tensor}; + std::vector tensor_fields{knowhere::ConstructFloatField("data")}; + auto tensor_schema = std::make_shared(tensor_fields); - auto id_array = ConstructInt64Array((uint8_t *) ids, nb * sizeof(int64_t)); - std::vector arrays{id_array}; - std::vector array_fields{ConstructInt64Field("id")}; - auto array_schema = std::make_shared(tensor_fields); + auto id_array = knowhere::ConstructInt64Array((uint8_t *) ids, nb * sizeof(int64_t)); + std::vector arrays{id_array}; + std::vector array_fields{knowhere::ConstructInt64Field("id")}; + auto array_schema = std::make_shared(tensor_fields); - auto dataset = std::make_shared(std::move(arrays), array_schema, + auto dataset = std::make_shared(std::move(arrays), array_schema, std::move(tensors), tensor_schema); return dataset; } -DatasetPtr +knowhere::DatasetPtr GenDataset(const int64_t &nb, const int64_t &dim, const float *xb) { std::vector shape{nb, dim}; - auto tensor = ConstructFloatTensor((uint8_t *) xb, nb * dim * sizeof(float), shape); - std::vector tensors{tensor}; - std::vector tensor_fields{ConstructFloatField("data")}; - auto tensor_schema = std::make_shared(tensor_fields); + auto tensor = knowhere::ConstructFloatTensor((uint8_t *) xb, nb * dim * sizeof(float), shape); + std::vector tensors{tensor}; + std::vector tensor_fields{knowhere::ConstructFloatField("data")}; + auto tensor_schema = std::make_shared(tensor_fields); - auto dataset = std::make_shared(std::move(tensors), tensor_schema); + auto dataset = std::make_shared(std::move(tensors), tensor_schema); return dataset; } -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/DataTransfer.h b/cpp/src/wrapper/DataTransfer.h index 10aaf355c9..070adca055 100644 --- a/cpp/src/wrapper/DataTransfer.h +++ b/cpp/src/wrapper/DataTransfer.h @@ -20,17 +20,16 @@ #include "knowhere/adapter/Structure.h" - namespace zilliz { namespace milvus { namespace engine { extern zilliz::knowhere::DatasetPtr -GenDatasetWithIds(const int64_t &nb, const int64_t &dim, const float *xb, const long *ids); +GenDatasetWithIds(const int64_t &nb, const int64_t &dim, const float *xb, const int64_t *ids); extern zilliz::knowhere::DatasetPtr GenDataset(const int64_t &nb, const int64_t &dim, const float *xb); -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/KnowhereResource.cpp b/cpp/src/wrapper/KnowhereResource.cpp index 8ba38a77b0..b46acb1a4a 100644 --- a/cpp/src/wrapper/KnowhereResource.cpp +++ b/cpp/src/wrapper/KnowhereResource.cpp @@ -16,11 +16,15 @@ // under the License. -#include "KnowhereResource.h" +#include "wrapper/KnowhereResource.h" #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #include "server/Config.h" #include +#include +#include +#include +#include namespace zilliz { namespace milvus { @@ -31,20 +35,20 @@ constexpr int64_t M_BYTE = 1024 * 1024; Status KnowhereResource::Initialize() { struct GpuResourceSetting { - int64_t pinned_memory = 300*M_BYTE; - int64_t temp_memory = 300*M_BYTE; + int64_t pinned_memory = 300 * M_BYTE; + int64_t temp_memory = 300 * M_BYTE; int64_t resource_num = 2; }; - using GpuResourcesArray = std::map; + using GpuResourcesArray = std::map; GpuResourcesArray gpu_resources; Status s; //get build index gpu resource - server::Config& config = server::Config::GetInstance(); + server::Config &config = server::Config::GetInstance(); int32_t build_index_gpu; s = config.GetDBConfigBuildIndexGPU(build_index_gpu); - if (!s.ok()) return s; + if (!s.ok()) return s; gpu_resources.insert(std::make_pair(build_index_gpu, GpuResourceSetting())); @@ -64,7 +68,7 @@ KnowhereResource::Initialize() { } //init gpu resources - for(auto iter = gpu_resources.begin(); iter != gpu_resources.end(); ++iter) { + for (auto iter = gpu_resources.begin(); iter != gpu_resources.end(); ++iter) { knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(iter->first, iter->second.pinned_memory, iter->second.temp_memory, @@ -80,6 +84,6 @@ KnowhereResource::Finalize() { return Status::OK(); } -} -} -} \ No newline at end of file +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/KnowhereResource.h b/cpp/src/wrapper/KnowhereResource.h index 9e8c27b5af..a8726f9542 100644 --- a/cpp/src/wrapper/KnowhereResource.h +++ b/cpp/src/wrapper/KnowhereResource.h @@ -25,7 +25,7 @@ namespace milvus { namespace engine { class KnowhereResource { -public: + public: static Status Initialize(); @@ -33,7 +33,6 @@ public: Finalize(); }; - -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/VecImpl.cpp b/cpp/src/wrapper/VecImpl.cpp index 24f6a2bb0e..bbc064aee5 100644 --- a/cpp/src/wrapper/VecImpl.cpp +++ b/cpp/src/wrapper/VecImpl.cpp @@ -16,27 +16,24 @@ // under the License. +#include "wrapper/VecImpl.h" #include "utils/Log.h" #include "knowhere/index/vector_index/IndexIDMAP.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/helpers/Cloner.h" -#include "VecImpl.h" #include "DataTransfer.h" - namespace zilliz { namespace milvus { namespace engine { -using namespace zilliz::knowhere; - Status -VecIndexImpl::BuildAll(const long &nb, +VecIndexImpl::BuildAll(const int64_t &nb, const float *xb, - const long *ids, + const int64_t *ids, const Config &cfg, - const long &nt, + const int64_t &nt, const float *xt) { try { dim = cfg["dim"].as(); @@ -47,7 +44,7 @@ VecIndexImpl::BuildAll(const long &nb, auto model = index_->Train(dataset, cfg); index_->set_index_model(model); index_->Add(dataset, cfg); - } catch (KnowhereException &e) { + } catch (knowhere::KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); } catch (jsoncons::json_exception &e) { @@ -61,12 +58,12 @@ VecIndexImpl::BuildAll(const long &nb, } Status -VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const Config &cfg) { +VecIndexImpl::Add(const int64_t &nb, const float *xb, const int64_t *ids, const Config &cfg) { try { auto dataset = GenDatasetWithIds(nb, dim, xb, ids); index_->Add(dataset, cfg); - } catch (KnowhereException &e) { + } catch (knowhere::KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); } catch (jsoncons::json_exception &e) { @@ -80,7 +77,7 @@ VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const Config } Status -VecIndexImpl::Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) { +VecIndexImpl::Search(const int64_t &nq, const float *xq, float *dist, int64_t *ids, const Config &cfg) { try { auto k = cfg["k"].as(); auto dataset = GenDataset(nq, dim, xq); @@ -116,8 +113,7 @@ VecIndexImpl::Search(const long &nq, const float *xq, float *dist, long *ids, co // TODO(linxj): avoid copy here. memcpy(ids, p_ids, sizeof(int64_t) * nq * k); memcpy(dist, p_dist, sizeof(float) * nq * k); - - } catch (KnowhereException &e) { + } catch (knowhere::KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); } catch (jsoncons::json_exception &e) { @@ -186,7 +182,7 @@ VecIndexImpl::Clone() { int64_t VecIndexImpl::GetDeviceId() { - if (auto device_idx = std::dynamic_pointer_cast(index_)) { + if (auto device_idx = std::dynamic_pointer_cast(index_)) { return device_idx->GetGpuDevice(); } // else @@ -195,22 +191,22 @@ VecIndexImpl::GetDeviceId() { float * BFIndex::GetRawVectors() { - auto raw_index = std::dynamic_pointer_cast(index_); + auto raw_index = std::dynamic_pointer_cast(index_); if (raw_index) { return raw_index->GetRawVectors(); } return nullptr; } int64_t * BFIndex::GetRawIds() { - return std::static_pointer_cast(index_)->GetRawIds(); + return std::static_pointer_cast(index_)->GetRawIds(); } ErrorCode BFIndex::Build(const Config &cfg) { try { dim = cfg["dim"].as(); - std::static_pointer_cast(index_)->Train(cfg); - } catch (KnowhereException &e) { + std::static_pointer_cast(index_)->Train(cfg); + } catch (knowhere::KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return KNOWHERE_UNEXPECTED_ERROR; } catch (jsoncons::json_exception &e) { @@ -224,19 +220,19 @@ BFIndex::Build(const Config &cfg) { } Status -BFIndex::BuildAll(const long &nb, +BFIndex::BuildAll(const int64_t &nb, const float *xb, - const long *ids, + const int64_t *ids, const Config &cfg, - const long &nt, + const int64_t &nt, const float *xt) { try { dim = cfg["dim"].as(); auto dataset = GenDatasetWithIds(nb, dim, xb, ids); - std::static_pointer_cast(index_)->Train(cfg); + std::static_pointer_cast(index_)->Train(cfg); index_->Add(dataset, cfg); - } catch (KnowhereException &e) { + } catch (knowhere::KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); } catch (jsoncons::json_exception &e) { @@ -251,11 +247,11 @@ BFIndex::BuildAll(const long &nb, // TODO(linxj): add lock here. Status -IVFMixIndex::BuildAll(const long &nb, +IVFMixIndex::BuildAll(const int64_t &nb, const float *xb, - const long *ids, + const int64_t *ids, const Config &cfg, - const long &nt, + const int64_t &nt, const float *xt) { try { dim = cfg["dim"].as(); @@ -267,7 +263,7 @@ IVFMixIndex::BuildAll(const long &nb, index_->set_index_model(model); index_->Add(dataset, cfg); - if (auto device_index = std::dynamic_pointer_cast(index_)) { + if (auto device_index = std::dynamic_pointer_cast(index_)) { auto host_index = device_index->CopyGpuToCpu(Config()); index_ = host_index; type = ConvertToCpuIndexType(type); @@ -275,7 +271,7 @@ IVFMixIndex::BuildAll(const long &nb, WRAPPER_LOG_ERROR << "Build IVFMIXIndex Failed"; return Status(KNOWHERE_ERROR, "Build IVFMIXIndex Failed"); } - } catch (KnowhereException &e) { + } catch (knowhere::KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); } catch (jsoncons::json_exception &e) { @@ -296,6 +292,6 @@ IVFMixIndex::Load(const zilliz::knowhere::BinarySet &index_binary) { return Status::OK(); } -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/VecImpl.h b/cpp/src/wrapper/VecImpl.h index c111eb75b4..a91b24562b 100644 --- a/cpp/src/wrapper/VecImpl.h +++ b/cpp/src/wrapper/VecImpl.h @@ -21,6 +21,8 @@ #include "knowhere/index/vector_index/VectorIndex.h" #include "VecIndex.h" +#include +#include namespace zilliz { namespace milvus { @@ -29,14 +31,15 @@ namespace engine { class VecIndexImpl : public VecIndex { public: explicit VecIndexImpl(std::shared_ptr index, const IndexType &type) - : index_(std::move(index)), type(type) {}; + : index_(std::move(index)), type(type) { + } Status - BuildAll(const long &nb, + BuildAll(const int64_t &nb, const float *xb, - const long *ids, + const int64_t *ids, const Config &cfg, - const long &nt, + const int64_t &nt, const float *xt) override; VecIndexPtr @@ -55,7 +58,7 @@ class VecIndexImpl : public VecIndex { Count() override; Status - Add(const long &nb, const float *xb, const long *ids, const Config &cfg) override; + Add(const int64_t &nb, const float *xb, const int64_t *ids, const Config &cfg) override; zilliz::knowhere::BinarySet Serialize() override; @@ -70,7 +73,7 @@ class VecIndexImpl : public VecIndex { GetDeviceId() override; Status - Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) override; + Search(const int64_t &nq, const float *xq, float *dist, int64_t *ids, const Config &cfg) override; protected: int64_t dim = 0; @@ -83,14 +86,15 @@ class VecIndexImpl : public VecIndex { class IVFMixIndex : public VecIndexImpl { public: explicit IVFMixIndex(std::shared_ptr index, const IndexType &type) - : VecIndexImpl(std::move(index), type) {}; + : VecIndexImpl(std::move(index), type) { + } Status - BuildAll(const long &nb, + BuildAll(const int64_t &nb, const float *xb, - const long *ids, + const int64_t *ids, const Config &cfg, - const long &nt, + const int64_t &nt, const float *xt) override; Status @@ -100,7 +104,8 @@ class IVFMixIndex : public VecIndexImpl { class BFIndex : public VecIndexImpl { public: explicit BFIndex(std::shared_ptr index) : VecIndexImpl(std::move(index), - IndexType::FAISS_IDMAP) {}; + IndexType::FAISS_IDMAP) { + } ErrorCode Build(const Config &cfg); @@ -109,17 +114,17 @@ class BFIndex : public VecIndexImpl { GetRawVectors(); Status - BuildAll(const long &nb, + BuildAll(const int64_t &nb, const float *xb, - const long *ids, + const int64_t *ids, const Config &cfg, - const long &nt, + const int64_t &nt, const float *xt) override; int64_t * GetRawIds(); }; -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/VecIndex.cpp b/cpp/src/wrapper/VecIndex.cpp index cd8843e5ac..c9fdbf8161 100644 --- a/cpp/src/wrapper/VecIndex.cpp +++ b/cpp/src/wrapper/VecIndex.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#include "wrapper/VecIndex.h" #include "knowhere/index/vector_index/IndexIVF.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" #include "knowhere/index/vector_index/IndexIVFSQ.h" @@ -25,13 +26,11 @@ #include "knowhere/index/vector_index/IndexKDT.h" #include "knowhere/index/vector_index/IndexNSG.h" #include "knowhere/common/Exception.h" -#include "VecIndex.h" #include "VecImpl.h" #include "utils/Log.h" #include - namespace zilliz { namespace milvus { namespace engine { @@ -42,7 +41,7 @@ struct FileIOReader { std::fstream fs; std::string name; - FileIOReader(const std::string &fname); + explicit FileIOReader(const std::string &fname); ~FileIOReader(); @@ -72,13 +71,11 @@ FileIOReader::operator()(void *ptr, size_t size, size_t pos) { return 0; } - - struct FileIOWriter { std::fstream fs; std::string name; - FileIOWriter(const std::string &fname); + explicit FileIOWriter(const std::string &fname); ~FileIOWriter(); size_t operator()(void *ptr, size_t size); }; @@ -97,7 +94,6 @@ FileIOWriter::operator()(void *ptr, size_t size) { fs.write(reinterpret_cast(ptr), size); } - VecIndexPtr GetVecIndexFactory(const IndexType &type, const Config &cfg) { std::shared_ptr index; @@ -215,7 +211,7 @@ write_index(VecIndexPtr index, const std::string &location) { FileIOWriter writer(location); writer(&index_type, sizeof(IndexType)); - for (auto &iter: binaryset.binary_map_) { + for (auto &iter : binaryset.binary_map_) { auto meta = iter.first.c_str(); size_t meta_length = iter.first.length(); writer(&meta_length, sizeof(meta_length)); @@ -242,15 +238,14 @@ write_index(VecIndexPtr index, const std::string &location) { return Status::OK(); } - // TODO(linxj): redo here. void -AutoGenParams(const IndexType &type, const long &size, zilliz::knowhere::Config &cfg) { +AutoGenParams(const IndexType &type, const int64_t &size, zilliz::knowhere::Config &cfg) { auto nlist = cfg.get_with_default("nlist", 0); if (size <= TYPICAL_COUNT / 16384 + 1) { //handle less row count, avoid nlist set to 0 cfg["nlist"] = 1; - } else if (int(size / TYPICAL_COUNT) * nlist == 0) { + } else if (int(size / TYPICAL_COUNT) *nlist == 0) { //calculate a proper nlist if nlist not specified or size less than TYPICAL_COUNT cfg["nlist"] = int(size / TYPICAL_COUNT * 16384); } @@ -341,7 +336,6 @@ ConvertToGpuIndexType(const IndexType &type) { } } - -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/VecIndex.h b/cpp/src/wrapper/VecIndex.h index 3737270f42..3e7eb0ffc4 100644 --- a/cpp/src/wrapper/VecIndex.h +++ b/cpp/src/wrapper/VecIndex.h @@ -25,7 +25,6 @@ #include "knowhere/common/Config.h" #include "knowhere/common/BinarySet.h" - namespace zilliz { namespace milvus { namespace engine { @@ -55,24 +54,24 @@ using VecIndexPtr = std::shared_ptr; class VecIndex { public: virtual Status - BuildAll(const long &nb, + BuildAll(const int64_t &nb, const float *xb, - const long *ids, + const int64_t *ids, const Config &cfg, - const long &nt = 0, + const int64_t &nt = 0, const float *xt = nullptr) = 0; virtual Status - Add(const long &nb, + Add(const int64_t &nb, const float *xb, - const long *ids, + const int64_t *ids, const Config &cfg = Config()) = 0; virtual Status - Search(const long &nq, + Search(const int64_t &nq, const float *xq, float *dist, - long *ids, + int64_t *ids, const Config &cfg = Config()) = 0; virtual VecIndexPtr @@ -117,7 +116,7 @@ extern VecIndexPtr LoadVecIndex(const IndexType &index_type, const zilliz::knowhere::BinarySet &index_binary); extern void -AutoGenParams(const IndexType &type, const long &size, Config &cfg); +AutoGenParams(const IndexType &type, const int64_t &size, Config &cfg); extern void ParameterValidation(const IndexType &type, Config &cfg); @@ -128,6 +127,6 @@ ConvertToCpuIndexType(const IndexType &type); extern IndexType ConvertToGpuIndexType(const IndexType &type); -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz From d5a825edfde47259df8fbdb148832ae23deb4025 Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 26 Sep 2019 18:59:01 +0800 Subject: [PATCH 120/217] format config code Former-commit-id: 6d87e0d669e5f8029453a94cc1817372263ff30f --- cpp/src/config/ConfigMgr.cpp | 11 +++-- cpp/src/config/ConfigMgr.h | 14 +++--- cpp/src/config/ConfigNode.cpp | 80 +++++++++++++++++--------------- cpp/src/config/ConfigNode.h | 16 +++---- cpp/src/config/YamlConfigMgr.cpp | 57 ++++++++++++----------- cpp/src/config/YamlConfigMgr.h | 23 +++++---- 6 files changed, 106 insertions(+), 95 deletions(-) diff --git a/cpp/src/config/ConfigMgr.cpp b/cpp/src/config/ConfigMgr.cpp index f39679fcdd..a7cfdc9fda 100644 --- a/cpp/src/config/ConfigMgr.cpp +++ b/cpp/src/config/ConfigMgr.cpp @@ -15,18 +15,19 @@ // specific language governing permissions and limitations // under the License. -#include "ConfigMgr.h" +#include "config/ConfigMgr.h" #include "YamlConfigMgr.h" namespace zilliz { namespace milvus { namespace server { -ConfigMgr * ConfigMgr::GetInstance() { +ConfigMgr * +ConfigMgr::GetInstance() { static YamlConfigMgr mgr; return &mgr; } -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/ConfigMgr.h b/cpp/src/config/ConfigMgr.h index 3e546103b6..96a63aa2c8 100644 --- a/cpp/src/config/ConfigMgr.h +++ b/cpp/src/config/ConfigMgr.h @@ -20,6 +20,8 @@ #include "utils/Error.h" #include "ConfigNode.h" +#include + namespace zilliz { namespace milvus { namespace server { @@ -40,16 +42,16 @@ namespace server { class ConfigMgr { public: - static ConfigMgr* GetInstance(); + static ConfigMgr *GetInstance(); virtual ErrorCode LoadConfigFile(const std::string &filename) = 0; virtual void Print() const = 0;//will be deleted virtual std::string DumpString() const = 0; - virtual const ConfigNode& GetRootNode() const = 0; - virtual ConfigNode& GetRootNode() = 0; + virtual const ConfigNode &GetRootNode() const = 0; + virtual ConfigNode &GetRootNode() = 0; }; -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/ConfigNode.cpp b/cpp/src/config/ConfigNode.cpp index 88954c1bdf..87b080d572 100644 --- a/cpp/src/config/ConfigNode.cpp +++ b/cpp/src/config/ConfigNode.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "ConfigNode.h" +#include "config/ConfigNode.h" #include "utils/Error.h" #include "utils/Log.h" @@ -27,33 +27,34 @@ namespace zilliz { namespace milvus { namespace server { -void ConfigNode::Combine(const ConfigNode& target) { - const std::map& kv = target.GetConfig(); - for(auto itr = kv.begin(); itr != kv.end(); ++itr){ +void +ConfigNode::Combine(const ConfigNode &target) { + const std::map &kv = target.GetConfig(); + for (auto itr = kv.begin(); itr != kv.end(); ++itr) { config_[itr->first] = itr->second; } - const std::map >& sequences = target.GetSequences(); - for(auto itr = sequences.begin(); itr != sequences.end(); ++itr){ + const std::map > &sequences = target.GetSequences(); + for (auto itr = sequences.begin(); itr != sequences.end(); ++itr) { sequences_[itr->first] = itr->second; } - const std::map& children = target.GetChildren(); - for(auto itr = children.begin(); itr != children.end(); ++itr){ + const std::map &children = target.GetChildren(); + for (auto itr = children.begin(); itr != children.end(); ++itr) { children_[itr->first] = itr->second; } } //key/value pair config void -ConfigNode::SetValue(const std::string& key, const std::string& value) { +ConfigNode::SetValue(const std::string &key, const std::string &value) { config_[key] = value; } std::string -ConfigNode::GetValue(const std::string& param_key, const std::string& default_val) const { +ConfigNode::GetValue(const std::string ¶m_key, const std::string &default_val) const { auto ref = config_.find(param_key); - if(ref != config_.end()) { + if (ref != config_.end()) { return ref->second; } @@ -76,7 +77,7 @@ int32_t ConfigNode::GetInt32Value(const std::string ¶m_key, int32_t default_val) const { std::string val = GetValue(param_key); if (!val.empty()) { - return (int32_t)std::strtol(val.c_str(), nullptr, 10); + return (int32_t) std::strtol(val.c_str(), nullptr, 10); } else { return default_val; } @@ -112,25 +113,26 @@ ConfigNode::GetDoubleValue(const std::string ¶m_key, double default_val) con } } -const std::map& +const std::map & ConfigNode::GetConfig() const { return config_; -}; +} -void ConfigNode::ClearConfig() { +void +ConfigNode::ClearConfig() { config_.clear(); } //key/object config void -ConfigNode::AddChild(const std::string& type_name, const ConfigNode& config) { +ConfigNode::AddChild(const std::string &type_name, const ConfigNode &config) { children_[type_name] = config; } ConfigNode -ConfigNode::GetChild(const std::string& type_name) const { +ConfigNode::GetChild(const std::string &type_name) const { auto ref = children_.find(type_name); - if(ref != children_.end()) { + if (ref != children_.end()) { return ref->second; } @@ -138,25 +140,26 @@ ConfigNode::GetChild(const std::string& type_name) const { return nc; } -ConfigNode& +ConfigNode & ConfigNode::GetChild(const std::string &type_name) { return children_[type_name]; } void -ConfigNode::GetChildren(ConfigNodeArr& arr) const { +ConfigNode::GetChildren(ConfigNodeArr &arr) const { arr.clear(); - for(auto ref : children_){ + for (auto ref : children_) { arr.push_back(ref.second); } } -const std::map& +const std::map & ConfigNode::GetChildren() const { return children_; } -void ConfigNode::ClearChildren() { +void +ConfigNode::ClearChildren() { children_.clear(); } @@ -169,7 +172,7 @@ ConfigNode::AddSequenceItem(const std::string &key, const std::string &item) { std::vector ConfigNode::GetSequence(const std::string &key) const { auto itr = sequences_.find(key); - if(itr != sequences_.end()) { + if (itr != sequences_.end()) { return itr->second; } else { std::vector temp; @@ -177,29 +180,30 @@ ConfigNode::GetSequence(const std::string &key) const { } } -const std::map >& +const std::map > & ConfigNode::GetSequences() const { return sequences_; } -void ConfigNode::ClearSequences() { +void +ConfigNode::ClearSequences() { sequences_.clear(); } void -ConfigNode::PrintAll(const std::string& prefix) const { - for(auto& elem : config_) { +ConfigNode::PrintAll(const std::string &prefix) const { + for (auto &elem : config_) { SERVER_LOG_INFO << prefix << elem.first + ": " << elem.second; } - for(auto& elem : sequences_) { + for (auto &elem : sequences_) { SERVER_LOG_INFO << prefix << elem.first << ": "; - for(auto& str : elem.second) { + for (auto &str : elem.second) { SERVER_LOG_INFO << prefix << " - " << str; } } - for(auto& elem : children_) { + for (auto &elem : children_) { SERVER_LOG_INFO << prefix << elem.first << ": "; elem.second.PrintAll(prefix + " "); } @@ -209,18 +213,18 @@ std::string ConfigNode::DumpString(const std::string &prefix) const { std::stringstream str_buffer; const std::string endl = "\n"; - for(auto& elem : config_) { + for (auto &elem : config_) { str_buffer << prefix << elem.first << ": " << elem.second << endl; } - for(auto& elem : sequences_) { + for (auto &elem : sequences_) { str_buffer << prefix << elem.first << ": " << endl; - for(auto& str : elem.second) { + for (auto &str : elem.second) { str_buffer << prefix + " - " << str << endl; } } - for(auto& elem : children_) { + for (auto &elem : children_) { str_buffer << prefix << elem.first << ": " << endl; str_buffer << elem.second.DumpString(prefix + " ") << endl; } @@ -228,6 +232,6 @@ ConfigNode::DumpString(const std::string &prefix) const { return str_buffer.str(); } -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/ConfigNode.h b/cpp/src/config/ConfigNode.h index 606e090d4b..d3fabc0655 100644 --- a/cpp/src/config/ConfigNode.h +++ b/cpp/src/config/ConfigNode.h @@ -30,7 +30,7 @@ typedef std::vector ConfigNodeArr; class ConfigNode { public: - void Combine(const ConfigNode& target); + void Combine(const ConfigNode &target); //key/value pair config void SetValue(const std::string &key, const std::string &value); @@ -42,23 +42,23 @@ class ConfigNode { float GetFloatValue(const std::string ¶m_key, float default_val = 0.0) const; double GetDoubleValue(const std::string ¶m_key, double default_val = 0.0) const; - const std::map& GetConfig() const; + const std::map &GetConfig() const; void ClearConfig(); //key/object config void AddChild(const std::string &type_name, const ConfigNode &config); ConfigNode GetChild(const std::string &type_name) const; - ConfigNode& GetChild(const std::string &type_name); + ConfigNode &GetChild(const std::string &type_name); void GetChildren(ConfigNodeArr &arr) const; - const std::map& GetChildren() const; + const std::map &GetChildren() const; void ClearChildren(); //key/sequence config void AddSequenceItem(const std::string &key, const std::string &item); std::vector GetSequence(const std::string &key) const; - const std::map >& GetSequences() const; + const std::map > &GetSequences() const; void ClearSequences(); void PrintAll(const std::string &prefix = "") const; @@ -70,6 +70,6 @@ class ConfigNode { std::map > sequences_; }; -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/YamlConfigMgr.cpp b/cpp/src/config/YamlConfigMgr.cpp index bddbc32245..1dd8a3d943 100644 --- a/cpp/src/config/YamlConfigMgr.cpp +++ b/cpp/src/config/YamlConfigMgr.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "YamlConfigMgr.h" +#include "config/YamlConfigMgr.h" #include "utils/Log.h" #include @@ -24,7 +24,8 @@ namespace zilliz { namespace milvus { namespace server { -ErrorCode YamlConfigMgr::LoadConfigFile(const std::string &filename) { +ErrorCode +YamlConfigMgr::LoadConfigFile(const std::string &filename) { struct stat directoryStat; int statOK = stat(filename.c_str(), &directoryStat); if (statOK != 0) { @@ -36,36 +37,40 @@ ErrorCode YamlConfigMgr::LoadConfigFile(const std::string &filename) { node_ = YAML::LoadFile(filename); LoadConfigNode(node_, config_); } - catch (YAML::Exception& e) { - SERVER_LOG_ERROR << "Failed to load config file: " << std::string(e.what ()); + catch (YAML::Exception &e) { + SERVER_LOG_ERROR << "Failed to load config file: " << std::string(e.what()); return SERVER_UNEXPECTED_ERROR; } return SERVER_SUCCESS; } -void YamlConfigMgr::Print() const { +void +YamlConfigMgr::Print() const { SERVER_LOG_INFO << "System config content:"; config_.PrintAll(); } -std::string YamlConfigMgr::DumpString() const { +std::string +YamlConfigMgr::DumpString() const { return config_.DumpString(""); } -const ConfigNode& YamlConfigMgr::GetRootNode() const { +const ConfigNode & +YamlConfigMgr::GetRootNode() const { return config_; } -ConfigNode& YamlConfigMgr::GetRootNode() { +ConfigNode & +YamlConfigMgr::GetRootNode() { return config_; } bool -YamlConfigMgr::SetConfigValue(const YAML::Node& node, - const std::string& key, - ConfigNode& config) { - if(node[key].IsDefined ()) { +YamlConfigMgr::SetConfigValue(const YAML::Node &node, + const std::string &key, + ConfigNode &config) { + if (node[key].IsDefined()) { config.SetValue(key, node[key].as()); return true; } @@ -73,10 +78,10 @@ YamlConfigMgr::SetConfigValue(const YAML::Node& node, } bool -YamlConfigMgr::SetChildConfig(const YAML::Node& node, - const std::string& child_name, - ConfigNode& config) { - if(node[child_name].IsDefined ()) { +YamlConfigMgr::SetChildConfig(const YAML::Node &node, + const std::string &child_name, + ConfigNode &config) { + if (node[child_name].IsDefined()) { ConfigNode sub_config; LoadConfigNode(node[child_name], sub_config); config.AddChild(child_name, sub_config); @@ -89,9 +94,9 @@ bool YamlConfigMgr::SetSequence(const YAML::Node &node, const std::string &child_name, ConfigNode &config) { - if(node[child_name].IsDefined ()) { + if (node[child_name].IsDefined()) { size_t cnt = node[child_name].size(); - for(size_t i = 0; i < cnt; i++){ + for (size_t i = 0; i < cnt; i++) { config.AddSequenceItem(child_name, node[child_name][i].as()); } return true; @@ -100,22 +105,22 @@ YamlConfigMgr::SetSequence(const YAML::Node &node, } void -YamlConfigMgr::LoadConfigNode(const YAML::Node& node, ConfigNode& config) { +YamlConfigMgr::LoadConfigNode(const YAML::Node &node, ConfigNode &config) { std::string key; for (YAML::const_iterator it = node.begin(); it != node.end(); ++it) { - if(!it->first.IsNull()){ + if (!it->first.IsNull()) { key = it->first.as(); } - if(node[key].IsScalar()) { + if (node[key].IsScalar()) { SetConfigValue(node, key, config); - } else if(node[key].IsMap()){ + } else if (node[key].IsMap()) { SetChildConfig(node, key, config); - } else if(node[key].IsSequence()){ + } else if (node[key].IsSequence()) { SetSequence(node, key, config); } } } -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/YamlConfigMgr.h b/cpp/src/config/YamlConfigMgr.h index ef4d02c3ce..6b84943bf8 100644 --- a/cpp/src/config/YamlConfigMgr.h +++ b/cpp/src/config/YamlConfigMgr.h @@ -21,6 +21,7 @@ #include "ConfigNode.h" #include "utils/Error.h" +#include #include namespace zilliz { @@ -33,15 +34,15 @@ class YamlConfigMgr : public ConfigMgr { virtual void Print() const; virtual std::string DumpString() const; - virtual const ConfigNode& GetRootNode() const; - virtual ConfigNode& GetRootNode(); + virtual const ConfigNode &GetRootNode() const; + virtual ConfigNode &GetRootNode(); private: - bool SetConfigValue(const YAML::Node& node, - const std::string& key, - ConfigNode& config); + bool SetConfigValue(const YAML::Node &node, + const std::string &key, + ConfigNode &config); - bool SetChildConfig(const YAML::Node& node, + bool SetChildConfig(const YAML::Node &node, const std::string &name, ConfigNode &config); @@ -50,15 +51,13 @@ class YamlConfigMgr : public ConfigMgr { const std::string &child_name, ConfigNode &config); - void LoadConfigNode(const YAML::Node& node, ConfigNode& config); + void LoadConfigNode(const YAML::Node &node, ConfigNode &config); private: YAML::Node node_; ConfigNode config_; }; -} -} -} - - +} // namespace server +} // namespace milvus +} // namespace zilliz From d9b6fd057540cd0abfa8a7e3a6b43ba7582f304f Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Thu, 26 Sep 2019 19:05:09 +0800 Subject: [PATCH 121/217] MS-574 rename config deploy_mode Former-commit-id: b3f2c2cc539c3cd7419c091b6eaf17df0a57f0eb --- cpp/src/db/DBImpl.cpp | 6 +++--- cpp/src/db/Options.h | 6 +++--- cpp/src/db/meta/MySQLMetaImpl.cpp | 4 ++-- cpp/src/server/Config.cpp | 26 +++++++++++++------------- cpp/src/server/Config.h | 12 ++++++------ cpp/src/server/DBWrapper.cpp | 12 ++++++------ 6 files changed, 33 insertions(+), 33 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index a763c9199c..ed1318ef1d 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -77,7 +77,7 @@ Status DBImpl::Start() { shutting_down_.store(false, std::memory_order_release); //for distribute version, some nodes are read only - if (options_.mode_ != DBOptions::MODE::READ_ONLY) { + if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) { ENGINE_LOG_TRACE << "StartTimerTasks"; bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this); } @@ -98,7 +98,7 @@ Status DBImpl::Stop() { //wait compaction/buildindex finish bg_timer_thread_.join(); - if (options_.mode_ != DBOptions::MODE::READ_ONLY) { + if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) { meta_ptr_->CleanUp(); } @@ -684,7 +684,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes - if (options_.mode_ == DBOptions::MODE::CLUSTER) { + if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { ttl = meta::D_SEC; } meta_ptr_->CleanUpFilesWithTTL(ttl); diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 2ba8c36a32..dbaf34b065 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -60,9 +60,9 @@ struct DBMetaOptions { struct DBOptions { typedef enum { - SINGLE, - CLUSTER, - READ_ONLY + SINGLE = 0, + CLUSTER_READONLY, + CLUSTER_WRITABLE } MODE; uint16_t merge_trigger_number_ = 2; diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 3e54686ed0..5a5470008d 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -284,7 +284,7 @@ Status MySQLMetaImpl::Initialize() { //step 5: create meta tables try { - if (mode_ != DBOptions::MODE::READ_ONLY) { + if (mode_ != DBOptions::MODE::CLUSTER_READONLY) { CleanUp(); } @@ -758,7 +758,7 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { } //Scoped Connection - if (mode_ == DBOptions::MODE::CLUSTER) { + if (mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { DeleteTableFiles(table_id); } diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 3730ff6f68..31dc5fe1a9 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -84,7 +84,7 @@ Config::ValidateConfig() { if (!s.ok()) return s; std::string server_mode; - s = GetServerConfigMode(server_mode); + s = GetServerConfigDeployMode(server_mode); if (!s.ok()) return s; std::string server_time_zone; @@ -219,7 +219,7 @@ Config::CheckServerConfigPort(const std::string &value) { } Status -Config::CheckServerConfigMode(const std::string &value) { +Config::CheckServerConfigDeployMode(const std::string &value) { if (value != "single" && value != "cluster_readonly" && value != "cluster_writable") { @@ -511,12 +511,12 @@ Config::GetServerConfigStrPort() { } std::string -Config::GetServerConfigStrMode() { +Config::GetServerConfigStrDeployMode() { std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_MODE, - CONFIG_SERVER_MODE_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value).ok()) { + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_DEPLOY_MODE, + CONFIG_SERVER_DEPLOY_MODE_DEFAULT); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); } return value; } @@ -755,9 +755,9 @@ Config::GetServerConfigPort(std::string& value) { } Status -Config::GetServerConfigMode(std::string& value) { - value = GetServerConfigStrMode(); - return CheckServerConfigMode(value); +Config::GetServerConfigDeployMode(std::string& value) { + value = GetServerConfigStrDeployMode(); + return CheckServerConfigDeployMode(value); } Status @@ -938,10 +938,10 @@ Config::SetServerConfigPort(const std::string& value) { } Status -Config::SetServerConfigMode(const std::string& value) { - Status s = CheckServerConfigMode(value); +Config::SetServerConfigDeployMode(const std::string& value) { + Status s = CheckServerConfigDeployMode(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); return Status::OK(); } diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index 710230eb0a..422632a01f 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -34,8 +34,8 @@ static const char* CONFIG_SERVER_ADDRESS = "address"; static const char* CONFIG_SERVER_ADDRESS_DEFAULT = "127.0.0.1"; static const char* CONFIG_SERVER_PORT = "port"; static const char* CONFIG_SERVER_PORT_DEFAULT = "19530"; -static const char* CONFIG_SERVER_MODE = "mode"; -static const char* CONFIG_SERVER_MODE_DEFAULT = "single"; +static const char* CONFIG_SERVER_DEPLOY_MODE = "deploy_mode"; +static const char* CONFIG_SERVER_DEPLOY_MODE_DEFAULT = "single"; static const char* CONFIG_SERVER_TIME_ZONE = "time_zone"; static const char* CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; @@ -117,7 +117,7 @@ class Config { /* server config */ Status CheckServerConfigAddress(const std::string& value); Status CheckServerConfigPort(const std::string& value); - Status CheckServerConfigMode(const std::string& value); + Status CheckServerConfigDeployMode(const std::string& value); Status CheckServerConfigTimeZone(const std::string& value); /* db config */ @@ -153,7 +153,7 @@ class Config { /* server config */ std::string GetServerConfigStrAddress(); std::string GetServerConfigStrPort(); - std::string GetServerConfigStrMode(); + std::string GetServerConfigStrDeployMode(); std::string GetServerConfigStrTimeZone(); /* db config */ @@ -188,7 +188,7 @@ class Config { /* server config */ Status GetServerConfigAddress(std::string& value); Status GetServerConfigPort(std::string& value); - Status GetServerConfigMode(std::string& value); + Status GetServerConfigDeployMode(std::string& value); Status GetServerConfigTimeZone(std::string& value); /* db config */ @@ -224,7 +224,7 @@ class Config { /* server config */ Status SetServerConfigAddress(const std::string& value); Status SetServerConfigPort(const std::string& value); - Status SetServerConfigMode(const std::string& value); + Status SetServerConfigDeployMode(const std::string& value); Status SetServerConfigTimeZone(const std::string& value); /* db config */ diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 745965dc6a..49c4cac8c8 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -60,20 +60,20 @@ Status DBWrapper::StartService() { if (!s.ok()) return s; std::string mode; - s = config.GetServerConfigMode(mode); + s = config.GetServerConfigDeployMode(mode); if (!s.ok()) return s; if (mode == "single") { opt.mode_ = engine::DBOptions::MODE::SINGLE; } - else if (mode == "cluster") { - opt.mode_ = engine::DBOptions::MODE::CLUSTER; + else if (mode == "cluster_readonly") { + opt.mode_ = engine::DBOptions::MODE::CLUSTER_READONLY; } - else if (mode == "read_only") { - opt.mode_ = engine::DBOptions::MODE::READ_ONLY; + else if (mode == "cluster_writable") { + opt.mode_ = engine::DBOptions::MODE::CLUSTER_WRITABLE; } else { - std::cerr << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" << std::endl; + std::cerr << "ERROR: mode specified in server_config must be ['single', 'cluster_readonly', 'cluster_writable']" << std::endl; kill(0, SIGUSR1); } From 27e7850bdf2ba7e0fe5f50a1b7d23db4340fa7fa Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 26 Sep 2019 19:13:12 +0800 Subject: [PATCH 122/217] format code Former-commit-id: b60db90436cc9d8de9fd86d721ffb48d3041da73 --- cpp/src/main.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index 6d28a3bd67..ca3283f4d0 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -101,15 +101,14 @@ main(int argc, char *argv[]) { } /* Handle Signal */ - using ::zilliz::milvus; - signal(SIGHUP, server::SignalUtil::HandleSignal); - signal(SIGINT, server::SignalUtil::HandleSignal); - signal(SIGUSR1, server::SignalUtil::HandleSignal); - signal(SIGSEGV, server::SignalUtil::HandleSignal); - signal(SIGUSR2, server::SignalUtil::HandleSignal); - signal(SIGTERM, server::SignalUtil::HandleSignal); + signal(SIGHUP, zilliz::milvus::server::SignalUtil::HandleSignal); + signal(SIGINT, zilliz::milvus::server::SignalUtil::HandleSignal); + signal(SIGUSR1, zilliz::milvus::server::SignalUtil::HandleSignal); + signal(SIGSEGV, zilliz::milvus::server::SignalUtil::HandleSignal); + signal(SIGUSR2, zilliz::milvus::server::SignalUtil::HandleSignal); + signal(SIGTERM, zilliz::milvus::server::SignalUtil::HandleSignal); - server::Server &server = server::Server::GetInstance(); + zilliz::milvus::server::Server &server = zilliz::milvus::server::Server::GetInstance(); server.Init(start_daemonized, pid_filename, config_filename, log_config_file); server.Start(); From 3d51d6955579614cc025e5d990c26083ec655422 Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 26 Sep 2019 19:58:25 +0800 Subject: [PATCH 123/217] format utils code Former-commit-id: aa281f8473100587c93afe7caf001ce640fbcc2d --- cpp/src/utils/BlockingQueue.h | 14 +-- cpp/src/utils/BlockingQueue.inl | 35 ++++--- cpp/src/utils/CommonUtil.cpp | 129 ++++++++++++++------------ cpp/src/utils/CommonUtil.h | 19 ++-- cpp/src/utils/Error.h | 38 ++++---- cpp/src/utils/Exception.h | 23 +++-- cpp/src/utils/Log.h | 2 +- cpp/src/utils/LogUtil.cpp | 17 ++-- cpp/src/utils/LogUtil.h | 14 +-- cpp/src/utils/SignalUtil.cpp | 17 ++-- cpp/src/utils/SignalUtil.h | 6 +- cpp/src/utils/Status.cpp | 41 ++++---- cpp/src/utils/Status.h | 14 ++- cpp/src/utils/StringHelpFunctions.cpp | 34 ++++--- cpp/src/utils/StringHelpFunctions.h | 12 +-- cpp/src/utils/ThreadPool.h | 65 +++++++------ cpp/src/utils/TimeRecorder.cpp | 7 +- cpp/src/utils/TimeRecorder.h | 11 +-- cpp/src/utils/ValidationUtil.cpp | 25 ++--- cpp/src/utils/ValidationUtil.h | 16 ++-- 20 files changed, 283 insertions(+), 256 deletions(-) diff --git a/cpp/src/utils/BlockingQueue.h b/cpp/src/utils/BlockingQueue.h index 6e46ed1bb1..b98fe28ef8 100644 --- a/cpp/src/utils/BlockingQueue.h +++ b/cpp/src/utils/BlockingQueue.h @@ -29,8 +29,9 @@ namespace server { template class BlockingQueue { -public: - BlockingQueue() : mtx(), full_(), empty_() {} + public: + BlockingQueue() : mtx(), full_(), empty_() { + } BlockingQueue(const BlockingQueue &rhs) = delete; @@ -50,7 +51,7 @@ public: void SetCapacity(const size_t capacity); -private: + private: mutable std::mutex mtx; std::condition_variable full_; std::condition_variable empty_; @@ -58,9 +59,8 @@ private: size_t capacity_ = 32; }; -} -} -} - +} // namespace server +} // namespace milvus +} // namespace zilliz #include "./BlockingQueue.inl" diff --git a/cpp/src/utils/BlockingQueue.inl b/cpp/src/utils/BlockingQueue.inl index 86237f3322..ed15aac77a 100644 --- a/cpp/src/utils/BlockingQueue.inl +++ b/cpp/src/utils/BlockingQueue.inl @@ -18,7 +18,6 @@ #pragma once - namespace zilliz { namespace milvus { namespace server { @@ -26,8 +25,10 @@ namespace server { template void BlockingQueue::Put(const T &task) { - std::unique_lock lock(mtx); - full_.wait(lock, [this] { return (queue_.size() < capacity_); }); + std::unique_lock lock(mtx); + full_.wait(lock, [this] { + return (queue_.size() < capacity_); + }); queue_.push(task); empty_.notify_all(); @@ -36,8 +37,10 @@ BlockingQueue::Put(const T &task) { template T BlockingQueue::Take() { - std::unique_lock lock(mtx); - empty_.wait(lock, [this] { return !queue_.empty(); }); + std::unique_lock lock(mtx); + empty_.wait(lock, [this] { + return !queue_.empty(); + }); T front(queue_.front()); queue_.pop(); @@ -48,15 +51,17 @@ BlockingQueue::Take() { template size_t BlockingQueue::Size() { - std::lock_guard lock(mtx); + std::lock_guard lock(mtx); return queue_.size(); } template T BlockingQueue::Front() { - std::unique_lock lock(mtx); - empty_.wait(lock, [this] { return !queue_.empty(); }); + std::unique_lock lock(mtx); + empty_.wait(lock, [this] { + return !queue_.empty(); + }); T front(queue_.front()); return front; @@ -65,8 +70,10 @@ BlockingQueue::Front() { template T BlockingQueue::Back() { - std::unique_lock lock(mtx); - empty_.wait(lock, [this] { return !queue_.empty(); }); + std::unique_lock lock(mtx); + empty_.wait(lock, [this] { + return !queue_.empty(); + }); T back(queue_.back()); return back; @@ -75,7 +82,7 @@ BlockingQueue::Back() { template bool BlockingQueue::Empty() { - std::unique_lock lock(mtx); + std::unique_lock lock(mtx); return queue_.empty(); } @@ -85,7 +92,7 @@ BlockingQueue::SetCapacity(const size_t capacity) { capacity_ = (capacity > 0 ? capacity : capacity_); } -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/CommonUtil.cpp b/cpp/src/utils/CommonUtil.cpp index ff985dcd7d..0116e321e0 100644 --- a/cpp/src/utils/CommonUtil.cpp +++ b/cpp/src/utils/CommonUtil.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "CommonUtil.h" +#include "utils/CommonUtil.h" #include "utils/Log.h" #include @@ -44,7 +44,8 @@ namespace server { namespace fs = boost::filesystem; -bool CommonUtil::GetSystemMemInfo(unsigned long &total_mem, unsigned long &free_mem) { +bool +CommonUtil::GetSystemMemInfo(uint64_t &total_mem, uint64_t &free_mem) { struct sysinfo info; int ret = sysinfo(&info); total_mem = info.totalram; @@ -53,7 +54,8 @@ bool CommonUtil::GetSystemMemInfo(unsigned long &total_mem, unsigned long &free_ return ret == 0;//succeed 0, failed -1 } -bool CommonUtil::GetSystemAvailableThreads(unsigned int &thread_count) { +bool +CommonUtil::GetSystemAvailableThreads(uint32_t &thread_count) { //threadCnt = std::thread::hardware_concurrency(); thread_count = sysconf(_SC_NPROCESSORS_CONF); thread_count *= THREAD_MULTIPLY_CPU; @@ -63,7 +65,8 @@ bool CommonUtil::GetSystemAvailableThreads(unsigned int &thread_count) { return true; } -bool CommonUtil::IsDirectoryExist(const std::string &path) { +bool +CommonUtil::IsDirectoryExist(const std::string &path) { DIR *dp = nullptr; if ((dp = opendir(path.c_str())) == nullptr) { return false; @@ -73,8 +76,9 @@ bool CommonUtil::IsDirectoryExist(const std::string &path) { return true; } -Status CommonUtil::CreateDirectory(const std::string &path) { - if(path.empty()) { +Status +CommonUtil::CreateDirectory(const std::string &path) { + if (path.empty()) { return Status::OK(); } @@ -87,7 +91,7 @@ Status CommonUtil::CreateDirectory(const std::string &path) { fs::path fs_path(path); fs::path parent_path = fs_path.parent_path(); Status err_status = CreateDirectory(parent_path.string()); - if(!err_status.ok()){ + if (!err_status.ok()) { return err_status; } @@ -96,7 +100,7 @@ Status CommonUtil::CreateDirectory(const std::string &path) { return Status::OK();//already exist } - int makeOK = mkdir(path.c_str(), S_IRWXU|S_IRGRP|S_IROTH); + int makeOK = mkdir(path.c_str(), S_IRWXU | S_IRGRP | S_IROTH); if (makeOK != 0) { return Status(SERVER_UNEXPECTED_ERROR, "failed to create directory: " + path); } @@ -105,37 +109,38 @@ Status CommonUtil::CreateDirectory(const std::string &path) { } namespace { - void RemoveDirectory(const std::string &path) { - DIR *dir = nullptr; - struct dirent *dmsg; - char file_name[256]; - char folder_name[256]; +void +RemoveDirectory(const std::string &path) { + DIR *dir = nullptr; + struct dirent *dmsg; + const int32_t buf_size = 256; + char file_name[buf_size]; - strcpy(folder_name, path.c_str()); - strcat(folder_name, "/%s"); - if ((dir = opendir(path.c_str())) != nullptr) { - while ((dmsg = readdir(dir)) != nullptr) { - if (strcmp(dmsg->d_name, ".") != 0 - && strcmp(dmsg->d_name, "..") != 0) { - sprintf(file_name, folder_name, dmsg->d_name); - std::string tmp = file_name; - if (tmp.find(".") == std::string::npos) { - RemoveDirectory(file_name); - } - remove(file_name); + std::string folder_name = path + "/%s"; + if ((dir = opendir(path.c_str())) != nullptr) { + while ((dmsg = readdir(dir)) != nullptr) { + if (strcmp(dmsg->d_name, ".") != 0 + && strcmp(dmsg->d_name, "..") != 0) { + snprintf(file_name, buf_size, folder_name.c_str(), dmsg->d_name); + std::string tmp = file_name; + if (tmp.find(".") == std::string::npos) { + RemoveDirectory(file_name); } + remove(file_name); } } - - if (dir != nullptr) { - closedir(dir); - } - remove(path.c_str()); } -} -Status CommonUtil::DeleteDirectory(const std::string &path) { - if(path.empty()) { + if (dir != nullptr) { + closedir(dir); + } + remove(path.c_str()); +} +} // namespace + +Status +CommonUtil::DeleteDirectory(const std::string &path) { + if (path.empty()) { return Status::OK(); } @@ -149,58 +154,63 @@ Status CommonUtil::DeleteDirectory(const std::string &path) { return Status::OK(); } -bool CommonUtil::IsFileExist(const std::string &path) { +bool +CommonUtil::IsFileExist(const std::string &path) { return (access(path.c_str(), F_OK) == 0); } -uint64_t CommonUtil::GetFileSize(const std::string &path) { +uint64_t +CommonUtil::GetFileSize(const std::string &path) { struct stat file_info; if (stat(path.c_str(), &file_info) < 0) { return 0; } else { - return (uint64_t)file_info.st_size; + return (uint64_t) file_info.st_size; } } -std::string CommonUtil::GetFileName(std::string filename) { +std::string +CommonUtil::GetFileName(std::string filename) { int pos = filename.find_last_of('/'); return filename.substr(pos + 1); } -std::string CommonUtil::GetExePath() { +std::string +CommonUtil::GetExePath() { const size_t buf_len = 1024; char buf[buf_len]; size_t cnt = readlink("/proc/self/exe", buf, buf_len); - if(cnt < 0|| cnt >= buf_len) { + if (cnt < 0 || cnt >= buf_len) { return ""; } buf[cnt] = '\0'; std::string exe_path = buf; - if(exe_path.rfind('/') != exe_path.length()){ + if (exe_path.rfind('/') != exe_path.length()) { std::string sub_str = exe_path.substr(0, exe_path.rfind('/')); return sub_str + "/"; } return exe_path; } -bool CommonUtil::TimeStrToTime(const std::string& time_str, - time_t &time_integer, - tm &time_struct, - const std::string& format) { +bool +CommonUtil::TimeStrToTime(const std::string &time_str, + time_t &time_integer, + tm &time_struct, + const std::string &format) { time_integer = 0; memset(&time_struct, 0, sizeof(tm)); int ret = sscanf(time_str.c_str(), - format.c_str(), - &(time_struct.tm_year), - &(time_struct.tm_mon), - &(time_struct.tm_mday), - &(time_struct.tm_hour), - &(time_struct.tm_min), - &(time_struct.tm_sec)); - if(ret <= 0) { + format.c_str(), + &(time_struct.tm_year), + &(time_struct.tm_mon), + &(time_struct.tm_mday), + &(time_struct.tm_hour), + &(time_struct.tm_min), + &(time_struct.tm_sec)); + if (ret <= 0) { return false; } @@ -211,15 +221,16 @@ bool CommonUtil::TimeStrToTime(const std::string& time_str, return true; } -void CommonUtil::ConvertTime(time_t time_integer, tm &time_struct) { - tm* t_m = localtime (&time_integer); - memcpy(&time_struct, t_m, sizeof(tm)); +void +CommonUtil::ConvertTime(time_t time_integer, tm &time_struct) { + localtime_r(&time_integer, &time_struct); } -void CommonUtil::ConvertTime(tm time_struct, time_t &time_integer) { +void +CommonUtil::ConvertTime(tm time_struct, time_t &time_integer) { time_integer = mktime(&time_struct); } -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/CommonUtil.h b/cpp/src/utils/CommonUtil.h index 7e1349bf9e..b059067d50 100755 --- a/cpp/src/utils/CommonUtil.h +++ b/cpp/src/utils/CommonUtil.h @@ -22,15 +22,14 @@ #include #include - namespace zilliz { namespace milvus { namespace server { class CommonUtil { public: - static bool GetSystemMemInfo(unsigned long &total_mem, unsigned long &free_mem); - static bool GetSystemAvailableThreads(unsigned int &thread_count); + static bool GetSystemMemInfo(uint64_t &total_mem, uint64_t &free_mem); + static bool GetSystemAvailableThreads(uint32_t &thread_count); static bool IsFileExist(const std::string &path); static uint64_t GetFileSize(const std::string &path); @@ -41,16 +40,16 @@ class CommonUtil { static std::string GetFileName(std::string filename); static std::string GetExePath(); - static bool TimeStrToTime(const std::string& time_str, - time_t &time_integer, - tm &time_struct, - const std::string& format = "%d-%d-%d %d:%d:%d"); + static bool TimeStrToTime(const std::string &time_str, + time_t &time_integer, + tm &time_struct, + const std::string &format = "%d-%d-%d %d:%d:%d"); static void ConvertTime(time_t time_integer, tm &time_struct); static void ConvertTime(tm time_struct, time_t &time_integer); }; -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index e903dcb2fb..d0bcc5cf8d 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -28,6 +28,7 @@ using ErrorCode = int32_t; constexpr ErrorCode SERVER_SUCCESS = 0; constexpr ErrorCode SERVER_ERROR_CODE_BASE = 0x30000; + constexpr ErrorCode ToServerErrorCode(const ErrorCode error_code) { return SERVER_ERROR_CODE_BASE + error_code; @@ -35,6 +36,7 @@ ToServerErrorCode(const ErrorCode error_code) { constexpr ErrorCode DB_SUCCESS = 0; constexpr ErrorCode DB_ERROR_CODE_BASE = 0x40000; + constexpr ErrorCode ToDbErrorCode(const ErrorCode error_code) { return DB_ERROR_CODE_BASE + error_code; @@ -42,6 +44,7 @@ ToDbErrorCode(const ErrorCode error_code) { constexpr ErrorCode KNOWHERE_SUCCESS = 0; constexpr ErrorCode KNOWHERE_ERROR_CODE_BASE = 0x50000; + constexpr ErrorCode ToKnowhereErrorCode(const ErrorCode error_code) { return KNOWHERE_ERROR_CODE_BASE + error_code; @@ -96,26 +99,27 @@ constexpr ErrorCode KNOWHERE_UNEXPECTED_ERROR = ToKnowhereErrorCode(3); constexpr ErrorCode KNOWHERE_NO_SPACE = ToKnowhereErrorCode(4); namespace server { - class ServerException : public std::exception { - public: - ServerException(ErrorCode error_code, - const std::string &message = std::string()) - : error_code_(error_code), message_(message) {} +class ServerException : public std::exception { + public: + ServerException(ErrorCode error_code, + const std::string &message = std::string()) + : error_code_(error_code), message_(message) { + } - public: - ErrorCode error_code() const { - return error_code_; - } + public: + ErrorCode error_code() const { + return error_code_; + } - virtual const char *what() const noexcept { - return message_.c_str(); - } + virtual const char *what() const noexcept { + return message_.c_str(); + } - private: - ErrorCode error_code_; - std::string message_; - }; -} + private: + ErrorCode error_code_; + std::string message_; +}; +} // namespace server } // namespace milvus } // namespace zilliz diff --git a/cpp/src/utils/Exception.h b/cpp/src/utils/Exception.h index 3a9ab89294..7e30c372bc 100644 --- a/cpp/src/utils/Exception.h +++ b/cpp/src/utils/Exception.h @@ -26,17 +26,17 @@ namespace zilliz { namespace milvus { class Exception : public std::exception { -public: - Exception(ErrorCode code, const std::string& message) + public: + Exception(ErrorCode code, const std::string &message) : code_(code), message_(message) { - } + } ErrorCode code() const throw() { return code_; } - virtual const char* what() const throw() { + virtual const char *what() const throw() { if (message_.empty()) { return "Default Exception."; } else { @@ -44,24 +44,23 @@ public: } } - virtual ~Exception() throw() {}; + virtual ~Exception() throw() { + } -protected: + protected: ErrorCode code_; std::string message_; }; class InvalidArgumentException : public Exception { -public: + public: InvalidArgumentException() : Exception(SERVER_INVALID_ARGUMENT, "Invalid Argument") { + } - }; - InvalidArgumentException(const std::string& message) + explicit InvalidArgumentException(const std::string &message) : Exception(SERVER_INVALID_ARGUMENT, message) { - - }; - + } }; } // namespace milvus diff --git a/cpp/src/utils/Log.h b/cpp/src/utils/Log.h index 2610a3a8bf..b1402d9e3e 100644 --- a/cpp/src/utils/Log.h +++ b/cpp/src/utils/Log.h @@ -17,7 +17,7 @@ #pragma once -#include "easylogging++.h" +#include "utils/easylogging++.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp index 3a3fe0c7e2..0e710a6e3a 100644 --- a/cpp/src/utils/LogUtil.cpp +++ b/cpp/src/utils/LogUtil.cpp @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. +#include "utils/LogUtil.h" + #include #include #include -#include "LogUtil.h" - namespace zilliz { namespace milvus { namespace server { @@ -35,7 +35,8 @@ static int fatal_idx = 0; } // TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename -void RolloutHandler(const char *filename, std::size_t size, el::Level level) { +void +RolloutHandler(const char *filename, std::size_t size, el::Level level) { char *dirc = strdup(filename); char *basec = strdup(filename); char *dir = dirname(dirc); @@ -80,7 +81,8 @@ void RolloutHandler(const char *filename, std::size_t size, el::Level level) { } } -Status InitLog(const std::string &log_config_file) { +Status +InitLog(const std::string &log_config_file) { el::Configurations conf(log_config_file); el::Loggers::reconfigureAllLoggers(conf); @@ -91,7 +93,6 @@ Status InitLog(const std::string &log_config_file) { return Status::OK(); } - -} // server -} // milvus -} // zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/LogUtil.h b/cpp/src/utils/LogUtil.h index d86b301d9f..e3f7bed51f 100644 --- a/cpp/src/utils/LogUtil.h +++ b/cpp/src/utils/LogUtil.h @@ -18,18 +18,20 @@ #pragma once #include "utils/Status.h" +#include "utils/easylogging++.h" #include #include -#include "easylogging++.h" namespace zilliz { namespace milvus { namespace server { -Status InitLog(const std::string& log_config_file); +Status +InitLog(const std::string &log_config_file); -void RolloutHandler(const char *filename, std::size_t size, el::Level level); +void +RolloutHandler(const char *filename, std::size_t size, el::Level level); #define SHOW_LOCATION #ifdef SHOW_LOCATION @@ -38,6 +40,6 @@ void RolloutHandler(const char *filename, std::size_t size, el::Level level); #define LOCATION_INFO "" #endif -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/SignalUtil.cpp b/cpp/src/utils/SignalUtil.cpp index 92678cfe6b..5f74852995 100644 --- a/cpp/src/utils/SignalUtil.cpp +++ b/cpp/src/utils/SignalUtil.cpp @@ -15,20 +15,20 @@ // specific language governing permissions and limitations // under the License. -#include "SignalUtil.h" +#include "utils/SignalUtil.h" #include "src/server/Server.h" #include "utils/Log.h" +#include #include #include - namespace zilliz { namespace milvus { namespace server { -void SignalUtil::HandleSignal(int signum) { - +void +SignalUtil::HandleSignal(int signum) { switch (signum) { case SIGINT: case SIGUSR2: { @@ -51,7 +51,8 @@ void SignalUtil::HandleSignal(int signum) { } } -void SignalUtil::PrintStacktrace() { +void +SignalUtil::PrintStacktrace() { SERVER_LOG_INFO << "Call stack:"; const int size = 32; @@ -65,6 +66,6 @@ void SignalUtil::PrintStacktrace() { free(stacktrace); } -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/SignalUtil.h b/cpp/src/utils/SignalUtil.h index 39645cf989..c9cd41839b 100644 --- a/cpp/src/utils/SignalUtil.h +++ b/cpp/src/utils/SignalUtil.h @@ -27,6 +27,6 @@ class SignalUtil { static void PrintStacktrace(); }; -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/Status.cpp b/cpp/src/utils/Status.cpp index e21a279a4c..5b512b3369 100644 --- a/cpp/src/utils/Status.cpp +++ b/cpp/src/utils/Status.cpp @@ -14,7 +14,8 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -#include "Status.h" + +#include "utils/Status.h" #include @@ -23,12 +24,12 @@ namespace milvus { constexpr int CODE_WIDTH = sizeof(StatusCode); -Status::Status(StatusCode code, const std::string& msg) { +Status::Status(StatusCode code, const std::string &msg) { //4 bytes store code //4 bytes store message length //the left bytes store message string - const uint32_t length = (uint32_t)msg.size(); - char* result = new char[length + sizeof(length) + CODE_WIDTH]; + const uint32_t length = (uint32_t) msg.size(); + char *result = new char[length + sizeof(length) + CODE_WIDTH]; std::memcpy(result, &code, CODE_WIDTH); std::memcpy(result + CODE_WIDTH, &length, sizeof(length)); memcpy(result + sizeof(length) + CODE_WIDTH, msg.data(), length); @@ -38,7 +39,6 @@ Status::Status(StatusCode code, const std::string& msg) { Status::Status() : state_(nullptr) { - } Status::~Status() { @@ -50,7 +50,7 @@ Status::Status(const Status &s) CopyFrom(s); } -Status& +Status & Status::operator=(const Status &s) { CopyFrom(s); return *this; @@ -61,7 +61,7 @@ Status::Status(Status &&s) MoveFrom(s); } -Status& +Status & Status::operator=(Status &&s) { MoveFrom(s); return *this; @@ -71,7 +71,7 @@ void Status::CopyFrom(const Status &s) { delete state_; state_ = nullptr; - if(s.state_ == nullptr) { + if (s.state_ == nullptr) { return; } @@ -79,7 +79,7 @@ Status::CopyFrom(const Status &s) { memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length)); int buff_len = length + sizeof(length) + CODE_WIDTH; state_ = new char[buff_len]; - memcpy((void*)state_, (void*)s.state_, buff_len); + memcpy((void *) state_, (void *) s.state_, buff_len); } void @@ -98,7 +98,7 @@ Status::message() const { std::string msg; uint32_t length = 0; memcpy(&length, state_ + CODE_WIDTH, sizeof(length)); - if(length > 0) { + if (length > 0) { msg.append(state_ + sizeof(length) + CODE_WIDTH, length); } @@ -113,26 +113,19 @@ Status::ToString() const { std::string result; switch (code()) { - case DB_SUCCESS: - result = "OK "; + case DB_SUCCESS:result = "OK "; break; - case DB_ERROR: - result = "Error: "; + case DB_ERROR:result = "Error: "; break; - case DB_META_TRANSACTION_FAILED: - result = "Database error: "; + case DB_META_TRANSACTION_FAILED:result = "Database error: "; break; - case DB_NOT_FOUND: - result = "Not found: "; + case DB_NOT_FOUND:result = "Not found: "; break; - case DB_ALREADY_EXIST: - result = "Already exist: "; + case DB_ALREADY_EXIST:result = "Already exist: "; break; - case DB_INVALID_PATH: - result = "Invalid path: "; + case DB_INVALID_PATH:result = "Invalid path: "; break; - default: - result = "Error code(" + std::to_string(code()) + "): "; + default:result = "Error code(" + std::to_string(code()) + "): "; break; } diff --git a/cpp/src/utils/Status.h b/cpp/src/utils/Status.h index fe06c8029d..8f8f238979 100644 --- a/cpp/src/utils/Status.h +++ b/cpp/src/utils/Status.h @@ -44,14 +44,18 @@ class Status { operator=(Status &&s); static Status - OK() { return Status(); } + OK() { + return Status(); + } bool - ok() const { return state_ == nullptr || code() == 0; } + ok() const { + return state_ == nullptr || code() == 0; + } StatusCode code() const { - return (state_ == nullptr) ? 0 : *(StatusCode*)(state_); + return (state_ == nullptr) ? 0 : *(StatusCode *) (state_); } std::string @@ -60,14 +64,14 @@ class Status { std::string ToString() const; -private: + private: inline void CopyFrom(const Status &s); inline void MoveFrom(Status &s); -private: + private: const char *state_ = nullptr; }; // Status diff --git a/cpp/src/utils/StringHelpFunctions.cpp b/cpp/src/utils/StringHelpFunctions.cpp index 068c376ea0..8c9e888d3a 100644 --- a/cpp/src/utils/StringHelpFunctions.cpp +++ b/cpp/src/utils/StringHelpFunctions.cpp @@ -15,13 +15,16 @@ // specific language governing permissions and limitations // under the License. -#include "StringHelpFunctions.h" +#include "utils/StringHelpFunctions.h" + +#include namespace zilliz { namespace milvus { namespace server { -void StringHelpFunctions::TrimStringBlank(std::string &string) { +void +StringHelpFunctions::TrimStringBlank(std::string &string) { if (!string.empty()) { static std::string s_format(" \n\r\t"); string.erase(0, string.find_first_not_of(s_format)); @@ -29,17 +32,19 @@ void StringHelpFunctions::TrimStringBlank(std::string &string) { } } -void StringHelpFunctions::TrimStringQuote(std::string &string, const std::string &qoute) { +void +StringHelpFunctions::TrimStringQuote(std::string &string, const std::string &qoute) { if (!string.empty()) { string.erase(0, string.find_first_not_of(qoute)); string.erase(string.find_last_not_of(qoute) + 1); } } -Status StringHelpFunctions::SplitStringByDelimeter(const std::string &str, - const std::string &delimeter, - std::vector &result) { - if(str.empty()) { +Status +StringHelpFunctions::SplitStringByDelimeter(const std::string &str, + const std::string &delimeter, + std::vector &result) { + if (str.empty()) { return Status::OK(); } @@ -58,10 +63,11 @@ Status StringHelpFunctions::SplitStringByDelimeter(const std::string &str, return Status::OK(); } -Status StringHelpFunctions::SplitStringByQuote(const std::string &str, - const std::string &delimeter, - const std::string "e, - std::vector &result) { +Status +StringHelpFunctions::SplitStringByQuote(const std::string &str, + const std::string &delimeter, + const std::string "e, + std::vector &result) { if (quote.empty()) { return SplitStringByDelimeter(str, delimeter, result); } @@ -120,6 +126,6 @@ Status StringHelpFunctions::SplitStringByQuote(const std::string &str, return Status::OK(); } -} -} -} \ No newline at end of file +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/StringHelpFunctions.h b/cpp/src/utils/StringHelpFunctions.h index d50b62d8a3..ebffa075ad 100644 --- a/cpp/src/utils/StringHelpFunctions.h +++ b/cpp/src/utils/StringHelpFunctions.h @@ -20,16 +20,17 @@ #include "utils/Status.h" #include +#include namespace zilliz { namespace milvus { namespace server { class StringHelpFunctions { -private: + private: StringHelpFunctions() = default; -public: + public: static void TrimStringBlank(std::string &string); static void TrimStringQuote(std::string &string, const std::string &qoute); @@ -56,9 +57,8 @@ public: const std::string &delimeter, const std::string "e, std::vector &result); - }; -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/ThreadPool.h b/cpp/src/utils/ThreadPool.h index 8060ee4cf6..0524f6206d 100644 --- a/cpp/src/utils/ThreadPool.h +++ b/cpp/src/utils/ThreadPool.h @@ -26,7 +26,7 @@ #include #include #include - +#include #define MAX_THREADS_NUM 32 @@ -34,8 +34,8 @@ namespace zilliz { namespace milvus { class ThreadPool { -public: - ThreadPool(size_t threads, size_t queue_size = 1000); + public: + explicit ThreadPool(size_t threads, size_t queue_size = 1000); template auto enqueue(F &&f, Args &&... args) @@ -43,7 +43,7 @@ public: ~ThreadPool(); -private: + private: // need to keep track of threads so we can join them std::vector workers_; @@ -60,53 +60,57 @@ private: bool stop; }; - // the constructor just launches some amount of workers inline ThreadPool::ThreadPool(size_t threads, size_t queue_size) - : max_queue_size_(queue_size), stop(false) { + : max_queue_size_(queue_size), stop(false) { for (size_t i = 0; i < threads; ++i) workers_.emplace_back( - [this] { - for (;;) { - std::function task; + [this] { + for (;;) { + std::function task; - { - std::unique_lock lock(this->queue_mutex_); - this->condition_.wait(lock, - [this] { return this->stop || !this->tasks_.empty(); }); - if (this->stop && this->tasks_.empty()) - return; - task = std::move(this->tasks_.front()); - this->tasks_.pop(); - } - this->condition_.notify_all(); - - task(); + { + std::unique_lock lock(this->queue_mutex_); + this->condition_.wait(lock, + [this] { + return this->stop || !this->tasks_.empty(); + }); + if (this->stop && this->tasks_.empty()) + return; + task = std::move(this->tasks_.front()); + this->tasks_.pop(); } + this->condition_.notify_all(); + + task(); } - ); + }); } // add new work item to the pool template -auto ThreadPool::enqueue(F &&f, Args &&... args) +auto +ThreadPool::enqueue(F &&f, Args &&... args) -> std::future::type> { using return_type = typename std::result_of::type; auto task = std::make_shared >( - std::bind(std::forward(f), std::forward(args)...) - ); + std::bind(std::forward(f), std::forward(args)...)); std::future res = task->get_future(); { std::unique_lock lock(queue_mutex_); this->condition_.wait(lock, - [this] { return this->tasks_.size() < max_queue_size_; }); + [this] { + return this->tasks_.size() < max_queue_size_; + }); // don't allow enqueueing after stopping the pool if (stop) throw std::runtime_error("enqueue on stopped ThreadPool"); - tasks_.emplace([task]() { (*task)(); }); + tasks_.emplace([task]() { + (*task)(); + }); } condition_.notify_all(); return res; @@ -119,10 +123,11 @@ inline ThreadPool::~ThreadPool() { stop = true; } condition_.notify_all(); - for (std::thread &worker: workers_) + for (std::thread &worker : workers_) { worker.join(); + } } -} -} +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/TimeRecorder.cpp b/cpp/src/utils/TimeRecorder.cpp index 6e4b5c1ab6..5246b35f13 100644 --- a/cpp/src/utils/TimeRecorder.cpp +++ b/cpp/src/utils/TimeRecorder.cpp @@ -15,10 +15,9 @@ // specific language governing permissions and limitations // under the License. -#include "TimeRecorder.h" +#include "utils/TimeRecorder.h" #include "utils/Log.h" - namespace zilliz { namespace milvus { @@ -100,5 +99,5 @@ TimeRecorder::ElapseFromBegin(const std::string &msg) { return span; } -} -} +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/TimeRecorder.h b/cpp/src/utils/TimeRecorder.h index c6efbaeb8a..2bb937e71f 100644 --- a/cpp/src/utils/TimeRecorder.h +++ b/cpp/src/utils/TimeRecorder.h @@ -20,14 +20,13 @@ #include #include - namespace zilliz { namespace milvus { class TimeRecorder { using stdclock = std::chrono::high_resolution_clock; -public: + public: TimeRecorder(const std::string &header, int64_t log_level = 1); @@ -39,15 +38,15 @@ public: static std::string GetTimeSpanStr(double span); -private: + private: void PrintTimeRecord(const std::string &msg, double span); -private: + private: std::string header_; stdclock::time_point start_; stdclock::time_point last_; int64_t log_level_; }; -} -} +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index c9ec2d046b..a6d83af1dc 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -16,16 +16,16 @@ // under the License. +#include "utils/ValidationUtil.h" #include "db/engine/ExecutionEngine.h" -#include "ValidationUtil.h" #include "Log.h" +#include #include #include #include #include - namespace zilliz { namespace milvus { namespace server { @@ -36,7 +36,6 @@ constexpr int32_t INDEX_FILE_SIZE_LIMIT = 4096; //index trigger size max = 4096 Status ValidationUtil::ValidateTableName(const std::string &table_name) { - // Table name shouldn't be empty. if (table_name.empty()) { std::string msg = "Empty table name"; @@ -78,8 +77,7 @@ ValidationUtil::ValidateTableDimension(int64_t dimension) { std::string msg = "Table dimension excceed the limitation: " + std::to_string(TABLE_DIMENSION_LIMIT); SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_VECTOR_DIMENSION, msg); - } - else { + } else { return Status::OK(); } } @@ -185,7 +183,6 @@ ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t &memory) { Status ValidationUtil::ValidateIpAddress(const std::string &ip_address) { - struct in_addr address; int result = inet_pton(AF_INET, ip_address.c_str(), &address); @@ -212,7 +209,7 @@ ValidationUtil::ValidateStringIsNumber(const std::string &str) { } try { int32_t value = std::stoi(str); - } catch(...) { + } catch (...) { return Status(SERVER_INVALID_ARGUMENT, "Invalid number"); } return Status::OK(); @@ -226,8 +223,7 @@ ValidationUtil::ValidateStringIsBool(const std::string &str) { s == "false" || s == "off" || s == "no" || s == "0" || s.empty()) { return Status::OK(); - } - else { + } else { return Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str); } } @@ -236,7 +232,7 @@ Status ValidationUtil::ValidateStringIsFloat(const std::string &str) { try { float val = std::stof(str); - } catch(...) { + } catch (...) { return Status(SERVER_INVALID_ARGUMENT, "Invalid float: " + str); } return Status::OK(); @@ -289,8 +285,7 @@ ValidationUtil::ValidateDbURI(const std::string &uri) { okay = false; } } - } - else { + } else { SERVER_LOG_ERROR << "Wrong URI format: URI = " << uri; okay = false; } @@ -298,6 +293,6 @@ ValidationUtil::ValidateDbURI(const std::string &uri) { return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid db backend uri")); } -} -} -} \ No newline at end of file +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/ValidationUtil.h b/cpp/src/utils/ValidationUtil.h index 789da0de40..44d6065a64 100644 --- a/cpp/src/utils/ValidationUtil.h +++ b/cpp/src/utils/ValidationUtil.h @@ -21,15 +21,17 @@ #include "db/meta/MetaTypes.h" #include "utils/Status.h" +#include + namespace zilliz { namespace milvus { namespace server { class ValidationUtil { -private: + private: ValidationUtil() = default; -public: + public: static Status ValidateTableName(const std::string &table_name); @@ -49,10 +51,10 @@ public: ValidateTableIndexMetricType(int32_t metric_type); static Status - ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema); + ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema &table_schema); static Status - ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema); + ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema &table_schema); static Status ValidateGpuIndex(uint32_t gpu_index); @@ -76,6 +78,6 @@ public: ValidateDbURI(const std::string &uri); }; -} -} -} \ No newline at end of file +} // namespace server +} // namespace milvus +} // namespace zilliz From 2b132a9cefe151967357a932e385aca2f0590eb3 Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 26 Sep 2019 20:25:57 +0800 Subject: [PATCH 124/217] format server code Former-commit-id: 5502293863810d216a6c8c46a28eb76bfa30bd12 --- cpp/src/server/Config.cpp | 134 ++++---- cpp/src/server/Config.h | 273 ++++++++-------- cpp/src/server/DBWrapper.cpp | 44 +-- cpp/src/server/DBWrapper.h | 14 +- cpp/src/server/Server.cpp | 20 +- cpp/src/server/Server.h | 7 +- .../server/grpc_impl/GrpcRequestHandler.cpp | 57 ++-- cpp/src/server/grpc_impl/GrpcRequestHandler.h | 43 ++- .../server/grpc_impl/GrpcRequestScheduler.cpp | 139 ++++---- .../server/grpc_impl/GrpcRequestScheduler.h | 42 ++- cpp/src/server/grpc_impl/GrpcRequestTask.cpp | 296 +++++++++--------- cpp/src/server/grpc_impl/GrpcRequestTask.h | 130 ++++---- cpp/src/server/grpc_impl/GrpcServer.cpp | 19 +- cpp/src/server/grpc_impl/GrpcServer.h | 10 +- 14 files changed, 608 insertions(+), 620 deletions(-) diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 8ee2f7c62b..0dbf4e1dd2 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "Config.h" +#include "server/Config.h" #include #include @@ -23,12 +23,13 @@ #include #include #include +#include +#include #include "config/ConfigMgr.h" #include "utils/CommonUtil.h" #include "utils/ValidationUtil.h" - namespace zilliz { namespace milvus { namespace server { @@ -71,11 +72,11 @@ Config::LoadConfigFile(const std::string &filename) { } void -Config::PrintConfigSection(const std::string& config_node_name) { +Config::PrintConfigSection(const std::string &config_node_name) { std::cout << std::endl; std::cout << config_node_name << ":" << std::endl; if (config_map_.find(config_node_name) != config_map_.end()) { - for (auto item: config_map_[config_node_name]) { + for (auto item : config_map_[config_node_name]) { std::cout << item.first << ": " << item.second << std::endl; } } @@ -182,7 +183,7 @@ Config::CheckDBConfigBufferSize(const std::string &value) { return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config buffer_size: " + value); } else { int64_t buffer_size = std::stoi(value) * GB; - unsigned long total_mem = 0, free_mem = 0; + uint64_t total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (buffer_size >= total_mem) { return Status(SERVER_INVALID_ARGUMENT, "DB config buffer_size exceed system memory: " + value); @@ -205,7 +206,7 @@ Config::CheckDBConfigBuildIndexGPU(const std::string &value) { } Status -Config::CheckMetricConfigAutoBootup(const std::string& value) { +Config::CheckMetricConfigAutoBootup(const std::string &value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config auto_bootup: " + value); } @@ -213,7 +214,7 @@ Config::CheckMetricConfigAutoBootup(const std::string& value) { } Status -Config::CheckMetricConfigCollector(const std::string& value) { +Config::CheckMetricConfigCollector(const std::string &value) { if (value != "prometheus") { return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config collector: " + value); } @@ -221,7 +222,7 @@ Config::CheckMetricConfigCollector(const std::string& value) { } Status -Config::CheckMetricConfigPrometheusPort(const std::string& value) { +Config::CheckMetricConfigPrometheusPort(const std::string &value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config prometheus_port: " + value); } @@ -229,12 +230,12 @@ Config::CheckMetricConfigPrometheusPort(const std::string& value) { } Status -Config::CheckCacheConfigCpuMemCapacity(const std::string& value) { +Config::CheckCacheConfigCpuMemCapacity(const std::string &value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_mem_capacity: " + value); } else { uint64_t cpu_cache_capacity = std::stoi(value) * GB; - unsigned long total_mem = 0, free_mem = 0; + uint64_t total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (cpu_cache_capacity >= total_mem) { return Status(SERVER_INVALID_ARGUMENT, "Cache config cpu_mem_capacity exceed system memory: " + value); @@ -254,7 +255,7 @@ Config::CheckCacheConfigCpuMemCapacity(const std::string& value) { } Status -Config::CheckCacheConfigCpuMemThreshold(const std::string& value) { +Config::CheckCacheConfigCpuMemThreshold(const std::string &value) { if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_mem_threshold: " + value); } else { @@ -267,7 +268,7 @@ Config::CheckCacheConfigCpuMemThreshold(const std::string& value) { } Status -Config::CheckCacheConfigGpuMemCapacity(const std::string& value) { +Config::CheckCacheConfigGpuMemCapacity(const std::string &value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { std::cerr << "ERROR: gpu_cache_capacity " << value << " is not a number" << std::endl; } else { @@ -290,7 +291,7 @@ Config::CheckCacheConfigGpuMemCapacity(const std::string& value) { } Status -Config::CheckCacheConfigGpuMemThreshold(const std::string& value) { +Config::CheckCacheConfigGpuMemThreshold(const std::string &value) { if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_mem_threshold: " + value); } else { @@ -303,7 +304,7 @@ Config::CheckCacheConfigGpuMemThreshold(const std::string& value) { } Status -Config::CheckCacheConfigCacheInsertData(const std::string& value) { +Config::CheckCacheConfigCacheInsertData(const std::string &value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cache_insert_data: " + value); } @@ -311,7 +312,7 @@ Config::CheckCacheConfigCacheInsertData(const std::string& value) { } Status -Config::CheckEngineConfigBlasThreshold(const std::string& value) { +Config::CheckEngineConfigBlasThreshold(const std::string &value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config blas threshold: " + value); } @@ -319,7 +320,7 @@ Config::CheckEngineConfigBlasThreshold(const std::string& value) { } Status -Config::CheckEngineConfigOmpThreadNum(const std::string& value) { +Config::CheckEngineConfigOmpThreadNum(const std::string &value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config omp_thread_num: " + value); } else { @@ -333,7 +334,7 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) { } Status -Config::CheckResourceConfigMode(const std::string& value) { +Config::CheckResourceConfigMode(const std::string &value) { if (value != "simple") { return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config mode: " + value); } @@ -341,7 +342,7 @@ Config::CheckResourceConfigMode(const std::string& value) { } Status -Config::CheckResourceConfigPool(const std::vector& value) { +Config::CheckResourceConfigPool(const std::vector &value) { if (value.empty()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config pool"); } @@ -632,52 +633,51 @@ Config::GetResourceConfigStrMode() { return value; } - //////////////////////////////////////////////////////////////////////////////// Status -Config::GetServerConfigAddress(std::string& value) { +Config::GetServerConfigAddress(std::string &value) { value = GetServerConfigStrAddress(); return CheckServerConfigAddress(value); } Status -Config::GetServerConfigPort(std::string& value) { +Config::GetServerConfigPort(std::string &value) { value = GetServerConfigStrPort(); return CheckServerConfigPort(value); } Status -Config::GetServerConfigMode(std::string& value) { +Config::GetServerConfigMode(std::string &value) { value = GetServerConfigStrMode(); return CheckServerConfigMode(value); } Status -Config::GetServerConfigTimeZone(std::string& value) { +Config::GetServerConfigTimeZone(std::string &value) { value = GetServerConfigStrTimeZone(); return CheckServerConfigTimeZone(value); } Status -Config::GetDBConfigPath(std::string& value) { +Config::GetDBConfigPath(std::string &value) { value = GetDBConfigStrPath(); return CheckDBConfigPath(value); } Status -Config::GetDBConfigSlavePath(std::string& value) { +Config::GetDBConfigSlavePath(std::string &value) { value = GetDBConfigStrSlavePath(); return Status::OK(); } Status -Config::GetDBConfigBackendUrl(std::string& value) { +Config::GetDBConfigBackendUrl(std::string &value) { value = GetDBConfigStrBackendUrl(); return CheckDBConfigBackendUrl(value); } Status -Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { +Config::GetDBConfigArchiveDiskThreshold(int32_t &value) { std::string str = GetDBConfigStrArchiveDiskThreshold(); Status s = CheckDBConfigArchiveDiskThreshold(str); if (!s.ok()) return s; @@ -686,7 +686,7 @@ Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { } Status -Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { +Config::GetDBConfigArchiveDaysThreshold(int32_t &value) { std::string str = GetDBConfigStrArchiveDaysThreshold(); Status s = CheckDBConfigArchiveDaysThreshold(str); if (!s.ok()) return s; @@ -695,7 +695,7 @@ Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { } Status -Config::GetDBConfigBufferSize(int32_t& value) { +Config::GetDBConfigBufferSize(int32_t &value) { std::string str = GetDBConfigStrBufferSize(); Status s = CheckDBConfigBufferSize(str); if (!s.ok()) return s; @@ -704,7 +704,7 @@ Config::GetDBConfigBufferSize(int32_t& value) { } Status -Config::GetDBConfigBuildIndexGPU(int32_t& value) { +Config::GetDBConfigBuildIndexGPU(int32_t &value) { std::string str = GetDBConfigStrBuildIndexGPU(); Status s = CheckDBConfigBuildIndexGPU(str); if (!s.ok()) return s; @@ -713,7 +713,7 @@ Config::GetDBConfigBuildIndexGPU(int32_t& value) { } Status -Config::GetMetricConfigAutoBootup(bool& value) { +Config::GetMetricConfigAutoBootup(bool &value) { std::string str = GetMetricConfigStrAutoBootup(); Status s = CheckMetricConfigPrometheusPort(str); if (!s.ok()) return s; @@ -723,19 +723,19 @@ Config::GetMetricConfigAutoBootup(bool& value) { } Status -Config::GetMetricConfigCollector(std::string& value) { +Config::GetMetricConfigCollector(std::string &value) { value = GetMetricConfigStrCollector(); return Status::OK(); } Status -Config::GetMetricConfigPrometheusPort(std::string& value) { +Config::GetMetricConfigPrometheusPort(std::string &value) { value = GetMetricConfigStrPrometheusPort(); return CheckMetricConfigPrometheusPort(value); } Status -Config::GetCacheConfigCpuMemCapacity(int32_t& value) { +Config::GetCacheConfigCpuMemCapacity(int32_t &value) { std::string str = GetCacheConfigStrCpuMemCapacity(); Status s = CheckCacheConfigCpuMemCapacity(str); if (!s.ok()) return s; @@ -744,7 +744,7 @@ Config::GetCacheConfigCpuMemCapacity(int32_t& value) { } Status -Config::GetCacheConfigCpuMemThreshold(float& value) { +Config::GetCacheConfigCpuMemThreshold(float &value) { std::string str = GetCacheConfigStrCpuMemThreshold(); Status s = CheckCacheConfigCpuMemThreshold(str); if (!s.ok()) return s; @@ -753,7 +753,7 @@ Config::GetCacheConfigCpuMemThreshold(float& value) { } Status -Config::GetCacheConfigGpuMemCapacity(int32_t& value) { +Config::GetCacheConfigGpuMemCapacity(int32_t &value) { std::string str = GetCacheConfigStrGpuMemCapacity(); Status s = CheckCacheConfigGpuMemCapacity(str); if (!s.ok()) return s; @@ -762,7 +762,7 @@ Config::GetCacheConfigGpuMemCapacity(int32_t& value) { } Status -Config::GetCacheConfigGpuMemThreshold(float& value) { +Config::GetCacheConfigGpuMemThreshold(float &value) { std::string str = GetCacheConfigStrGpuMemThreshold(); Status s = CheckCacheConfigGpuMemThreshold(str); if (!s.ok()) return s; @@ -771,7 +771,7 @@ Config::GetCacheConfigGpuMemThreshold(float& value) { } Status -Config::GetCacheConfigCacheInsertData(bool& value) { +Config::GetCacheConfigCacheInsertData(bool &value) { std::string str = GetCacheConfigStrCacheInsertData(); Status s = CheckCacheConfigCacheInsertData(str); if (!s.ok()) return s; @@ -781,7 +781,7 @@ Config::GetCacheConfigCacheInsertData(bool& value) { } Status -Config::GetEngineConfigBlasThreshold(int32_t& value) { +Config::GetEngineConfigBlasThreshold(int32_t &value) { std::string str = GetEngineConfigStrBlasThreshold(); Status s = CheckEngineConfigBlasThreshold(str); if (!s.ok()) return s; @@ -790,7 +790,7 @@ Config::GetEngineConfigBlasThreshold(int32_t& value) { } Status -Config::GetEngineConfigOmpThreadNum(int32_t& value) { +Config::GetEngineConfigOmpThreadNum(int32_t &value) { std::string str = GetEngineConfigStrOmpThreadNum(); Status s = CheckEngineConfigOmpThreadNum(str); if (!s.ok()) return s; @@ -799,13 +799,13 @@ Config::GetEngineConfigOmpThreadNum(int32_t& value) { } Status -Config::GetResourceConfigMode(std::string& value) { +Config::GetResourceConfigMode(std::string &value) { value = GetResourceConfigStrMode(); return CheckResourceConfigMode(value); } Status -Config::GetResourceConfigPool(std::vector& value) { +Config::GetResourceConfigPool(std::vector &value) { ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE); value = resource_config.GetSequence(CONFIG_RESOURCE_POOL); return CheckResourceConfigPool(value); @@ -814,7 +814,7 @@ Config::GetResourceConfigPool(std::vector& value) { /////////////////////////////////////////////////////////////////////////////// /* server config */ Status -Config::SetServerConfigAddress(const std::string& value) { +Config::SetServerConfigAddress(const std::string &value) { Status s = CheckServerConfigAddress(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); @@ -822,7 +822,7 @@ Config::SetServerConfigAddress(const std::string& value) { } Status -Config::SetServerConfigPort(const std::string& value) { +Config::SetServerConfigPort(const std::string &value) { Status s = CheckServerConfigPort(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); @@ -830,7 +830,7 @@ Config::SetServerConfigPort(const std::string& value) { } Status -Config::SetServerConfigMode(const std::string& value) { +Config::SetServerConfigMode(const std::string &value) { Status s = CheckServerConfigMode(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); @@ -838,7 +838,7 @@ Config::SetServerConfigMode(const std::string& value) { } Status -Config::SetServerConfigTimeZone(const std::string& value) { +Config::SetServerConfigTimeZone(const std::string &value) { Status s = CheckServerConfigTimeZone(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); @@ -847,7 +847,7 @@ Config::SetServerConfigTimeZone(const std::string& value) { /* db config */ Status -Config::SetDBConfigPath(const std::string& value) { +Config::SetDBConfigPath(const std::string &value) { Status s = CheckDBConfigPath(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); @@ -855,7 +855,7 @@ Config::SetDBConfigPath(const std::string& value) { } Status -Config::SetDBConfigSlavePath(const std::string& value) { +Config::SetDBConfigSlavePath(const std::string &value) { Status s = CheckDBConfigSlavePath(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); @@ -863,7 +863,7 @@ Config::SetDBConfigSlavePath(const std::string& value) { } Status -Config::SetDBConfigBackendUrl(const std::string& value) { +Config::SetDBConfigBackendUrl(const std::string &value) { Status s = CheckDBConfigBackendUrl(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); @@ -871,7 +871,7 @@ Config::SetDBConfigBackendUrl(const std::string& value) { } Status -Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { +Config::SetDBConfigArchiveDiskThreshold(const std::string &value) { Status s = CheckDBConfigArchiveDiskThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); @@ -879,7 +879,7 @@ Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { } Status -Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { +Config::SetDBConfigArchiveDaysThreshold(const std::string &value) { Status s = CheckDBConfigArchiveDaysThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); @@ -887,7 +887,7 @@ Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { } Status -Config::SetDBConfigBufferSize(const std::string& value) { +Config::SetDBConfigBufferSize(const std::string &value) { Status s = CheckDBConfigBufferSize(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); @@ -895,7 +895,7 @@ Config::SetDBConfigBufferSize(const std::string& value) { } Status -Config::SetDBConfigBuildIndexGPU(const std::string& value) { +Config::SetDBConfigBuildIndexGPU(const std::string &value) { Status s = CheckDBConfigBuildIndexGPU(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); @@ -904,7 +904,7 @@ Config::SetDBConfigBuildIndexGPU(const std::string& value) { /* metric config */ Status -Config::SetMetricConfigAutoBootup(const std::string& value) { +Config::SetMetricConfigAutoBootup(const std::string &value) { Status s = CheckMetricConfigAutoBootup(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_AUTO_BOOTUP, value); @@ -912,7 +912,7 @@ Config::SetMetricConfigAutoBootup(const std::string& value) { } Status -Config::SetMetricConfigCollector(const std::string& value) { +Config::SetMetricConfigCollector(const std::string &value) { Status s = CheckMetricConfigCollector(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_COLLECTOR, value); @@ -920,7 +920,7 @@ Config::SetMetricConfigCollector(const std::string& value) { } Status -Config::SetMetricConfigPrometheusPort(const std::string& value) { +Config::SetMetricConfigPrometheusPort(const std::string &value) { Status s = CheckMetricConfigPrometheusPort(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_PROMETHEUS_PORT, value); @@ -929,7 +929,7 @@ Config::SetMetricConfigPrometheusPort(const std::string& value) { /* cache config */ Status -Config::SetCacheConfigCpuMemCapacity(const std::string& value) { +Config::SetCacheConfigCpuMemCapacity(const std::string &value) { Status s = CheckCacheConfigCpuMemCapacity(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_MEM_CAPACITY, value); @@ -937,7 +937,7 @@ Config::SetCacheConfigCpuMemCapacity(const std::string& value) { } Status -Config::SetCacheConfigCpuMemThreshold(const std::string& value) { +Config::SetCacheConfigCpuMemThreshold(const std::string &value) { Status s = CheckCacheConfigCpuMemThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_MEM_THRESHOLD, value); @@ -945,7 +945,7 @@ Config::SetCacheConfigCpuMemThreshold(const std::string& value) { } Status -Config::SetCacheConfigGpuMemCapacity(const std::string& value) { +Config::SetCacheConfigGpuMemCapacity(const std::string &value) { Status s = CheckCacheConfigGpuMemCapacity(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_MEM_CAPACITY, value); @@ -953,7 +953,7 @@ Config::SetCacheConfigGpuMemCapacity(const std::string& value) { } Status -Config::SetCacheConfigGpuMemThreshold(const std::string& value) { +Config::SetCacheConfigGpuMemThreshold(const std::string &value) { Status s = CheckCacheConfigGpuMemThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_MEM_THRESHOLD, value); @@ -961,7 +961,7 @@ Config::SetCacheConfigGpuMemThreshold(const std::string& value) { } Status -Config::SetCacheConfigCacheInsertData(const std::string& value) { +Config::SetCacheConfigCacheInsertData(const std::string &value) { Status s = CheckCacheConfigCacheInsertData(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CACHE_INSERT_DATA, value); @@ -970,7 +970,7 @@ Config::SetCacheConfigCacheInsertData(const std::string& value) { /* engine config */ Status -Config::SetEngineConfigBlasThreshold(const std::string& value) { +Config::SetEngineConfigBlasThreshold(const std::string &value) { Status s = CheckEngineConfigBlasThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_BLAS_THRESHOLD, value); @@ -978,7 +978,7 @@ Config::SetEngineConfigBlasThreshold(const std::string& value) { } Status -Config::SetEngineConfigOmpThreadNum(const std::string& value) { +Config::SetEngineConfigOmpThreadNum(const std::string &value) { Status s = CheckEngineConfigOmpThreadNum(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_OMP_THREAD_NUM, value); @@ -987,13 +987,13 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) { /* resource config */ Status -Config::SetResourceConfigMode(const std::string& value) { +Config::SetResourceConfigMode(const std::string &value) { Status s = CheckResourceConfigMode(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_MODE, value); return Status::OK(); } -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index 956c08100f..f2d1b96999 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -17,136 +17,137 @@ #pragma once +#include +#include #include #include -#include "yaml-cpp/yaml.h" +#include + #include "utils/Status.h" #include "config/ConfigNode.h" - namespace zilliz { namespace milvus { namespace server { /* server config */ -static const char* CONFIG_SERVER = "server_config"; -static const char* CONFIG_SERVER_ADDRESS = "address"; -static const char* CONFIG_SERVER_ADDRESS_DEFAULT = "127.0.0.1"; -static const char* CONFIG_SERVER_PORT = "port"; -static const char* CONFIG_SERVER_PORT_DEFAULT = "19530"; -static const char* CONFIG_SERVER_MODE = "mode"; -static const char* CONFIG_SERVER_MODE_DEFAULT = "single"; -static const char* CONFIG_SERVER_TIME_ZONE = "time_zone"; -static const char* CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; +static const char *CONFIG_SERVER = "server_config"; +static const char *CONFIG_SERVER_ADDRESS = "address"; +static const char *CONFIG_SERVER_ADDRESS_DEFAULT = "127.0.0.1"; +static const char *CONFIG_SERVER_PORT = "port"; +static const char *CONFIG_SERVER_PORT_DEFAULT = "19530"; +static const char *CONFIG_SERVER_MODE = "mode"; +static const char *CONFIG_SERVER_MODE_DEFAULT = "single"; +static const char *CONFIG_SERVER_TIME_ZONE = "time_zone"; +static const char *CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; /* db config */ -static const char* CONFIG_DB = "db_config"; -static const char* CONFIG_DB_PATH = "path"; -static const char* CONFIG_DB_PATH_DEFAULT = "/tmp/milvus"; -static const char* CONFIG_DB_SLAVE_PATH = "slave_path"; -static const char* CONFIG_DB_SLAVE_PATH_DEFAULT = ""; -static const char* CONFIG_DB_BACKEND_URL = "backend_url"; -static const char* CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/"; -static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold"; -static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0"; -static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold"; -static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; -static const char* CONFIG_DB_BUFFER_SIZE = "buffer_size"; -static const char* CONFIG_DB_BUFFER_SIZE_DEFAULT = "4"; -static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu"; -static const char* CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0"; +static const char *CONFIG_DB = "db_config"; +static const char *CONFIG_DB_PATH = "path"; +static const char *CONFIG_DB_PATH_DEFAULT = "/tmp/milvus"; +static const char *CONFIG_DB_SLAVE_PATH = "slave_path"; +static const char *CONFIG_DB_SLAVE_PATH_DEFAULT = ""; +static const char *CONFIG_DB_BACKEND_URL = "backend_url"; +static const char *CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/"; +static const char *CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold"; +static const char *CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0"; +static const char *CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold"; +static const char *CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; +static const char *CONFIG_DB_BUFFER_SIZE = "buffer_size"; +static const char *CONFIG_DB_BUFFER_SIZE_DEFAULT = "4"; +static const char *CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu"; +static const char *CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0"; /* cache config */ -static const char* CONFIG_CACHE = "cache_config"; -static const char* CONFIG_CACHE_CPU_MEM_CAPACITY = "cpu_mem_capacity"; -static const char* CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT = "16"; -static const char* CONFIG_CACHE_GPU_MEM_CAPACITY = "gpu_mem_capacity"; -static const char* CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT = "0"; -static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD = "cpu_mem_threshold"; -static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT = "0.85"; -static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD = "gpu_mem_threshold"; -static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT = "0.85"; -static const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; -static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; +static const char *CONFIG_CACHE = "cache_config"; +static const char *CONFIG_CACHE_CPU_MEM_CAPACITY = "cpu_mem_capacity"; +static const char *CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT = "16"; +static const char *CONFIG_CACHE_GPU_MEM_CAPACITY = "gpu_mem_capacity"; +static const char *CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT = "0"; +static const char *CONFIG_CACHE_CPU_MEM_THRESHOLD = "cpu_mem_threshold"; +static const char *CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT = "0.85"; +static const char *CONFIG_CACHE_GPU_MEM_THRESHOLD = "gpu_mem_threshold"; +static const char *CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT = "0.85"; +static const char *CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; +static const char *CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; /* metric config */ -static const char* CONFIG_METRIC = "metric_config"; -static const char* CONFIG_METRIC_AUTO_BOOTUP = "auto_bootup"; -static const char* CONFIG_METRIC_AUTO_BOOTUP_DEFAULT = "false"; -static const char* CONFIG_METRIC_COLLECTOR = "collector"; -static const char* CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus"; -static const char* CONFIG_METRIC_PROMETHEUS = "prometheus_config"; -static const char* CONFIG_METRIC_PROMETHEUS_PORT = "port"; -static const char* CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT = "8080"; +static const char *CONFIG_METRIC = "metric_config"; +static const char *CONFIG_METRIC_AUTO_BOOTUP = "auto_bootup"; +static const char *CONFIG_METRIC_AUTO_BOOTUP_DEFAULT = "false"; +static const char *CONFIG_METRIC_COLLECTOR = "collector"; +static const char *CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus"; +static const char *CONFIG_METRIC_PROMETHEUS = "prometheus_config"; +static const char *CONFIG_METRIC_PROMETHEUS_PORT = "port"; +static const char *CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT = "8080"; /* engine config */ -static const char* CONFIG_ENGINE = "engine_config"; -static const char* CONFIG_ENGINE_BLAS_THRESHOLD = "blas_threshold"; -static const char* CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT = "20"; -static const char* CONFIG_ENGINE_OMP_THREAD_NUM = "omp_thread_num"; -static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; +static const char *CONFIG_ENGINE = "engine_config"; +static const char *CONFIG_ENGINE_BLAS_THRESHOLD = "blas_threshold"; +static const char *CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT = "20"; +static const char *CONFIG_ENGINE_OMP_THREAD_NUM = "omp_thread_num"; +static const char *CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; /* resource config */ -static const char* CONFIG_RESOURCE = "resource_config"; -static const char* CONFIG_RESOURCE_MODE = "mode"; -static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; -static const char* CONFIG_RESOURCE_POOL = "pool"; - +static const char *CONFIG_RESOURCE = "resource_config"; +static const char *CONFIG_RESOURCE_MODE = "mode"; +static const char *CONFIG_RESOURCE_MODE_DEFAULT = "simple"; +static const char *CONFIG_RESOURCE_POOL = "pool"; class Config { public: - static Config& GetInstance(); - Status LoadConfigFile(const std::string& filename); + static Config &GetInstance(); + Status LoadConfigFile(const std::string &filename); void PrintAll(); private: - ConfigNode& GetConfigNode(const std::string& name); + ConfigNode &GetConfigNode(const std::string &name); - Status GetConfigValueInMem(const std::string& parent_key, - const std::string& child_key, - std::string& value); + Status GetConfigValueInMem(const std::string &parent_key, + const std::string &child_key, + std::string &value); - void SetConfigValueInMem(const std::string& parent_key, - const std::string& child_key, - const std::string& value); + void SetConfigValueInMem(const std::string &parent_key, + const std::string &child_key, + const std::string &value); - void PrintConfigSection(const std::string& config_node_name); + void PrintConfigSection(const std::string &config_node_name); /////////////////////////////////////////////////////////////////////////// /* server config */ - Status CheckServerConfigAddress(const std::string& value); - Status CheckServerConfigPort(const std::string& value); - Status CheckServerConfigMode(const std::string& value); - Status CheckServerConfigTimeZone(const std::string& value); + Status CheckServerConfigAddress(const std::string &value); + Status CheckServerConfigPort(const std::string &value); + Status CheckServerConfigMode(const std::string &value); + Status CheckServerConfigTimeZone(const std::string &value); /* db config */ - Status CheckDBConfigPath(const std::string& value); - Status CheckDBConfigSlavePath(const std::string& value); - Status CheckDBConfigBackendUrl(const std::string& value); - Status CheckDBConfigArchiveDiskThreshold(const std::string& value); - Status CheckDBConfigArchiveDaysThreshold(const std::string& value); - Status CheckDBConfigBufferSize(const std::string& value); - Status CheckDBConfigBuildIndexGPU(const std::string& value); + Status CheckDBConfigPath(const std::string &value); + Status CheckDBConfigSlavePath(const std::string &value); + Status CheckDBConfigBackendUrl(const std::string &value); + Status CheckDBConfigArchiveDiskThreshold(const std::string &value); + Status CheckDBConfigArchiveDaysThreshold(const std::string &value); + Status CheckDBConfigBufferSize(const std::string &value); + Status CheckDBConfigBuildIndexGPU(const std::string &value); /* metric config */ - Status CheckMetricConfigAutoBootup(const std::string& value); - Status CheckMetricConfigCollector(const std::string& value); - Status CheckMetricConfigPrometheusPort(const std::string& value); + Status CheckMetricConfigAutoBootup(const std::string &value); + Status CheckMetricConfigCollector(const std::string &value); + Status CheckMetricConfigPrometheusPort(const std::string &value); /* cache config */ - Status CheckCacheConfigCpuMemCapacity(const std::string& value); - Status CheckCacheConfigCpuMemThreshold(const std::string& value); - Status CheckCacheConfigGpuMemCapacity(const std::string& value); - Status CheckCacheConfigGpuMemThreshold(const std::string& value); - Status CheckCacheConfigCacheInsertData(const std::string& value); + Status CheckCacheConfigCpuMemCapacity(const std::string &value); + Status CheckCacheConfigCpuMemThreshold(const std::string &value); + Status CheckCacheConfigGpuMemCapacity(const std::string &value); + Status CheckCacheConfigGpuMemThreshold(const std::string &value); + Status CheckCacheConfigCacheInsertData(const std::string &value); /* engine config */ - Status CheckEngineConfigBlasThreshold(const std::string& value); - Status CheckEngineConfigOmpThreadNum(const std::string& value); + Status CheckEngineConfigBlasThreshold(const std::string &value); + Status CheckEngineConfigOmpThreadNum(const std::string &value); /* resource config */ - Status CheckResourceConfigMode(const std::string& value); - Status CheckResourceConfigPool(const std::vector& value); + Status CheckResourceConfigMode(const std::string &value); + Status CheckResourceConfigPool(const std::vector &value); /////////////////////////////////////////////////////////////////////////// /* server config */ @@ -185,81 +186,81 @@ class Config { public: /* server config */ - Status GetServerConfigAddress(std::string& value); - Status GetServerConfigPort(std::string& value); - Status GetServerConfigMode(std::string& value); - Status GetServerConfigTimeZone(std::string& value); + Status GetServerConfigAddress(std::string &value); + Status GetServerConfigPort(std::string &value); + Status GetServerConfigMode(std::string &value); + Status GetServerConfigTimeZone(std::string &value); /* db config */ - Status GetDBConfigPath(std::string& value); - Status GetDBConfigSlavePath(std::string& value); - Status GetDBConfigBackendUrl(std::string& value); - Status GetDBConfigArchiveDiskThreshold(int32_t& value); - Status GetDBConfigArchiveDaysThreshold(int32_t& value); - Status GetDBConfigBufferSize(int32_t& value); - Status GetDBConfigBuildIndexGPU(int32_t& value); + Status GetDBConfigPath(std::string &value); + Status GetDBConfigSlavePath(std::string &value); + Status GetDBConfigBackendUrl(std::string &value); + Status GetDBConfigArchiveDiskThreshold(int32_t &value); + Status GetDBConfigArchiveDaysThreshold(int32_t &value); + Status GetDBConfigBufferSize(int32_t &value); + Status GetDBConfigBuildIndexGPU(int32_t &value); /* metric config */ - Status GetMetricConfigAutoBootup(bool& value); - Status GetMetricConfigCollector(std::string& value); - Status GetMetricConfigPrometheusPort(std::string& value); + Status GetMetricConfigAutoBootup(bool &value); + Status GetMetricConfigCollector(std::string &value); + Status GetMetricConfigPrometheusPort(std::string &value); /* cache config */ - Status GetCacheConfigCpuMemCapacity(int32_t& value); - Status GetCacheConfigCpuMemThreshold(float& value); - Status GetCacheConfigGpuMemCapacity(int32_t& value); - Status GetCacheConfigGpuMemThreshold(float& value); - Status GetCacheConfigCacheInsertData(bool& value); + Status GetCacheConfigCpuMemCapacity(int32_t &value); + Status GetCacheConfigCpuMemThreshold(float &value); + Status GetCacheConfigGpuMemCapacity(int32_t &value); + Status GetCacheConfigGpuMemThreshold(float &value); + Status GetCacheConfigCacheInsertData(bool &value); /* engine config */ - Status GetEngineConfigBlasThreshold(int32_t& value); - Status GetEngineConfigOmpThreadNum(int32_t& value); + Status GetEngineConfigBlasThreshold(int32_t &value); + Status GetEngineConfigOmpThreadNum(int32_t &value); /* resource config */ - Status GetResourceConfigMode(std::string& value); - Status GetResourceConfigPool(std::vector& value); + Status GetResourceConfigMode(std::string &value); + Status GetResourceConfigPool(std::vector &value); public: /* server config */ - Status SetServerConfigAddress(const std::string& value); - Status SetServerConfigPort(const std::string& value); - Status SetServerConfigMode(const std::string& value); - Status SetServerConfigTimeZone(const std::string& value); + Status SetServerConfigAddress(const std::string &value); + Status SetServerConfigPort(const std::string &value); + Status SetServerConfigMode(const std::string &value); + Status SetServerConfigTimeZone(const std::string &value); /* db config */ - Status SetDBConfigPath(const std::string& value); - Status SetDBConfigSlavePath(const std::string& value); - Status SetDBConfigBackendUrl(const std::string& value); - Status SetDBConfigArchiveDiskThreshold(const std::string& value); - Status SetDBConfigArchiveDaysThreshold(const std::string& value); - Status SetDBConfigBufferSize(const std::string& value); - Status SetDBConfigBuildIndexGPU(const std::string& value); + Status SetDBConfigPath(const std::string &value); + Status SetDBConfigSlavePath(const std::string &value); + Status SetDBConfigBackendUrl(const std::string &value); + Status SetDBConfigArchiveDiskThreshold(const std::string &value); + Status SetDBConfigArchiveDaysThreshold(const std::string &value); + Status SetDBConfigBufferSize(const std::string &value); + Status SetDBConfigBuildIndexGPU(const std::string &value); /* metric config */ - Status SetMetricConfigAutoBootup(const std::string& value); - Status SetMetricConfigCollector(const std::string& value); - Status SetMetricConfigPrometheusPort(const std::string& value); + Status SetMetricConfigAutoBootup(const std::string &value); + Status SetMetricConfigCollector(const std::string &value); + Status SetMetricConfigPrometheusPort(const std::string &value); /* cache config */ - Status SetCacheConfigCpuMemCapacity(const std::string& value); - Status SetCacheConfigCpuMemThreshold(const std::string& value); - Status SetCacheConfigGpuMemCapacity(const std::string& value); - Status SetCacheConfigGpuMemThreshold(const std::string& value); - Status SetCacheConfigCacheInsertData(const std::string& value); + Status SetCacheConfigCpuMemCapacity(const std::string &value); + Status SetCacheConfigCpuMemThreshold(const std::string &value); + Status SetCacheConfigGpuMemCapacity(const std::string &value); + Status SetCacheConfigGpuMemThreshold(const std::string &value); + Status SetCacheConfigCacheInsertData(const std::string &value); /* engine config */ - Status SetEngineConfigBlasThreshold(const std::string& value); - Status SetEngineConfigOmpThreadNum(const std::string& value); + Status SetEngineConfigBlasThreshold(const std::string &value); + Status SetEngineConfigOmpThreadNum(const std::string &value); /* resource config */ - Status SetResourceConfigMode(const std::string& value); + Status SetResourceConfigMode(const std::string &value); private: std::unordered_map> config_map_; std::mutex mutex_; }; -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 16b13a1249..b142f31d10 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -16,13 +16,14 @@ // under the License. -#include "DBWrapper.h" +#include "server/DBWrapper.h" #include "Config.h" #include "db/DBFactory.h" #include "utils/CommonUtil.h" #include "utils/Log.h" #include "utils/StringHelpFunctions.h" +#include #include #include @@ -31,11 +32,11 @@ namespace milvus { namespace server { DBWrapper::DBWrapper() { - } -Status DBWrapper::StartService() { - Config& config = Config::GetInstance(); +Status +DBWrapper::StartService() { + Config &config = Config::GetInstance(); Status s; //db config engine::DBOptions opt; @@ -65,15 +66,13 @@ Status DBWrapper::StartService() { if (mode == "single") { opt.mode_ = engine::DBOptions::MODE::SINGLE; - } - else if (mode == "cluster") { + } else if (mode == "cluster") { opt.mode_ = engine::DBOptions::MODE::CLUSTER; - } - else if (mode == "read_only") { + } else if (mode == "read_only") { opt.mode_ = engine::DBOptions::MODE::READ_ONLY; - } - else { - std::cerr << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" << std::endl; + } else { + std::cerr << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" + << std::endl; kill(0, SIGUSR1); } @@ -86,8 +85,8 @@ Status DBWrapper::StartService() { SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; } else { uint32_t sys_thread_cnt = 8; - if(CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { - omp_thread = (int32_t)ceil(sys_thread_cnt*0.5); + if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { + omp_thread = (int32_t) ceil(sys_thread_cnt * 0.5); omp_set_num_threads(omp_thread); } } @@ -116,14 +115,14 @@ Status DBWrapper::StartService() { //create db root folder Status status = CommonUtil::CreateDirectory(opt.meta_.path_); - if(!status.ok()) { + if (!status.ok()) { std::cerr << "ERROR! Failed to create database root path: " << opt.meta_.path_ << std::endl; kill(0, SIGUSR1); } - for(auto& path : opt.meta_.slave_paths_) { + for (auto &path : opt.meta_.slave_paths_) { status = CommonUtil::CreateDirectory(path); - if(!status.ok()) { + if (!status.ok()) { std::cerr << "ERROR! Failed to create database slave path: " << path << std::endl; kill(0, SIGUSR1); } @@ -132,7 +131,7 @@ Status DBWrapper::StartService() { //create db instance try { db_ = engine::DBFactory::Build(opt); - } catch(std::exception& ex) { + } catch (std::exception &ex) { std::cerr << "ERROR! Failed to open database: " << ex.what() << std::endl; kill(0, SIGUSR1); } @@ -142,14 +141,15 @@ Status DBWrapper::StartService() { return Status::OK(); } -Status DBWrapper::StopService() { - if(db_) { +Status +DBWrapper::StopService() { + if (db_) { db_->Stop(); } return Status::OK(); } -} -} -} \ No newline at end of file +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/DBWrapper.h b/cpp/src/server/DBWrapper.h index 99b8f90500..4648f4b59d 100644 --- a/cpp/src/server/DBWrapper.h +++ b/cpp/src/server/DBWrapper.h @@ -27,12 +27,12 @@ namespace milvus { namespace server { class DBWrapper { -private: + private: DBWrapper(); ~DBWrapper() = default; -public: - static DBWrapper& GetInstance() { + public: + static DBWrapper &GetInstance() { static DBWrapper wrapper; return wrapper; } @@ -48,10 +48,10 @@ public: return db_; } -private: + private: engine::DBPtr db_; }; -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 57081845a5..336514631c 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#include "server/Server.h" + #include #include #include @@ -24,7 +26,6 @@ #include #include -#include "Server.h" #include "server/grpc_impl/GrpcServer.h" #include "server/Config.h" #include "utils/Log.h" @@ -36,7 +37,6 @@ #include "wrapper/KnowhereResource.h" #include "DBWrapper.h" - namespace zilliz { namespace milvus { namespace server { @@ -116,7 +116,7 @@ Server::Daemonize() { } // Close all open fd - for (long fd = sysconf(_SC_OPEN_MAX); fd > 0; fd--) { + for (int64_t fd = sysconf(_SC_OPEN_MAX); fd > 0; fd--) { close(fd); } @@ -172,9 +172,9 @@ Server::Start() { time_zone = "CUT"; } else { int time_bias = std::stoi(time_zone.substr(3, std::string::npos)); - if (time_bias == 0) + if (time_bias == 0) { time_zone = "CUT"; - else if (time_bias > 0) { + } else if (time_bias > 0) { time_zone = "CUT" + std::to_string(-time_bias); } else { time_zone = "CUT+" + std::to_string(-time_bias); @@ -194,7 +194,6 @@ Server::Start() { StartService(); std::cout << "Milvus server start successfully." << std::endl; - } catch (std::exception &ex) { std::cerr << "Milvus server encounter exception: " << ex.what(); } @@ -232,10 +231,9 @@ Server::Stop() { std::cerr << "Milvus server is closed!" << std::endl; } - ErrorCode Server::LoadConfig() { - Config& config = Config::GetInstance(); + Config &config = Config::GetInstance(); Status s = config.LoadConfigFile(config_filename_); if (!s.ok()) { std::cerr << "Failed to load config file: " << config_filename_ << std::endl; @@ -260,6 +258,6 @@ Server::StopService() { engine::KnowhereResource::Finalize(); } -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/Server.h b/cpp/src/server/Server.h index 8507552b75..9bd0ce13b5 100644 --- a/cpp/src/server/Server.h +++ b/cpp/src/server/Server.h @@ -22,7 +22,6 @@ #include #include - namespace zilliz { namespace milvus { namespace server { @@ -58,6 +57,6 @@ class Server { std::string log_config_file_; }; // Server -} // server -} // sql -} // zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp b/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp index c85d1b2fcd..a64422aab7 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp @@ -16,10 +16,12 @@ // under the License. -#include "GrpcRequestHandler.h" -#include "GrpcRequestTask.h" +#include "server/grpc_impl/GrpcRequestHandler.h" +#include "server/grpc_impl/GrpcRequestTask.h" #include "utils/TimeRecorder.h" +#include + namespace zilliz { namespace milvus { namespace server { @@ -29,7 +31,6 @@ namespace grpc { GrpcRequestHandler::CreateTable(::grpc::ServerContext *context, const ::milvus::grpc::TableSchema *request, ::milvus::grpc::Status *response) { - BaseTaskPtr task_ptr = CreateTableTask::Create(request); GrpcRequestScheduler::ExecTask(task_ptr, response); return ::grpc::Status::OK; @@ -39,7 +40,6 @@ GrpcRequestHandler::CreateTable(::grpc::ServerContext *context, GrpcRequestHandler::HasTable(::grpc::ServerContext *context, const ::milvus::grpc::TableName *request, ::milvus::grpc::BoolReply *response) { - bool has_table = false; BaseTaskPtr task_ptr = HasTableTask::Create(request->table_name(), has_table); ::milvus::grpc::Status grpc_status; @@ -61,9 +61,8 @@ GrpcRequestHandler::DropTable(::grpc::ServerContext *context, ::grpc::Status GrpcRequestHandler::CreateIndex(::grpc::ServerContext *context, - const ::milvus::grpc::IndexParam *request, - ::milvus::grpc::Status *response) { - + const ::milvus::grpc::IndexParam *request, + ::milvus::grpc::Status *response) { BaseTaskPtr task_ptr = CreateIndexTask::Create(request); GrpcRequestScheduler::ExecTask(task_ptr, response); return ::grpc::Status::OK; @@ -71,9 +70,8 @@ GrpcRequestHandler::CreateIndex(::grpc::ServerContext *context, ::grpc::Status GrpcRequestHandler::Insert(::grpc::ServerContext *context, - const ::milvus::grpc::InsertParam *request, - ::milvus::grpc::VectorIds *response) { - + const ::milvus::grpc::InsertParam *request, + ::milvus::grpc::VectorIds *response) { BaseTaskPtr task_ptr = InsertTask::Create(request, response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -86,7 +84,6 @@ GrpcRequestHandler::Insert(::grpc::ServerContext *context, GrpcRequestHandler::Search(::grpc::ServerContext *context, const ::milvus::grpc::SearchParam *request, ::milvus::grpc::TopKQueryResultList *response) { - std::vector file_id_array; BaseTaskPtr task_ptr = SearchTask::Create(request, file_id_array, response); ::milvus::grpc::Status grpc_status; @@ -100,7 +97,6 @@ GrpcRequestHandler::Search(::grpc::ServerContext *context, GrpcRequestHandler::SearchInFiles(::grpc::ServerContext *context, const ::milvus::grpc::SearchInFilesParam *request, ::milvus::grpc::TopKQueryResultList *response) { - std::vector file_id_array; for (int i = 0; i < request->file_id_array_size(); i++) { file_id_array.push_back(request->file_id_array(i)); @@ -118,7 +114,6 @@ GrpcRequestHandler::SearchInFiles(::grpc::ServerContext *context, GrpcRequestHandler::DescribeTable(::grpc::ServerContext *context, const ::milvus::grpc::TableName *request, ::milvus::grpc::TableSchema *response) { - BaseTaskPtr task_ptr = DescribeTableTask::Create(request->table_name(), response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -129,9 +124,8 @@ GrpcRequestHandler::DescribeTable(::grpc::ServerContext *context, ::grpc::Status GrpcRequestHandler::CountTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::TableRowCount *response) { - + const ::milvus::grpc::TableName *request, + ::milvus::grpc::TableRowCount *response) { int64_t row_count = 0; BaseTaskPtr task_ptr = CountTableTask::Create(request->table_name(), row_count); ::milvus::grpc::Status grpc_status; @@ -146,7 +140,6 @@ GrpcRequestHandler::CountTable(::grpc::ServerContext *context, GrpcRequestHandler::ShowTables(::grpc::ServerContext *context, const ::milvus::grpc::Command *request, ::milvus::grpc::TableNameList *response) { - BaseTaskPtr task_ptr = ShowTablesTask::Create(response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -157,9 +150,8 @@ GrpcRequestHandler::ShowTables(::grpc::ServerContext *context, ::grpc::Status GrpcRequestHandler::Cmd(::grpc::ServerContext *context, - const ::milvus::grpc::Command *request, - ::milvus::grpc::StringReply *response) { - + const ::milvus::grpc::Command *request, + ::milvus::grpc::StringReply *response) { std::string result; BaseTaskPtr task_ptr = CmdTask::Create(request->cmd(), result); ::milvus::grpc::Status grpc_status; @@ -172,8 +164,8 @@ GrpcRequestHandler::Cmd(::grpc::ServerContext *context, ::grpc::Status GrpcRequestHandler::DeleteByRange(::grpc::ServerContext *context, - const ::milvus::grpc::DeleteByRangeParam *request, - ::milvus::grpc::Status *response) { + const ::milvus::grpc::DeleteByRangeParam *request, + ::milvus::grpc::Status *response) { BaseTaskPtr task_ptr = DeleteByRangeTask::Create(request); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -184,8 +176,8 @@ GrpcRequestHandler::DeleteByRange(::grpc::ServerContext *context, ::grpc::Status GrpcRequestHandler::PreloadTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::Status *response) { + const ::milvus::grpc::TableName *request, + ::milvus::grpc::Status *response) { BaseTaskPtr task_ptr = PreloadTableTask::Create(request->table_name()); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -196,8 +188,8 @@ GrpcRequestHandler::PreloadTable(::grpc::ServerContext *context, ::grpc::Status GrpcRequestHandler::DescribeIndex(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::IndexParam *response) { + const ::milvus::grpc::TableName *request, + ::milvus::grpc::IndexParam *response) { BaseTaskPtr task_ptr = DescribeIndexTask::Create(request->table_name(), response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -208,8 +200,8 @@ GrpcRequestHandler::DescribeIndex(::grpc::ServerContext *context, ::grpc::Status GrpcRequestHandler::DropIndex(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::Status *response) { + const ::milvus::grpc::TableName *request, + ::milvus::grpc::Status *response) { BaseTaskPtr task_ptr = DropIndexTask::Create(request->table_name()); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -218,8 +210,7 @@ GrpcRequestHandler::DropIndex(::grpc::ServerContext *context, return ::grpc::Status::OK; } - -} -} -} -} \ No newline at end of file +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestHandler.h b/cpp/src/server/grpc_impl/GrpcRequestHandler.h index a8a70eb88d..549e1d9d35 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestHandler.h +++ b/cpp/src/server/grpc_impl/GrpcRequestHandler.h @@ -20,15 +20,15 @@ #include #include -#include "milvus.grpc.pb.h" -#include "status.pb.h" +#include "grpc/gen-milvus/milvus.grpc.pb.h" +#include "grpc/gen-status/status.pb.h" namespace zilliz { namespace milvus { namespace server { namespace grpc { class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { -public: + public: /** * @brief Create table method * @@ -103,8 +103,7 @@ public: */ ::grpc::Status CreateIndex(::grpc::ServerContext *context, - const ::milvus::grpc::IndexParam *request, ::milvus::grpc::Status *response) override; - + const ::milvus::grpc::IndexParam *request, ::milvus::grpc::Status *response) override; /** * @brief Insert vector array to table @@ -123,8 +122,8 @@ public: */ ::grpc::Status Insert(::grpc::ServerContext *context, - const ::milvus::grpc::InsertParam *request, - ::milvus::grpc::VectorIds *response) override; + const ::milvus::grpc::InsertParam *request, + ::milvus::grpc::VectorIds *response) override; /** * @brief Query vector @@ -213,8 +212,8 @@ public: */ ::grpc::Status CountTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::TableRowCount *response) override; + const ::milvus::grpc::TableName *request, + ::milvus::grpc::TableRowCount *response) override; /** * @brief List all tables in database @@ -253,8 +252,8 @@ public: */ ::grpc::Status Cmd(::grpc::ServerContext *context, - const ::milvus::grpc::Command *request, - ::milvus::grpc::StringReply *response) override; + const ::milvus::grpc::Command *request, + ::milvus::grpc::StringReply *response) override; /** * @brief delete table by range @@ -291,8 +290,8 @@ public: */ ::grpc::Status PreloadTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::Status *response) override; + const ::milvus::grpc::TableName *request, + ::milvus::grpc::Status *response) override; /** * @brief Describe index @@ -310,8 +309,8 @@ public: */ ::grpc::Status DescribeIndex(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::IndexParam *response) override; + const ::milvus::grpc::TableName *request, + ::milvus::grpc::IndexParam *response) override; /** * @brief Drop index @@ -329,12 +328,12 @@ public: */ ::grpc::Status DropIndex(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::Status *response) override; - + const ::milvus::grpc::TableName *request, + ::milvus::grpc::Status *response) override; }; -} -} -} -} + +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp index fb8daa3540..4c58195f37 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp @@ -15,101 +15,107 @@ // specific language governing permissions and limitations // under the License. -#include "GrpcRequestScheduler.h" +#include "server/grpc_impl/GrpcRequestScheduler.h" #include "utils/Log.h" -#include "src/grpc/gen-status/status.pb.h" +#include "grpc/gen-status/status.pb.h" + +#include namespace zilliz { namespace milvus { namespace server { namespace grpc { -using namespace ::milvus; - namespace { - ::milvus::grpc::ErrorCode ErrorMap(ErrorCode code) { - static const std::map code_map = { - {SERVER_UNEXPECTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, - {SERVER_UNSUPPORTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, - {SERVER_NULL_POINTER, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, - {SERVER_INVALID_ARGUMENT, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_FILE_NOT_FOUND, ::milvus::grpc::ErrorCode::FILE_NOT_FOUND}, - {SERVER_NOT_IMPLEMENT, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, - {SERVER_BLOCKING_QUEUE_EMPTY, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, - {SERVER_CANNOT_CREATE_FOLDER, ::milvus::grpc::ErrorCode::CANNOT_CREATE_FOLDER}, - {SERVER_CANNOT_CREATE_FILE, ::milvus::grpc::ErrorCode::CANNOT_CREATE_FILE}, - {SERVER_CANNOT_DELETE_FOLDER, ::milvus::grpc::ErrorCode::CANNOT_DELETE_FOLDER}, - {SERVER_CANNOT_DELETE_FILE, ::milvus::grpc::ErrorCode::CANNOT_DELETE_FILE}, - {SERVER_TABLE_NOT_EXIST, ::milvus::grpc::ErrorCode::TABLE_NOT_EXISTS}, - {SERVER_INVALID_TABLE_NAME, ::milvus::grpc::ErrorCode::ILLEGAL_TABLE_NAME}, - {SERVER_INVALID_TABLE_DIMENSION, ::milvus::grpc::ErrorCode::ILLEGAL_DIMENSION}, - {SERVER_INVALID_TIME_RANGE, ::milvus::grpc::ErrorCode::ILLEGAL_RANGE}, - {SERVER_INVALID_VECTOR_DIMENSION, ::milvus::grpc::ErrorCode::ILLEGAL_DIMENSION}, +::milvus::grpc::ErrorCode +ErrorMap(ErrorCode code) { + static const std::map code_map = { + {SERVER_UNEXPECTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_UNSUPPORTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_NULL_POINTER, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_INVALID_ARGUMENT, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, + {SERVER_FILE_NOT_FOUND, ::milvus::grpc::ErrorCode::FILE_NOT_FOUND}, + {SERVER_NOT_IMPLEMENT, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_BLOCKING_QUEUE_EMPTY, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_CANNOT_CREATE_FOLDER, ::milvus::grpc::ErrorCode::CANNOT_CREATE_FOLDER}, + {SERVER_CANNOT_CREATE_FILE, ::milvus::grpc::ErrorCode::CANNOT_CREATE_FILE}, + {SERVER_CANNOT_DELETE_FOLDER, ::milvus::grpc::ErrorCode::CANNOT_DELETE_FOLDER}, + {SERVER_CANNOT_DELETE_FILE, ::milvus::grpc::ErrorCode::CANNOT_DELETE_FILE}, + {SERVER_TABLE_NOT_EXIST, ::milvus::grpc::ErrorCode::TABLE_NOT_EXISTS}, + {SERVER_INVALID_TABLE_NAME, ::milvus::grpc::ErrorCode::ILLEGAL_TABLE_NAME}, + {SERVER_INVALID_TABLE_DIMENSION, ::milvus::grpc::ErrorCode::ILLEGAL_DIMENSION}, + {SERVER_INVALID_TIME_RANGE, ::milvus::grpc::ErrorCode::ILLEGAL_RANGE}, + {SERVER_INVALID_VECTOR_DIMENSION, ::milvus::grpc::ErrorCode::ILLEGAL_DIMENSION}, - {SERVER_INVALID_INDEX_TYPE, ::milvus::grpc::ErrorCode::ILLEGAL_INDEX_TYPE}, - {SERVER_INVALID_ROWRECORD, ::milvus::grpc::ErrorCode::ILLEGAL_ROWRECORD}, - {SERVER_INVALID_ROWRECORD_ARRAY, ::milvus::grpc::ErrorCode::ILLEGAL_ROWRECORD}, - {SERVER_INVALID_TOPK, ::milvus::grpc::ErrorCode::ILLEGAL_TOPK}, - {SERVER_INVALID_NPROBE, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_INVALID_INDEX_NLIST, ::milvus::grpc::ErrorCode::ILLEGAL_NLIST}, - {SERVER_INVALID_INDEX_METRIC_TYPE,::milvus::grpc::ErrorCode::ILLEGAL_METRIC_TYPE}, - {SERVER_INVALID_INDEX_FILE_SIZE, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_ILLEGAL_VECTOR_ID, ::milvus::grpc::ErrorCode::ILLEGAL_VECTOR_ID}, - {SERVER_ILLEGAL_SEARCH_RESULT, ::milvus::grpc::ErrorCode::ILLEGAL_SEARCH_RESULT}, - {SERVER_CACHE_ERROR, ::milvus::grpc::ErrorCode::CACHE_FAILED}, - {DB_META_TRANSACTION_FAILED, ::milvus::grpc::ErrorCode::META_FAILED}, - {SERVER_BUILD_INDEX_ERROR, ::milvus::grpc::ErrorCode::BUILD_INDEX_ERROR}, - {SERVER_OUT_OF_MEMORY, ::milvus::grpc::ErrorCode::OUT_OF_MEMORY}, - }; + {SERVER_INVALID_INDEX_TYPE, ::milvus::grpc::ErrorCode::ILLEGAL_INDEX_TYPE}, + {SERVER_INVALID_ROWRECORD, ::milvus::grpc::ErrorCode::ILLEGAL_ROWRECORD}, + {SERVER_INVALID_ROWRECORD_ARRAY, ::milvus::grpc::ErrorCode::ILLEGAL_ROWRECORD}, + {SERVER_INVALID_TOPK, ::milvus::grpc::ErrorCode::ILLEGAL_TOPK}, + {SERVER_INVALID_NPROBE, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, + {SERVER_INVALID_INDEX_NLIST, ::milvus::grpc::ErrorCode::ILLEGAL_NLIST}, + {SERVER_INVALID_INDEX_METRIC_TYPE, ::milvus::grpc::ErrorCode::ILLEGAL_METRIC_TYPE}, + {SERVER_INVALID_INDEX_FILE_SIZE, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, + {SERVER_ILLEGAL_VECTOR_ID, ::milvus::grpc::ErrorCode::ILLEGAL_VECTOR_ID}, + {SERVER_ILLEGAL_SEARCH_RESULT, ::milvus::grpc::ErrorCode::ILLEGAL_SEARCH_RESULT}, + {SERVER_CACHE_ERROR, ::milvus::grpc::ErrorCode::CACHE_FAILED}, + {DB_META_TRANSACTION_FAILED, ::milvus::grpc::ErrorCode::META_FAILED}, + {SERVER_BUILD_INDEX_ERROR, ::milvus::grpc::ErrorCode::BUILD_INDEX_ERROR}, + {SERVER_OUT_OF_MEMORY, ::milvus::grpc::ErrorCode::OUT_OF_MEMORY}, + }; - if(code_map.find(code) != code_map.end()) { - return code_map.at(code); - } else { - return ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR; - } + if (code_map.find(code) != code_map.end()) { + return code_map.at(code); + } else { + return ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR; } } +} // namespace //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// GrpcBaseTask::GrpcBaseTask(const std::string &task_group, bool async) - : task_group_(task_group), - async_(async), - done_(false) { - + : task_group_(task_group), + async_(async), + done_(false) { } GrpcBaseTask::~GrpcBaseTask() { WaitToFinish(); } -Status GrpcBaseTask::Execute() { +Status +GrpcBaseTask::Execute() { status_ = OnExecute(); Done(); return status_; } -void GrpcBaseTask::Done() { +void +GrpcBaseTask::Done() { done_ = true; finish_cond_.notify_all(); } -Status GrpcBaseTask::SetStatus(ErrorCode error_code, const std::string &error_msg) { +Status +GrpcBaseTask::SetStatus(ErrorCode error_code, const std::string &error_msg) { status_ = Status(error_code, error_msg); SERVER_LOG_ERROR << error_msg; return status_; } -Status GrpcBaseTask::WaitToFinish() { +Status +GrpcBaseTask::WaitToFinish() { std::unique_lock lock(finish_mtx_); - finish_cond_.wait(lock, [this] { return done_; }); + finish_cond_.wait(lock, [this] { + return done_; + }); return status_; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// GrpcRequestScheduler::GrpcRequestScheduler() - : stopped_(false) { + : stopped_(false) { Start(); } @@ -117,7 +123,8 @@ GrpcRequestScheduler::~GrpcRequestScheduler() { Stop(); } -void GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Status *grpc_status) { +void +GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Status *grpc_status) { if (task_ptr == nullptr) { return; } @@ -127,7 +134,7 @@ void GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Statu if (!task_ptr->IsAsync()) { task_ptr->WaitToFinish(); - const Status& status = task_ptr->status(); + const Status &status = task_ptr->status(); if (!status.ok()) { grpc_status->set_reason(status.message()); grpc_status->set_error_code(ErrorMap(status.code())); @@ -135,7 +142,8 @@ void GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Statu } } -void GrpcRequestScheduler::Start() { +void +GrpcRequestScheduler::Start() { if (!stopped_) { return; } @@ -143,7 +151,8 @@ void GrpcRequestScheduler::Start() { stopped_ = false; } -void GrpcRequestScheduler::Stop() { +void +GrpcRequestScheduler::Stop() { if (stopped_) { return; } @@ -168,7 +177,8 @@ void GrpcRequestScheduler::Stop() { SERVER_LOG_INFO << "Scheduler stopped"; } -Status GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr &task_ptr) { +Status +GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr &task_ptr) { if (task_ptr == nullptr) { return Status::OK(); } @@ -186,8 +196,8 @@ Status GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr &task_ptr) { return task_ptr->WaitToFinish();//sync execution } - -void GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) { +void +GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) { if (task_queue == nullptr) { return; } @@ -210,7 +220,8 @@ void GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) { } } -Status GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) { +Status +GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) { std::lock_guard lock(queue_mtx_); std::string group_name = task_ptr->TaskGroup(); @@ -230,7 +241,7 @@ Status GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) { return Status::OK(); } -} -} -} -} +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.h b/cpp/src/server/grpc_impl/GrpcRequestScheduler.h index e1217540fc..df5357a4bb 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.h +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.h @@ -19,12 +19,14 @@ #include "utils/Status.h" #include "utils/BlockingQueue.h" -#include "status.grpc.pb.h" -#include "status.pb.h" +#include "grpc/gen-status/status.grpc.pb.h" +#include "grpc/gen-status/status.pb.h" #include #include #include +#include +#include namespace zilliz { namespace milvus { @@ -32,30 +34,36 @@ namespace server { namespace grpc { class GrpcBaseTask { -protected: - GrpcBaseTask(const std::string &task_group, bool async = false); + protected: + explicit GrpcBaseTask(const std::string &task_group, bool async = false); virtual ~GrpcBaseTask(); -public: + public: Status Execute(); void Done(); Status WaitToFinish(); - std::string TaskGroup() const { return task_group_; } + std::string TaskGroup() const { + return task_group_; + } - const Status& status() const { return status_; } + const Status &status() const { + return status_; + } - bool IsAsync() const { return async_; } + bool IsAsync() const { + return async_; + } -protected: + protected: virtual Status OnExecute() = 0; Status SetStatus(ErrorCode error_code, const std::string &msg); -protected: + protected: mutable std::mutex finish_mtx_; std::condition_variable finish_cond_; @@ -71,7 +79,7 @@ using TaskQueuePtr = std::shared_ptr; using ThreadPtr = std::shared_ptr; class GrpcRequestScheduler { -public: + public: static GrpcRequestScheduler &GetInstance() { static GrpcRequestScheduler scheduler; return scheduler; @@ -85,7 +93,7 @@ public: static void ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Status *grpc_status); -protected: + protected: GrpcRequestScheduler(); virtual ~GrpcRequestScheduler(); @@ -94,7 +102,7 @@ protected: Status PutTaskToQueue(const BaseTaskPtr &task_ptr); -private: + private: mutable std::mutex queue_mtx_; std::map task_groups_; @@ -104,7 +112,7 @@ private: bool stopped_; }; -} -} -} -} +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index ca6b77e960..5702b80bec 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -15,22 +15,24 @@ // specific language governing permissions and limitations // under the License. -#include +#include "server/grpc_impl/GrpcRequestTask.h" -#include "GrpcRequestTask.h" +#include +#include +#include +#include +//#include + +#include "server/Server.h" +#include "server/DBWrapper.h" #include "utils/CommonUtil.h" #include "utils/Log.h" #include "utils/TimeRecorder.h" #include "utils/ValidationUtil.h" -#include "../DBWrapper.h" -#include "version.h" #include "GrpcServer.h" #include "db/Utils.h" #include "scheduler/SchedInst.h" -//#include - -#include "server/Server.h" - +#include "../../../version.h" namespace zilliz { namespace milvus { @@ -45,86 +47,87 @@ using DB_META = zilliz::milvus::engine::meta::Meta; using DB_DATE = zilliz::milvus::engine::meta::DateT; namespace { - engine::EngineType EngineType(int type) { - static std::map map_type = { - {0, engine::EngineType::INVALID}, - {1, engine::EngineType::FAISS_IDMAP}, - {2, engine::EngineType::FAISS_IVFFLAT}, - {3, engine::EngineType::FAISS_IVFSQ8}, - }; +engine::EngineType +EngineType(int type) { + static std::map map_type = { + {0, engine::EngineType::INVALID}, + {1, engine::EngineType::FAISS_IDMAP}, + {2, engine::EngineType::FAISS_IVFFLAT}, + {3, engine::EngineType::FAISS_IVFSQ8}, + }; - if (map_type.find(type) == map_type.end()) { - return engine::EngineType::INVALID; - } - - return map_type[type]; + if (map_type.find(type) == map_type.end()) { + return engine::EngineType::INVALID; } - int IndexType(engine::EngineType type) { - static std::map map_type = { - {engine::EngineType::INVALID, 0}, - {engine::EngineType::FAISS_IDMAP, 1}, - {engine::EngineType::FAISS_IVFFLAT, 2}, - {engine::EngineType::FAISS_IVFSQ8, 3}, - }; - - if (map_type.find(type) == map_type.end()) { - return 0; - } - - return map_type[type]; - } - - constexpr long DAY_SECONDS = 24 * 60 * 60; - - Status - ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range> &range_array, - std::vector &dates) { - dates.clear(); - for (auto &range : range_array) { - time_t tt_start, tt_end; - tm tm_start, tm_end; - if (!CommonUtil::TimeStrToTime(range.start_value(), tt_start, tm_start)) { - return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); - } - - if (!CommonUtil::TimeStrToTime(range.end_value(), tt_end, tm_end)) { - return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); - } - - long days = (tt_end > tt_start) ? (tt_end - tt_start) / DAY_SECONDS : (tt_start - tt_end) / - DAY_SECONDS; - if (days == 0) { - return Status(SERVER_INVALID_TIME_RANGE, - "Invalid time range: " + range.start_value() + " to " + range.end_value()); - } - - //range: [start_day, end_day) - for (long i = 0; i < days; i++) { - time_t tt_day = tt_start + DAY_SECONDS * i; - tm tm_day; - CommonUtil::ConvertTime(tt_day, tm_day); - - long date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 + - tm_day.tm_mday;//according to db logic - dates.push_back(date); - } - } - - return Status::OK(); - } + return map_type[type]; } +int +IndexType(engine::EngineType type) { + static std::map map_type = { + {engine::EngineType::INVALID, 0}, + {engine::EngineType::FAISS_IDMAP, 1}, + {engine::EngineType::FAISS_IVFFLAT, 2}, + {engine::EngineType::FAISS_IVFSQ8, 3}, + }; + + if (map_type.find(type) == map_type.end()) { + return 0; + } + + return map_type[type]; +} + +constexpr int64_t DAY_SECONDS = 24 * 60 * 60; + +Status +ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range> &range_array, + std::vector &dates) { + dates.clear(); + for (auto &range : range_array) { + time_t tt_start, tt_end; + tm tm_start, tm_end; + if (!CommonUtil::TimeStrToTime(range.start_value(), tt_start, tm_start)) { + return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); + } + + if (!CommonUtil::TimeStrToTime(range.end_value(), tt_end, tm_end)) { + return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); + } + + int64_t days = (tt_end > tt_start) ? (tt_end - tt_start) / DAY_SECONDS : (tt_start - tt_end) / + DAY_SECONDS; + if (days == 0) { + return Status(SERVER_INVALID_TIME_RANGE, + "Invalid time range: " + range.start_value() + " to " + range.end_value()); + } + + //range: [start_day, end_day) + for (int64_t i = 0; i < days; i++) { + time_t tt_day = tt_start + DAY_SECONDS * i; + tm tm_day; + CommonUtil::ConvertTime(tt_day, tm_day); + + int64_t date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 + + tm_day.tm_mday;//according to db logic + dates.push_back(date); + } + } + + return Status::OK(); +} +} // namespace + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// CreateTableTask::CreateTableTask(const ::milvus::grpc::TableSchema *schema) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - schema_(schema) { - + : GrpcBaseTask(DDL_DML_TASK_GROUP), + schema_(schema) { } BaseTaskPtr CreateTableTask::Create(const ::milvus::grpc::TableSchema *schema) { - if(schema == nullptr) { + if (schema == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; } @@ -168,12 +171,11 @@ CreateTableTask::OnExecute() { status = DBWrapper::DB()->CreateTable(table_info); if (!status.ok()) { //table could exist - if(status.code() == DB_ALREADY_EXIST) { + if (status.code() == DB_ALREADY_EXIST) { return Status(SERVER_INVALID_TABLE_NAME, status.message()); } return status; } - } catch (std::exception &ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -185,9 +187,9 @@ CreateTableTask::OnExecute() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// DescribeTableTask::DescribeTableTask(const std::string &table_name, ::milvus::grpc::TableSchema *schema) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name), - schema_(schema) { + : GrpcBaseTask(DDL_DML_TASK_GROUP), + table_name_(table_name), + schema_(schema) { } BaseTaskPtr @@ -218,7 +220,6 @@ DescribeTableTask::OnExecute() { schema_->set_dimension(table_info.dimension_); schema_->set_index_file_size(table_info.index_file_size_); schema_->set_metric_type(table_info.metric_type_); - } catch (std::exception &ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -230,13 +231,13 @@ DescribeTableTask::OnExecute() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// CreateIndexTask::CreateIndexTask(const ::milvus::grpc::IndexParam *index_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - index_param_(index_param) { + : GrpcBaseTask(DDL_DML_TASK_GROUP), + index_param_(index_param) { } BaseTaskPtr CreateIndexTask::Create(const ::milvus::grpc::IndexParam *index_param) { - if(index_param == nullptr) { + if (index_param == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; } @@ -295,10 +296,9 @@ CreateIndexTask::OnExecute() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// HasTableTask::HasTableTask(const std::string &table_name, bool &has_table) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name), - has_table_(has_table) { - + : GrpcBaseTask(DDL_DML_TASK_GROUP), + table_name_(table_name), + has_table_(has_table) { } BaseTaskPtr @@ -333,9 +333,8 @@ HasTableTask::OnExecute() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// DropTableTask::DropTableTask(const std::string &table_name) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name) { - + : GrpcBaseTask(DDL_DML_TASK_GROUP), + table_name_(table_name) { } BaseTaskPtr @@ -385,9 +384,8 @@ DropTableTask::OnExecute() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ShowTablesTask::ShowTablesTask(::milvus::grpc::TableNameList *table_name_list) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_list_(table_name_list) { - + : GrpcBaseTask(DDL_DML_TASK_GROUP), + table_name_list_(table_name_list) { } BaseTaskPtr @@ -419,8 +417,8 @@ InsertTask::InsertTask(const ::milvus::grpc::InsertParam *insert_param, BaseTaskPtr InsertTask::Create(const ::milvus::grpc::InsertParam *insert_param, - ::milvus::grpc::VectorIds *record_ids) { - if(insert_param == nullptr) { + ::milvus::grpc::VectorIds *record_ids) { + if (insert_param == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; } @@ -444,7 +442,7 @@ InsertTask::OnExecute() { if (!record_ids_->vector_id_array().empty()) { if (record_ids_->vector_id_array().size() != insert_param_->row_record_array_size()) { return Status(SERVER_ILLEGAL_VECTOR_ID, - "Size of vector ids is not equal to row record array size"); + "Size of vector ids is not equal to row record array size"); } } @@ -455,7 +453,7 @@ InsertTask::OnExecute() { if (!status.ok()) { if (status.code() == DB_NOT_FOUND) { return Status(SERVER_TABLE_NOT_EXIST, - "Table " + insert_param_->table_name() + " not exists"); + "Table " + insert_param_->table_name() + " not exists"); } else { return status; } @@ -465,19 +463,22 @@ InsertTask::OnExecute() { //all user provide id, or all internal id bool user_provide_ids = !insert_param_->row_id_array().empty(); //user already provided id before, all insert action require user id - if((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) && !user_provide_ids) { - return Status(SERVER_ILLEGAL_VECTOR_ID, "Table vector ids are user defined, please provide id for this batch"); + if ((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) && !user_provide_ids) { + return Status(SERVER_ILLEGAL_VECTOR_ID, + "Table vector ids are user defined, please provide id for this batch"); } //user didn't provided id before, no need to provide user id - if((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) && user_provide_ids) { - return Status(SERVER_ILLEGAL_VECTOR_ID, "Table vector ids are auto generated, no need to provide id for this batch"); + if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) && user_provide_ids) { + return Status(SERVER_ILLEGAL_VECTOR_ID, + "Table vector ids are auto generated, no need to provide id for this batch"); } rc.RecordSection("check validation"); #ifdef MILVUS_ENABLE_PROFILING - std::string fname = "/tmp/insert_" + std::to_string(this->insert_param_->row_record_array_size()) + ".profiling"; + std::string fname = "/tmp/insert_" + std::to_string(this->insert_param_->row_record_array_size()) + + ".profiling"; ProfilerStart(fname.c_str()); #endif @@ -493,8 +494,8 @@ InsertTask::OnExecute() { if (vec_dim != table_info.dimension_) { ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; std::string error_msg = "Invalid row record dimension: " + std::to_string(vec_dim) - + " vs. table dimension:" + - std::to_string(table_info.dimension_); + + " vs. table dimension:" + + std::to_string(table_info.dimension_); return Status(error_code, error_msg); } memcpy(&vec_f[i * table_info.dimension_], @@ -507,10 +508,10 @@ InsertTask::OnExecute() { //step 5: insert vectors auto vec_count = (uint64_t) insert_param_->row_record_array_size(); std::vector vec_ids(insert_param_->row_id_array_size(), 0); - if(!insert_param_->row_id_array().empty()) { - const int64_t* src_data = insert_param_->row_id_array().data(); - int64_t* target_data = vec_ids.data(); - memcpy(target_data, src_data, (size_t)(sizeof(int64_t)*insert_param_->row_id_array_size())); + if (!insert_param_->row_id_array().empty()) { + const int64_t *src_data = insert_param_->row_id_array().data(); + int64_t *target_data = vec_ids.data(); + memcpy(target_data, src_data, (size_t) (sizeof(int64_t) * insert_param_->row_id_array_size())); } status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), vec_count, vec_f.data(), vec_ids); @@ -525,13 +526,13 @@ InsertTask::OnExecute() { auto ids_size = record_ids_->vector_id_array_size(); if (ids_size != vec_count) { std::string msg = "Add " + std::to_string(vec_count) + " vectors but only return " - + std::to_string(ids_size) + " id"; + + std::to_string(ids_size) + " id"; return Status(SERVER_ILLEGAL_VECTOR_ID, msg); } //step 6: update table flag user_provide_ids ? table_info.flag_ |= engine::meta::FLAG_MASK_HAS_USERID - : table_info.flag_ |= engine::meta::FLAG_MASK_NO_USERID; + : table_info.flag_ |= engine::meta::FLAG_MASK_NO_USERID; status = DBWrapper::DB()->UpdateTableFlag(insert_param_->table_name(), table_info.flag_); #ifdef MILVUS_ENABLE_PROFILING @@ -540,7 +541,6 @@ InsertTask::OnExecute() { rc.RecordSection("add vectors to engine"); rc.ElapseFromBegin("total cost"); - } catch (std::exception &ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -556,19 +556,17 @@ SearchTask::SearchTask(const ::milvus::grpc::SearchParam *search_vector_infos, search_param_(search_vector_infos), file_id_array_(file_id_array), topk_result_list(response) { - } BaseTaskPtr SearchTask::Create(const ::milvus::grpc::SearchParam *search_vector_infos, const std::vector &file_id_array, ::milvus::grpc::TopKQueryResultList *response) { - if(search_vector_infos == nullptr) { + if (search_vector_infos == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; } - return std::shared_ptr(new SearchTask(search_vector_infos, file_id_array, - response)); + return std::shared_ptr(new SearchTask(search_vector_infos, file_id_array, response)); } Status @@ -640,7 +638,7 @@ SearchTask::OnExecute() { if (query_vec_dim != table_info.dimension_) { ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; std::string error_msg = "Invalid row record dimension: " + std::to_string(query_vec_dim) - + " vs. table dimension:" + std::to_string(table_info.dimension_); + + " vs. table dimension:" + std::to_string(table_info.dimension_); return Status(error_code, error_msg); } @@ -655,16 +653,17 @@ SearchTask::OnExecute() { auto record_count = (uint64_t) search_param_->query_record_array().size(); #ifdef MILVUS_ENABLE_PROFILING - std::string fname = "/tmp/search_nq_" + std::to_string(this->search_param_->query_record_array_size()) + ".profiling"; + std::string fname = "/tmp/search_nq_" + std::to_string(this->search_param_->query_record_array_size()) + + ".profiling"; ProfilerStart(fname.c_str()); #endif if (file_id_array_.empty()) { status = DBWrapper::DB()->Query(table_name_, (size_t) top_k, record_count, nprobe, - vec_f.data(), dates, results); + vec_f.data(), dates, results); } else { status = DBWrapper::DB()->Query(table_name_, file_id_array_, (size_t) top_k, - record_count, nprobe, vec_f.data(), dates, results); + record_count, nprobe, vec_f.data(), dates, results); } #ifdef MILVUS_ENABLE_PROFILING @@ -682,7 +681,7 @@ SearchTask::OnExecute() { if (results.size() != record_count) { std::string msg = "Search " + std::to_string(record_count) + " vectors but only return " - + std::to_string(results.size()) + " results"; + + std::to_string(results.size()) + " results"; return Status(SERVER_ILLEGAL_SEARCH_RESULT, msg); } @@ -699,8 +698,6 @@ SearchTask::OnExecute() { //step 8: print time cost percent rc.RecordSection("construct result and send"); rc.ElapseFromBegin("totally cost"); - - } catch (std::exception &ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -710,10 +707,9 @@ SearchTask::OnExecute() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// CountTableTask::CountTableTask(const std::string &table_name, int64_t &row_count) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name), - row_count_(row_count) { - + : GrpcBaseTask(DDL_DML_TASK_GROUP), + table_name_(table_name), + row_count_(row_count) { } BaseTaskPtr @@ -742,7 +738,6 @@ CountTableTask::OnExecute() { row_count_ = (int64_t) row_count; rc.ElapseFromBegin("total cost"); - } catch (std::exception &ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -752,10 +747,9 @@ CountTableTask::OnExecute() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// CmdTask::CmdTask(const std::string &cmd, std::string &result) - : GrpcBaseTask(PING_TASK_GROUP), - cmd_(cmd), - result_(result) { - + : GrpcBaseTask(PING_TASK_GROUP), + cmd_(cmd), + result_(result) { } BaseTaskPtr @@ -769,8 +763,7 @@ CmdTask::OnExecute() { result_ = MILVUS_VERSION; } else if (cmd_ == "tasktable") { result_ = scheduler::ResMgrInst::GetInstance()->DumpTaskTables(); - } - else { + } else { result_ = "OK"; } @@ -779,16 +772,17 @@ CmdTask::OnExecute() { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// DeleteByRangeTask::DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - delete_by_range_param_(delete_by_range_param){ + : GrpcBaseTask(DDL_DML_TASK_GROUP), + delete_by_range_param_(delete_by_range_param) { } BaseTaskPtr DeleteByRangeTask::Create(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param) { - if(delete_by_range_param == nullptr) { + if (delete_by_range_param == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; } + return std::shared_ptr(new DeleteByRangeTask(delete_by_range_param)); } @@ -838,23 +832,21 @@ DeleteByRangeTask::OnExecute() { if (!status.ok()) { return status; } - } catch (std::exception &ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } - + return Status::OK(); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// PreloadTableTask::PreloadTableTask(const std::string &table_name) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name) { - + : GrpcBaseTask(DDL_DML_TASK_GROUP), + table_name_(table_name) { } BaseTaskPtr -PreloadTableTask::Create(const std::string &table_name){ +PreloadTableTask::Create(const std::string &table_name) { return std::shared_ptr(new PreloadTableTask(table_name)); } @@ -889,12 +881,11 @@ DescribeIndexTask::DescribeIndexTask(const std::string &table_name, : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name), index_param_(index_param) { - } BaseTaskPtr DescribeIndexTask::Create(const std::string &table_name, - ::milvus::grpc::IndexParam *index_param){ + ::milvus::grpc::IndexParam *index_param) { return std::shared_ptr(new DescribeIndexTask(table_name, index_param)); } @@ -932,11 +923,10 @@ DescribeIndexTask::OnExecute() { DropIndexTask::DropIndexTask(const std::string &table_name) : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name) { - } BaseTaskPtr -DropIndexTask::Create(const std::string &table_name){ +DropIndexTask::Create(const std::string &table_name) { return std::shared_ptr(new DropIndexTask(table_name)); } @@ -975,7 +965,7 @@ DropIndexTask::OnExecute() { return Status::OK(); } -} -} -} -} +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.h b/cpp/src/server/grpc_impl/GrpcRequestTask.h index a0c4540cb3..4c8c038d44 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.h +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.h @@ -16,15 +16,18 @@ // under the License. #pragma once -#include "GrpcRequestScheduler.h" + +#include "server/grpc_impl/GrpcRequestScheduler.h" #include "utils/Status.h" #include "db/Types.h" -#include "milvus.grpc.pb.h" -#include "status.pb.h" +#include "grpc/gen-milvus/milvus.grpc.pb.h" +#include "grpc/gen-status/status.pb.h" #include #include +#include +#include namespace zilliz { namespace milvus { @@ -33,138 +36,130 @@ namespace grpc { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CreateTableTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const ::milvus::grpc::TableSchema *schema); -protected: - explicit - CreateTableTask(const ::milvus::grpc::TableSchema *request); + protected: + explicit CreateTableTask(const ::milvus::grpc::TableSchema *request); Status OnExecute() override; -private: + private: const ::milvus::grpc::TableSchema *schema_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class HasTableTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const std::string &table_name, bool &has_table); -protected: + protected: HasTableTask(const std::string &request, bool &has_table); Status OnExecute() override; - -private: + private: std::string table_name_; bool &has_table_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DescribeTableTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const std::string &table_name, ::milvus::grpc::TableSchema *schema); -protected: + protected: DescribeTableTask(const std::string &table_name, ::milvus::grpc::TableSchema *schema); Status OnExecute() override; - -private: + private: std::string table_name_; ::milvus::grpc::TableSchema *schema_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DropTableTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const std::string &table_name); -protected: - explicit - DropTableTask(const std::string &table_name); + protected: + explicit DropTableTask(const std::string &table_name); Status OnExecute() override; - -private: + private: std::string table_name_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CreateIndexTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const ::milvus::grpc::IndexParam *index_Param); -protected: - explicit - CreateIndexTask(const ::milvus::grpc::IndexParam *index_Param); + protected: + explicit CreateIndexTask(const ::milvus::grpc::IndexParam *index_Param); Status OnExecute() override; - -private: + private: const ::milvus::grpc::IndexParam *index_param_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class ShowTablesTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(::milvus::grpc::TableNameList *table_name_list); -protected: - explicit - ShowTablesTask(::milvus::grpc::TableNameList *table_name_list); + protected: + explicit ShowTablesTask(::milvus::grpc::TableNameList *table_name_list); Status OnExecute() override; -private: + private: ::milvus::grpc::TableNameList *table_name_list_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class InsertTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const ::milvus::grpc::InsertParam *insert_Param, ::milvus::grpc::VectorIds *record_ids_); -protected: + protected: InsertTask(const ::milvus::grpc::InsertParam *insert_Param, - ::milvus::grpc::VectorIds *record_ids_); + ::milvus::grpc::VectorIds *record_ids_); Status OnExecute() override; -private: + private: const ::milvus::grpc::InsertParam *insert_param_; ::milvus::grpc::VectorIds *record_ids_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class SearchTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const ::milvus::grpc::SearchParam *search_param, const std::vector &file_id_array, ::milvus::grpc::TopKQueryResultList *response); -protected: + protected: SearchTask(const ::milvus::grpc::SearchParam *search_param, const std::vector &file_id_array, ::milvus::grpc::TopKQueryResultList *response); @@ -172,7 +167,7 @@ protected: Status OnExecute() override; -private: + private: const ::milvus::grpc::SearchParam *search_param_; std::vector file_id_array_; ::milvus::grpc::TopKQueryResultList *topk_result_list; @@ -180,107 +175,106 @@ private: //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CountTableTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const std::string &table_name, int64_t &row_count); -protected: + protected: CountTableTask(const std::string &table_name, int64_t &row_count); Status OnExecute() override; -private: + private: std::string table_name_; int64_t &row_count_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CmdTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const std::string &cmd, std::string &result); -protected: + protected: CmdTask(const std::string &cmd, std::string &result); Status OnExecute() override; -private: + private: std::string cmd_; std::string &result_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DeleteByRangeTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param); -protected: - DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param); + protected: + explicit DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param); Status OnExecute() override; -private: + private: const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class PreloadTableTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const std::string &table_name); -protected: - PreloadTableTask(const std::string &table_name); + protected: + explicit PreloadTableTask(const std::string &table_name); Status OnExecute() override; -private: + private: std::string table_name_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DescribeIndexTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const std::string &table_name, - ::milvus::grpc::IndexParam *index_param); + ::milvus::grpc::IndexParam *index_param); -protected: + protected: DescribeIndexTask(const std::string &table_name, - ::milvus::grpc::IndexParam *index_param); + ::milvus::grpc::IndexParam *index_param); Status OnExecute() override; -private: + private: std::string table_name_; ::milvus::grpc::IndexParam *index_param_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DropIndexTask : public GrpcBaseTask { -public: + public: static BaseTaskPtr Create(const std::string &table_name); -protected: - DropIndexTask(const std::string &table_name); + protected: + explicit DropIndexTask(const std::string &table_name); Status OnExecute() override; -private: + private: std::string table_name_; - }; -} -} -} -} \ No newline at end of file +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcServer.cpp b/cpp/src/server/grpc_impl/GrpcServer.cpp index 7e20921549..065271dd55 100644 --- a/cpp/src/server/grpc_impl/GrpcServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcServer.cpp @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -#include "milvus.grpc.pb.h" -#include "GrpcServer.h" +#include "grpc/gen-milvus/milvus.grpc.pb.h" +#include "server/grpc_impl/GrpcServer.h" #include "server/Config.h" #include "server/DBWrapper.h" #include "utils/Log.h" @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -35,14 +36,12 @@ #include #include - namespace zilliz { namespace milvus { namespace server { namespace grpc { - -constexpr long MESSAGE_SIZE = -1; +constexpr int64_t MESSAGE_SIZE = -1; //this class is to check port occupation during server start class NoReusePortOption : public ::grpc::ServerBuilderOption { @@ -52,11 +51,9 @@ class NoReusePortOption : public ::grpc::ServerBuilderOption { } void UpdatePlugins(std::vector> *plugins) override { - } }; - void GrpcServer::Start() { thread_ptr_ = std::make_shared(&GrpcServer::StartService, this); @@ -117,7 +114,7 @@ GrpcServer::StopService() { return Status::OK(); } -} -} -} -} \ No newline at end of file +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcServer.h b/cpp/src/server/grpc_impl/GrpcServer.h index a861692fac..9101f144b3 100644 --- a/cpp/src/server/grpc_impl/GrpcServer.h +++ b/cpp/src/server/grpc_impl/GrpcServer.h @@ -19,12 +19,12 @@ #include "utils/Status.h" +#include #include #include #include #include - namespace zilliz { namespace milvus { namespace server { @@ -52,7 +52,7 @@ class GrpcServer { std::shared_ptr thread_ptr_; }; -} -} -} -} +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz From 464553521781f0683ac0384334c8cc25cee72f0d Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 27 Sep 2019 10:09:09 +0800 Subject: [PATCH 125/217] format cache code Former-commit-id: 99ff6affb78c09941a59c53183427a14b8ff00d1 --- cpp/src/cache/Cache.h | 40 +++++++++++++++--------- cpp/src/cache/Cache.inl | 53 +++++++++++++++++++------------- cpp/src/cache/CacheMgr.h | 26 ++++++++-------- cpp/src/cache/CacheMgr.inl | 58 ++++++++++++++++++++--------------- cpp/src/cache/CpuCacheMgr.cpp | 24 +++++++++------ cpp/src/cache/CpuCacheMgr.h | 17 +++++----- cpp/src/cache/DataObj.h | 37 ++++++++++++---------- cpp/src/cache/GpuCacheMgr.cpp | 26 +++++++++------- cpp/src/cache/GpuCacheMgr.h | 15 ++++----- cpp/src/cache/LRU.h | 22 +++++++------ 10 files changed, 184 insertions(+), 134 deletions(-) diff --git a/cpp/src/cache/Cache.h b/cpp/src/cache/Cache.h index b2a57ebdfb..b526e9339f 100644 --- a/cpp/src/cache/Cache.h +++ b/cpp/src/cache/Cache.h @@ -32,30 +32,40 @@ namespace cache { template class Cache { -public: + public: //mem_capacity, units:GB Cache(int64_t capacity_gb, uint64_t cache_max_count); ~Cache() = default; - int64_t usage() const { return usage_; } - int64_t capacity() const { return capacity_; } //unit: BYTE + int64_t usage() const { + return usage_; + } + + int64_t capacity() const { + return capacity_; + } //unit: BYTE void set_capacity(int64_t capacity); //unit: BYTE - double freemem_percent() const { return freemem_percent_; }; - void set_freemem_percent(double percent) { freemem_percent_ = percent; } + double freemem_percent() const { + return freemem_percent_; + } + + void set_freemem_percent(double percent) { + freemem_percent_ = percent; + } size_t size() const; - bool exists(const std::string& key); - ItemObj get(const std::string& key); - void insert(const std::string& key, const ItemObj& item); - void erase(const std::string& key); + bool exists(const std::string &key); + ItemObj get(const std::string &key); + void insert(const std::string &key, const ItemObj &item); + void erase(const std::string &key); void print(); void clear(); -private: + private: void free_memory(); -private: + private: int64_t usage_; int64_t capacity_; double freemem_percent_; @@ -64,8 +74,8 @@ private: mutable std::mutex mutex_; }; -} // cache -} // milvus -} // zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz -#include "cache/Cache.inl" \ No newline at end of file +#include "cache/Cache.inl" diff --git a/cpp/src/cache/Cache.inl b/cpp/src/cache/Cache.inl index 3eaffc4556..ebd8f3c25d 100644 --- a/cpp/src/cache/Cache.inl +++ b/cpp/src/cache/Cache.inl @@ -33,29 +33,33 @@ Cache::Cache(int64_t capacity, uint64_t cache_max_count) } template -void Cache::set_capacity(int64_t capacity) { - if(capacity > 0) { +void +Cache::set_capacity(int64_t capacity) { + if (capacity > 0) { capacity_ = capacity; free_memory(); } } template -size_t Cache::size() const { +size_t +Cache::size() const { std::lock_guard lock(mutex_); return lru_.size(); } template -bool Cache::exists(const std::string& key) { +bool +Cache::exists(const std::string &key) { std::lock_guard lock(mutex_); return lru_.exists(key); } template -ItemObj Cache::get(const std::string& key) { +ItemObj +Cache::get(const std::string &key) { std::lock_guard lock(mutex_); - if(!lru_.exists(key)){ + if (!lru_.exists(key)) { return nullptr; } @@ -63,8 +67,9 @@ ItemObj Cache::get(const std::string& key) { } template -void Cache::insert(const std::string& key, const ItemObj& item) { - if(item == nullptr) { +void +Cache::insert(const std::string &key, const ItemObj &item) { + if (item == nullptr) { return; } @@ -80,7 +85,7 @@ void Cache::insert(const std::string& key, const ItemObj& item) { //if key already exist, subtract old item size if (lru_.exists(key)) { - const ItemObj& old_item = lru_.get(key); + const ItemObj &old_item = lru_.get(key); usage_ -= old_item->size(); } @@ -107,13 +112,14 @@ void Cache::insert(const std::string& key, const ItemObj& item) { } template -void Cache::erase(const std::string& key) { +void +Cache::erase(const std::string &key) { std::lock_guard lock(mutex_); - if(!lru_.exists(key)){ + if (!lru_.exists(key)) { return; } - const ItemObj& old_item = lru_.get(key); + const ItemObj &old_item = lru_.get(key); usage_ -= old_item->size(); SERVER_LOG_DEBUG << "Erase " << key << " size: " << old_item->size(); @@ -122,7 +128,8 @@ void Cache::erase(const std::string& key) { } template -void Cache::clear() { +void +Cache::clear() { std::lock_guard lock(mutex_); lru_.clear(); usage_ = 0; @@ -131,12 +138,13 @@ void Cache::clear() { /* free memory space when CACHE occupation exceed its capacity */ template -void Cache::free_memory() { +void +Cache::free_memory() { if (usage_ <= capacity_) return; int64_t threshhold = capacity_ * freemem_percent_; int64_t delta_size = usage_ - threshhold; - if(delta_size <= 0) { + if (delta_size <= 0) { delta_size = 1;//ensure at least one item erased } @@ -148,8 +156,8 @@ void Cache::free_memory() { auto it = lru_.rbegin(); while (it != lru_.rend() && released_size < delta_size) { - auto& key = it->first; - auto& obj_ptr = it->second; + auto &key = it->first; + auto &obj_ptr = it->second; key_array.emplace(key); released_size += obj_ptr->size(); @@ -159,7 +167,7 @@ void Cache::free_memory() { SERVER_LOG_DEBUG << "to be released memory size: " << released_size; - for (auto& key : key_array) { + for (auto &key : key_array) { erase(key); } @@ -167,7 +175,8 @@ void Cache::free_memory() { } template -void Cache::print() { +void +Cache::print() { size_t cache_count = 0; { std::lock_guard lock(mutex_); @@ -179,7 +188,7 @@ void Cache::print() { SERVER_LOG_DEBUG << "[Cache capacity]: " << capacity_ << " bytes"; } -} // cache -} // milvus -} // zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/CacheMgr.h b/cpp/src/cache/CacheMgr.h index 1dd630e14d..53004d10b2 100644 --- a/cpp/src/cache/CacheMgr.h +++ b/cpp/src/cache/CacheMgr.h @@ -22,22 +22,25 @@ #include "utils/Log.h" #include "metrics/Metrics.h" +#include +#include + namespace zilliz { namespace milvus { namespace cache { template class CacheMgr { -public: + public: virtual uint64_t ItemCount() const; - virtual bool ItemExists(const std::string& key); + virtual bool ItemExists(const std::string &key); - virtual ItemObj GetItem(const std::string& key); + virtual ItemObj GetItem(const std::string &key); - virtual void InsertItem(const std::string& key, const ItemObj& data); + virtual void InsertItem(const std::string &key, const ItemObj &data); - virtual void EraseItem(const std::string& key); + virtual void EraseItem(const std::string &key); virtual void PrintInfo(); @@ -47,18 +50,17 @@ public: int64_t CacheCapacity() const; void SetCapacity(int64_t capacity); -protected: + protected: CacheMgr(); virtual ~CacheMgr(); -protected: + protected: using CachePtr = std::shared_ptr>; CachePtr cache_; }; +} // namespace cache +} // namespace milvus +} // namespace zilliz -} -} -} - -#include "cache/CacheMgr.inl" \ No newline at end of file +#include "cache/CacheMgr.inl" diff --git a/cpp/src/cache/CacheMgr.inl b/cpp/src/cache/CacheMgr.inl index 36c3d94642..b0c47b3dc3 100644 --- a/cpp/src/cache/CacheMgr.inl +++ b/cpp/src/cache/CacheMgr.inl @@ -30,18 +30,20 @@ CacheMgr::~CacheMgr() { } template -uint64_t CacheMgr::ItemCount() const { - if(cache_ == nullptr) { +uint64_t +CacheMgr::ItemCount() const { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return 0; } - return (uint64_t)(cache_->size()); + return (uint64_t) (cache_->size()); } template -bool CacheMgr::ItemExists(const std::string& key) { - if(cache_ == nullptr) { +bool +CacheMgr::ItemExists(const std::string &key) { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return false; } @@ -50,8 +52,9 @@ bool CacheMgr::ItemExists(const std::string& key) { } template -ItemObj CacheMgr::GetItem(const std::string& key) { - if(cache_ == nullptr) { +ItemObj +CacheMgr::GetItem(const std::string &key) { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return nullptr; } @@ -60,8 +63,9 @@ ItemObj CacheMgr::GetItem(const std::string& key) { } template -void CacheMgr::InsertItem(const std::string& key, const ItemObj& data) { - if(cache_ == nullptr) { +void +CacheMgr::InsertItem(const std::string &key, const ItemObj &data) { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -71,8 +75,9 @@ void CacheMgr::InsertItem(const std::string& key, const ItemObj& data) } template -void CacheMgr::EraseItem(const std::string& key) { - if(cache_ == nullptr) { +void +CacheMgr::EraseItem(const std::string &key) { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -82,8 +87,9 @@ void CacheMgr::EraseItem(const std::string& key) { } template -void CacheMgr::PrintInfo() { - if(cache_ == nullptr) { +void +CacheMgr::PrintInfo() { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -92,8 +98,9 @@ void CacheMgr::PrintInfo() { } template -void CacheMgr::ClearCache() { - if(cache_ == nullptr) { +void +CacheMgr::ClearCache() { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return; } @@ -102,8 +109,9 @@ void CacheMgr::ClearCache() { } template -int64_t CacheMgr::CacheUsage() const { - if(cache_ == nullptr) { +int64_t +CacheMgr::CacheUsage() const { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return 0; } @@ -112,8 +120,9 @@ int64_t CacheMgr::CacheUsage() const { } template -int64_t CacheMgr::CacheCapacity() const { - if(cache_ == nullptr) { +int64_t +CacheMgr::CacheCapacity() const { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return 0; } @@ -122,14 +131,15 @@ int64_t CacheMgr::CacheCapacity() const { } template -void CacheMgr::SetCapacity(int64_t capacity) { - if(cache_ == nullptr) { +void +CacheMgr::SetCapacity(int64_t capacity) { + if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return; } cache_->set_capacity(capacity); } -} -} -} +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/CpuCacheMgr.cpp b/cpp/src/cache/CpuCacheMgr.cpp index 1f93ec92bd..d26004b2be 100644 --- a/cpp/src/cache/CpuCacheMgr.cpp +++ b/cpp/src/cache/CpuCacheMgr.cpp @@ -16,20 +16,22 @@ // under the License. -#include "CpuCacheMgr.h" +#include "cache/CpuCacheMgr.h" #include "server/Config.h" #include "utils/Log.h" +#include + namespace zilliz { namespace milvus { namespace cache { namespace { - constexpr int64_t unit = 1024 * 1024 * 1024; +constexpr int64_t unit = 1024 * 1024 * 1024; } CpuCacheMgr::CpuCacheMgr() { - server::Config& config = server::Config::GetInstance(); + server::Config &config = server::Config::GetInstance(); Status s; int32_t cpu_mem_cap; @@ -38,7 +40,7 @@ CpuCacheMgr::CpuCacheMgr() { SERVER_LOG_ERROR << s.message(); } int64_t cap = cpu_mem_cap * unit; - cache_ = std::make_shared>(cap, 1UL<<32); + cache_ = std::make_shared>(cap, 1UL << 32); float cpu_mem_threshold; s = config.GetCacheConfigCpuMemThreshold(cpu_mem_threshold); @@ -53,20 +55,22 @@ CpuCacheMgr::CpuCacheMgr() { } } -CpuCacheMgr* CpuCacheMgr::GetInstance() { +CpuCacheMgr * +CpuCacheMgr::GetInstance() { static CpuCacheMgr s_mgr; return &s_mgr; } -engine::VecIndexPtr CpuCacheMgr::GetIndex(const std::string& key) { +engine::VecIndexPtr +CpuCacheMgr::GetIndex(const std::string &key) { DataObjPtr obj = GetItem(key); - if(obj != nullptr) { + if (obj != nullptr) { return obj->data(); } return nullptr; } -} -} -} \ No newline at end of file +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/CpuCacheMgr.h b/cpp/src/cache/CpuCacheMgr.h index 5a2cdb599a..32a83c4cc0 100644 --- a/cpp/src/cache/CpuCacheMgr.h +++ b/cpp/src/cache/CpuCacheMgr.h @@ -20,21 +20,24 @@ #include "CacheMgr.h" #include "DataObj.h" +#include +#include + namespace zilliz { namespace milvus { namespace cache { class CpuCacheMgr : public CacheMgr { -private: + private: CpuCacheMgr(); -public: + public: //TODO: use smart pointer instead - static CpuCacheMgr* GetInstance(); + static CpuCacheMgr *GetInstance(); - engine::VecIndexPtr GetIndex(const std::string& key); + engine::VecIndexPtr GetIndex(const std::string &key); }; -} -} -} +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/DataObj.h b/cpp/src/cache/DataObj.h index f9215bc152..6ed3256eee 100644 --- a/cpp/src/cache/DataObj.h +++ b/cpp/src/cache/DataObj.h @@ -27,38 +27,43 @@ namespace milvus { namespace cache { class DataObj { -public: - DataObj(const engine::VecIndexPtr& index) - : index_(index) - {} + public: + explicit DataObj(const engine::VecIndexPtr &index) + : index_(index) { + } - DataObj(const engine::VecIndexPtr& index, int64_t size) - : index_(index), - size_(size) - {} + DataObj(const engine::VecIndexPtr &index, int64_t size) + : index_(index), + size_(size) { + } - engine::VecIndexPtr data() { return index_; } - const engine::VecIndexPtr& data() const { return index_; } + engine::VecIndexPtr data() { + return index_; + } + + const engine::VecIndexPtr &data() const { + return index_; + } int64_t size() const { - if(index_ == nullptr) { + if (index_ == nullptr) { return 0; } - if(size_ > 0) { + if (size_ > 0) { return size_; } return index_->Count() * index_->Dimension() * sizeof(float); } -private: + private: engine::VecIndexPtr index_ = nullptr; int64_t size_ = 0; }; using DataObjPtr = std::shared_ptr; -} -} -} \ No newline at end of file +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/GpuCacheMgr.cpp b/cpp/src/cache/GpuCacheMgr.cpp index 5c104ba0e7..ad68a6ebef 100644 --- a/cpp/src/cache/GpuCacheMgr.cpp +++ b/cpp/src/cache/GpuCacheMgr.cpp @@ -16,11 +16,13 @@ // under the License. -#include +#include "cache/GpuCacheMgr.h" #include "utils/Log.h" -#include "GpuCacheMgr.h" #include "server/Config.h" +#include +#include + namespace zilliz { namespace milvus { namespace cache { @@ -29,11 +31,11 @@ std::mutex GpuCacheMgr::mutex_; std::unordered_map GpuCacheMgr::instance_; namespace { - constexpr int64_t G_BYTE = 1024 * 1024 * 1024; +constexpr int64_t G_BYTE = 1024 * 1024 * 1024; } GpuCacheMgr::GpuCacheMgr() { - server::Config& config = server::Config::GetInstance(); + server::Config &config = server::Config::GetInstance(); Status s; int32_t gpu_mem_cap; @@ -42,7 +44,7 @@ GpuCacheMgr::GpuCacheMgr() { SERVER_LOG_ERROR << s.message(); } int32_t cap = gpu_mem_cap * G_BYTE; - cache_ = std::make_shared>(cap, 1UL<<32); + cache_ = std::make_shared>(cap, 1UL << 32); float gpu_mem_threshold; s = config.GetCacheConfigGpuMemThreshold(gpu_mem_threshold); @@ -57,7 +59,8 @@ GpuCacheMgr::GpuCacheMgr() { } } -GpuCacheMgr* GpuCacheMgr::GetInstance(uint64_t gpu_id) { +GpuCacheMgr * +GpuCacheMgr::GetInstance(uint64_t gpu_id) { if (instance_.find(gpu_id) == instance_.end()) { std::lock_guard lock(mutex_); if (instance_.find(gpu_id) == instance_.end()) { @@ -70,15 +73,16 @@ GpuCacheMgr* GpuCacheMgr::GetInstance(uint64_t gpu_id) { } } -engine::VecIndexPtr GpuCacheMgr::GetIndex(const std::string& key) { +engine::VecIndexPtr +GpuCacheMgr::GetIndex(const std::string &key) { DataObjPtr obj = GetItem(key); - if(obj != nullptr) { + if (obj != nullptr) { return obj->data(); } return nullptr; } -} -} -} \ No newline at end of file +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/GpuCacheMgr.h b/cpp/src/cache/GpuCacheMgr.h index 57539b6981..843a5ff67d 100644 --- a/cpp/src/cache/GpuCacheMgr.h +++ b/cpp/src/cache/GpuCacheMgr.h @@ -21,6 +21,7 @@ #include #include +#include namespace zilliz { namespace milvus { @@ -30,18 +31,18 @@ class GpuCacheMgr; using GpuCacheMgrPtr = std::shared_ptr; class GpuCacheMgr : public CacheMgr { -public: + public: GpuCacheMgr(); - static GpuCacheMgr* GetInstance(uint64_t gpu_id); + static GpuCacheMgr *GetInstance(uint64_t gpu_id); - engine::VecIndexPtr GetIndex(const std::string& key); + engine::VecIndexPtr GetIndex(const std::string &key); -private: + private: static std::mutex mutex_; static std::unordered_map instance_; }; -} -} -} +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/LRU.h b/cpp/src/cache/LRU.h index a4f4dcf5f2..5446dd0f14 100644 --- a/cpp/src/cache/LRU.h +++ b/cpp/src/cache/LRU.h @@ -22,6 +22,7 @@ #include #include #include +#include namespace zilliz { namespace milvus { @@ -29,14 +30,15 @@ namespace cache { template class LRU { -public: + public: typedef typename std::pair key_value_pair_t; typedef typename std::list::iterator list_iterator_t; typedef typename std::list::reverse_iterator reverse_list_iterator_t; - LRU(size_t max_size) : max_size_(max_size) {} + explicit LRU(size_t max_size) : max_size_(max_size) { + } - void put(const key_t& key, const value_t& value) { + void put(const key_t &key, const value_t &value) { auto it = cache_items_map_.find(key); cache_items_list_.push_front(key_value_pair_t(key, value)); if (it != cache_items_map_.end()) { @@ -53,7 +55,7 @@ public: } } - const value_t& get(const key_t& key) { + const value_t &get(const key_t &key) { auto it = cache_items_map_.find(key); if (it == cache_items_map_.end()) { throw std::range_error("There is no such key in cache"); @@ -63,7 +65,7 @@ public: } } - void erase(const key_t& key) { + void erase(const key_t &key) { auto it = cache_items_map_.find(key); if (it != cache_items_map_.end()) { cache_items_list_.erase(it->second); @@ -71,7 +73,7 @@ public: } } - bool exists(const key_t& key) const { + bool exists(const key_t &key) const { return cache_items_map_.find(key) != cache_items_map_.end(); } @@ -101,14 +103,14 @@ public: cache_items_map_.clear(); } -private: + private: std::list cache_items_list_; std::unordered_map cache_items_map_; size_t max_size_; list_iterator_t iter_; }; -} // cache -} // milvus -} // zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz From 3a6448b3e8e291094043d2da40ad4320a230b03c Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Fri, 27 Sep 2019 10:37:07 +0800 Subject: [PATCH 126/217] MS-574 update config unittest Former-commit-id: f60c3ada442acea12827bcae136a29adbe6f9c2e --- cpp/unittest/server/appendix/log_config.conf | 14 ++--- .../server/appendix/server_config.yaml | 54 ++++++++---------- cpp/unittest/server/config_test.cpp | 57 ++----------------- 3 files changed, 36 insertions(+), 89 deletions(-) diff --git a/cpp/unittest/server/appendix/log_config.conf b/cpp/unittest/server/appendix/log_config.conf index 29d46a7fe5..0a3e0d21af 100644 --- a/cpp/unittest/server/appendix/log_config.conf +++ b/cpp/unittest/server/appendix/log_config.conf @@ -1,19 +1,19 @@ * GLOBAL: FORMAT = "%datetime | %level | %logger | %msg" - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-global.log" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-global.log" ENABLED = true TO_FILE = true TO_STANDARD_OUTPUT = false SUBSECOND_PRECISION = 3 PERFORMANCE_TRACKING = false - MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB + MAX_LOG_FILE_SIZE = 209715200 ## Throw log files away after 200MB * DEBUG: - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-debug.log" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-debug.log" ENABLED = true * WARNING: - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-warning.log" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-warning.log" * TRACE: - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-trace.log" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-trace.log" * VERBOSE: FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" TO_FILE = false @@ -21,7 +21,7 @@ ## Error logs * ERROR: ENABLED = true - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-error.log" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-error.log" * FATAL: ENABLED = true - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log" diff --git a/cpp/unittest/server/appendix/server_config.yaml b/cpp/unittest/server/appendix/server_config.yaml index e4b3de31b0..f92b2f1a18 100644 --- a/cpp/unittest/server/appendix/server_config.yaml +++ b/cpp/unittest/server/appendix/server_config.yaml @@ -1,45 +1,37 @@ +# All the following configurations are default values. + server_config: - address: 0.0.0.0 # milvus server ip address (IPv4) - port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 - mode: single # milvus deployment type: single, cluster, read_only - time_zone: UTC+8 # Use the UTC-x or UTC+x to specify a time zone. eg. UTC+8 for China Standard Time + address: 0.0.0.0 # milvus server ip address (IPv4) + port: 19530 # port range: 1025 ~ 65534 + deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable + time_zone: UTC+8 db_config: - db_path: /tmp/milvus # milvus data storage path - db_slave_path: # secondry data storage path, split by semicolon + primary_path: /tmp/milvus # path used to store data and meta + secondary_path: # path used to store data only, split by semicolon - # URI format: dialect://username:password@host:port/database - # All parts except dialect are optional, but you MUST include the delimiters - # Currently dialect supports mysql or sqlite - db_backend_url: sqlite://:@:/ + backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database + # Keep 'dialect://:@:/', and replace other texts with real values. + # Replace 'dialect' with 'mysql' or 'sqlite' - archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB - archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day - insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB. - # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB - build_index_gpu: 0 # which gpu is used to build index, default: 0, range: 0 ~ gpu number - 1 + insert_buffer_size: 4 # GB, maximum insert buffer size allowed + build_index_gpu: 0 # gpu id used for building index metric_config: - is_startup: off # if monitoring start: on, off - collector: prometheus # metrics collector: prometheus - prometheus_config: # following are prometheus configure - port: 8080 # the port prometheus use to fetch metrics - push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address - push_gateway_port: 9091 # push method configure: push gateway port + enable_monitor: false # enable monitoring or not + collector: prometheus # prometheus + prometheus_config: + port: 8080 # port prometheus used to fetch metrics cache_config: - cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory - cpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 - insert_cache_immediately: false # insert data will be load into cache immediately for hot query - gpu_cache_capacity: 5 # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory - gpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 + cpu_mem_capacity: 16 # GB, CPU memory used for cache + cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered + cache_insert_data: false # whether load inserted data into cache engine_config: - use_blas_threshold: 20 + blas_threshold: 20 resource_config: - mode: simple - resources: - # - cpu + resource_pool: + - cpu - gpu0 - diff --git a/cpp/unittest/server/config_test.cpp b/cpp/unittest/server/config_test.cpp index f886e4f05a..61e9ce44ad 100644 --- a/cpp/unittest/server/config_test.cpp +++ b/cpp/unittest/server/config_test.cpp @@ -99,60 +99,15 @@ TEST(ConfigTest, CONFIG_TEST) { server_config.ClearSequences(); auto seqs = server_config.GetSequences(); ASSERT_TRUE(seqs.empty()); - - const server::ConfigNode const_node = root_config.GetChild("cache_config"); - float flt = const_node.GetFloatValue("cpu_cache_capacity"); - ASSERT_GT(flt, 0.0); } TEST(ConfigTest, SERVER_CONFIG_TEST) { - auto status = server::Config::GetInstance().LoadConfigFile(CONFIG_FILE_PATH); - ASSERT_TRUE(status.ok()); + server::Config& config = server::Config::GetInstance(); + Status s = config.LoadConfigFile(CONFIG_FILE_PATH); + ASSERT_TRUE(s.ok()); -// status = server::Config::GetInstance().ValidateConfig(); -// ASSERT_TRUE(status.ok()); -// -// const server::ServerConfig& config_const = config; -// server::ConfigNode node1 = config_const.GetConfig("server_config"); -// server::ConfigNode& node2 = config.GetConfig("cache_config"); -// node1.Combine(node2); -// -// int32_t cap = node1.GetInt32Value("cpu_cache_capacity"); -// ASSERT_GT(cap, 0); -// -// node1.SetValue("bool", "true"); -// bool bt = node1.GetBoolValue("bool"); -// ASSERT_TRUE(bt); + s = config.ValidateConfig(); + ASSERT_TRUE(s.ok()); -// server::Config::GetInstance().PrintAll(); -// -// unsigned long total_mem = 0, free_mem = 0; -// server::CommonUtil::GetSystemMemInfo(total_mem, free_mem); -// -// size_t gpu_mem = 0; -// server::ValidationUtil::GetGpuMemory(0, gpu_mem); -// -// server::ConfigNode& server_config = config.GetConfig("server_config"); -// server::ConfigNode& db_config = config.GetConfig("db_config"); -// server::ConfigNode& cache_config = config.GetConfig(server::CONFIG_CACHE); -// cache_config.SetValue(server::CACHE_FREE_PERCENT, "2.0"); -// status = config.ValidateConfig(); -// ASSERT_FALSE(status.ok()); -// -// size_t cache_cap = 16; -// size_t insert_buffer_size = (total_mem - cache_cap*GB + 1*GB)/GB; -// db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size)); -// cache_config.SetValue(server::CONFIG_CPU_CACHE_CAPACITY, std::to_string(cache_cap)); -// status = config.ValidateConfig(); -// ASSERT_FALSE(status.ok()); -// -// cache_cap = total_mem/GB + 2; -// cache_config.SetValue(server::CONFIG_CPU_CACHE_CAPACITY, std::to_string(cache_cap)); -// status = config.ValidateConfig(); -// ASSERT_FALSE(status.ok()); -// -// insert_buffer_size = total_mem/GB + 2; -// db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size)); -// status = config.ValidateConfig(); -// ASSERT_FALSE(status.ok()); + config.PrintAll(); } \ No newline at end of file From a4f1a6392faba86bb2fd3e01b97782d40e41b914 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 27 Sep 2019 11:13:26 +0800 Subject: [PATCH 127/217] format sdk code Former-commit-id: be186a4338c2921bd22bae6091e6f145f20ccc46 --- cpp/src/main.cpp | 2 - cpp/src/sdk/examples/grpcsimple/main.cpp | 9 +- .../examples/grpcsimple/src/ClientTest.cpp | 182 ++++++++++-------- .../sdk/examples/grpcsimple/src/ClientTest.h | 6 +- cpp/src/sdk/grpc/ClientProxy.cpp | 54 +++--- cpp/src/sdk/grpc/ClientProxy.h | 68 ++++--- cpp/src/sdk/grpc/GrpcClient.cpp | 43 +++-- cpp/src/sdk/grpc/GrpcClient.h | 44 ++--- cpp/src/sdk/include/MilvusApi.h | 25 +-- cpp/src/sdk/include/Status.h | 19 +- cpp/src/sdk/interface/ConnectionImpl.cpp | 25 ++- cpp/src/sdk/interface/ConnectionImpl.h | 68 ++++--- cpp/src/sdk/interface/Status.cpp | 26 +-- 13 files changed, 295 insertions(+), 276 deletions(-) diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index ca3283f4d0..22c282d4ea 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -76,7 +76,6 @@ main(int argc, char *argv[]) { std::cout << "Initial log config from: " << log_config_file << std::endl; break; } - case 'p': { char *pid_filename_ptr = strdup(optarg); pid_filename = pid_filename_ptr; @@ -84,7 +83,6 @@ main(int argc, char *argv[]) { std::cout << pid_filename << std::endl; break; } - case 'd': start_daemonized = 1; break; diff --git a/cpp/src/sdk/examples/grpcsimple/main.cpp b/cpp/src/sdk/examples/grpcsimple/main.cpp index 4de060590c..166707259a 100644 --- a/cpp/src/sdk/examples/grpcsimple/main.cpp +++ b/cpp/src/sdk/examples/grpcsimple/main.cpp @@ -23,8 +23,8 @@ #include "src/ClientTest.h" -void print_help(const std::string &app_name); - +void +print_help(const std::string &app_name); int main(int argc, char *argv[]) { @@ -56,8 +56,7 @@ main(int argc, char *argv[]) { break; } case 'h': - default: - print_help(app_name); + default:print_help(app_name); return EXIT_SUCCESS; } } @@ -77,4 +76,4 @@ print_help(const std::string &app_name) { printf(" -p --port Server port, default 19530\n"); printf(" -h --help Print help information\n"); printf("\n"); -} \ No newline at end of file +} diff --git a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp index 16fc2ee168..e29bcc30f3 100644 --- a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "ClientTest.h" +#include "sdk/examples/grpcsimple/src/ClientTest.h" #include "MilvusApi.h" #include "cache/CpuCacheMgr.h" @@ -24,15 +24,17 @@ #include #include #include - -using namespace milvus; +#include +#include +#include //#define SET_VECTOR_IDS; namespace { -std::string GetTableName(); +const std::string& +GetTableName(); -const std::string TABLE_NAME = GetTableName(); +const char* TABLE_NAME = GetTableName().c_str(); constexpr int64_t TABLE_DIMENSION = 512; constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; constexpr int64_t BATCH_ROW_COUNT = 100000; @@ -44,26 +46,28 @@ constexpr int64_t SECONDS_EACH_HOUR = 3600; #define BLOCK_SPLITER std::cout << "===========================================" << std::endl; -void PrintTableSchema(const TableSchema& tb_schema) { +void +PrintTableSchema(const milvus::TableSchema &tb_schema) { BLOCK_SPLITER std::cout << "Table name: " << tb_schema.table_name << std::endl; std::cout << "Table dimension: " << tb_schema.dimension << std::endl; BLOCK_SPLITER } -void PrintSearchResult(const std::vector>& search_record_array, - const std::vector& topk_query_result_array) { +void +PrintSearchResult(const std::vector> &search_record_array, + const std::vector &topk_query_result_array) { BLOCK_SPLITER std::cout << "Returned result count: " << topk_query_result_array.size() << std::endl; int32_t index = 0; - for(auto& result : topk_query_result_array) { + for (auto &result : topk_query_result_array) { auto search_id = search_record_array[index].first; index++; std::cout << "No." << std::to_string(index) << " vector " << std::to_string(search_id) << " top " << std::to_string(result.query_result_arrays.size()) << " search result:" << std::endl; - for(auto& item : result.query_result_arrays) { + for (auto &item : result.query_result_arrays) { std::cout << "\t" << std::to_string(item.id) << "\tdistance:" << std::to_string(item.distance); std::cout << std::endl; } @@ -72,80 +76,88 @@ void PrintSearchResult(const std::vector>& search_ BLOCK_SPLITER } -std::string CurrentTime() { +std::string +CurrentTime() { time_t tt; - time( &tt ); - tt = tt + 8*SECONDS_EACH_HOUR; - tm* t= gmtime( &tt ); + time(&tt); + tt = tt + 8 * SECONDS_EACH_HOUR; + tm t; + gmtime_r(&tt, &t); - std::string str = std::to_string(t->tm_year + 1900) + "_" + std::to_string(t->tm_mon + 1) - + "_" + std::to_string(t->tm_mday) + "_" + std::to_string(t->tm_hour) - + "_" + std::to_string(t->tm_min) + "_" + std::to_string(t->tm_sec); + std::string str = std::to_string(t.tm_year + 1900) + "_" + std::to_string(t.tm_mon + 1) + + "_" + std::to_string(t.tm_mday) + "_" + std::to_string(t.tm_hour) + + "_" + std::to_string(t.tm_min) + "_" + std::to_string(t.tm_sec); return str; } -std::string CurrentTmDate(int64_t offset_day = 0) { +std::string +CurrentTmDate(int64_t offset_day = 0) { time_t tt; - time( &tt ); - tt = tt + 8*SECONDS_EACH_HOUR; - tt = tt + 24*SECONDS_EACH_HOUR*offset_day; - tm* t= gmtime( &tt ); + time(&tt); + tt = tt + 8 * SECONDS_EACH_HOUR; + tt = tt + 24 * SECONDS_EACH_HOUR * offset_day; + tm t; + gmtime_r(&tt, &t); - std::string str = std::to_string(t->tm_year + 1900) + "-" + std::to_string(t->tm_mon + 1) - + "-" + std::to_string(t->tm_mday); + std::string str = std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1) + + "-" + std::to_string(t.tm_mday); return str; } -std::string GetTableName() { - static std::string s_id(CurrentTime()); - return "tbl_" + s_id; +const std::string& +GetTableName() { + static std::string s_id("tbl_" + CurrentTime()); + return s_id; } -TableSchema BuildTableSchema() { - TableSchema tb_schema; +milvus::TableSchema +BuildTableSchema() { + milvus::TableSchema tb_schema; tb_schema.table_name = TABLE_NAME; tb_schema.dimension = TABLE_DIMENSION; tb_schema.index_file_size = TABLE_INDEX_FILE_SIZE; - tb_schema.metric_type = MetricType::L2; + tb_schema.metric_type = milvus::MetricType::L2; return tb_schema; } -void BuildVectors(int64_t from, int64_t to, - std::vector& vector_record_array) { - if(to <= from){ +void +BuildVectors(int64_t from, int64_t to, + std::vector &vector_record_array) { + if (to <= from) { return; } vector_record_array.clear(); for (int64_t k = from; k < to; k++) { - RowRecord record; + milvus::RowRecord record; record.data.resize(TABLE_DIMENSION); - for(int64_t i = 0; i < TABLE_DIMENSION; i++) { - record.data[i] = (float)(k%(i+1)); + for (int64_t i = 0; i < TABLE_DIMENSION; i++) { + record.data[i] = (float) (k % (i + 1)); } vector_record_array.emplace_back(record); } } -void Sleep(int seconds) { +void +Sleep(int seconds) { std::cout << "Waiting " << seconds << " seconds ..." << std::endl; sleep(seconds); } class TimeRecorder { public: - explicit TimeRecorder(const std::string& title) + explicit TimeRecorder(const std::string &title) : title_(title) { start_ = std::chrono::system_clock::now(); } ~TimeRecorder() { std::chrono::system_clock::time_point end = std::chrono::system_clock::now(); - long span = (std::chrono::duration_cast (end - start_)).count(); + int64_t span = (std::chrono::duration_cast(end - start_)).count(); std::cout << title_ << " totally cost: " << span << " ms" << std::endl; } @@ -154,14 +166,15 @@ class TimeRecorder { std::chrono::system_clock::time_point start_; }; -void CheckResult(const std::vector>& search_record_array, - const std::vector& topk_query_result_array) { +void +CheckResult(const std::vector> &search_record_array, + const std::vector &topk_query_result_array) { BLOCK_SPLITER int64_t index = 0; - for(auto& result : topk_query_result_array) { + for (auto &result : topk_query_result_array) { auto result_id = result.query_result_arrays[0].id; auto search_id = search_record_array[index++].first; - if(result_id != search_id) { + if (result_id != search_id) { std::cout << "The top 1 result is wrong: " << result_id << " vs. " << search_id << std::endl; } else { @@ -171,42 +184,45 @@ void CheckResult(const std::vector>& search_record BLOCK_SPLITER } -void DoSearch(std::shared_ptr conn, - const std::vector>& search_record_array, - const std::string& phase_name) { - std::vector query_range_array; - Range rg; +void +DoSearch(std::shared_ptr conn, + const std::vector> &search_record_array, + const std::string &phase_name) { + std::vector query_range_array; + milvus::Range rg; rg.start_value = CurrentTmDate(); rg.end_value = CurrentTmDate(1); query_range_array.emplace_back(rg); - std::vector record_array; - for(auto& pair : search_record_array) { + std::vector record_array; + for (auto &pair : search_record_array) { record_array.push_back(pair.second); } auto start = std::chrono::high_resolution_clock::now(); - std::vector topk_query_result_array; + std::vector topk_query_result_array; { TimeRecorder rc(phase_name); - Status stat = conn->Search(TABLE_NAME, record_array, query_range_array, TOP_K, 32, topk_query_result_array); + milvus::Status stat = + conn->Search(TABLE_NAME, record_array, query_range_array, TOP_K, 32, topk_query_result_array); std::cout << "SearchVector function call status: " << stat.message() << std::endl; } auto finish = std::chrono::high_resolution_clock::now(); - std::cout << "SEARCHVECTOR COST: " << std::chrono::duration_cast>(finish - start).count() << "s\n"; + std::cout << "SEARCHVECTOR COST: " + << std::chrono::duration_cast>(finish - start).count() << "s\n"; PrintSearchResult(search_record_array, topk_query_result_array); CheckResult(search_record_array, topk_query_result_array); } -} +} // namespace void -ClientTest::Test(const std::string& address, const std::string& port) { - std::shared_ptr conn = Connection::Create(); +ClientTest::Test(const std::string &address, const std::string &port) { + std::shared_ptr conn = milvus::Connection::Create(); {//connect server - ConnectParam param = {address, port}; - Status stat = conn->Connect(param); + milvus::ConnectParam param = {address, port}; + milvus::Status stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } @@ -222,10 +238,10 @@ ClientTest::Test(const std::string& address, const std::string& port) { { std::vector tables; - Status stat = conn->ShowTables(tables); + milvus::Status stat = conn->ShowTables(tables); std::cout << "ShowTables function call status: " << stat.message() << std::endl; std::cout << "All tables: " << std::endl; - for(auto& table : tables) { + for (auto &table : tables) { int64_t row_count = 0; // conn->DropTable(table); stat = conn->CountTable(table, row_count); @@ -234,28 +250,28 @@ ClientTest::Test(const std::string& address, const std::string& port) { } {//create table - TableSchema tb_schema = BuildTableSchema(); - Status stat = conn->CreateTable(tb_schema); + milvus::TableSchema tb_schema = BuildTableSchema(); + milvus::Status stat = conn->CreateTable(tb_schema); std::cout << "CreateTable function call status: " << stat.message() << std::endl; PrintTableSchema(tb_schema); bool has_table = conn->HasTable(tb_schema.table_name); - if(has_table) { + if (has_table) { std::cout << "Table is created" << std::endl; } } {//describe table - TableSchema tb_schema; - Status stat = conn->DescribeTable(TABLE_NAME, tb_schema); + milvus::TableSchema tb_schema; + milvus::Status stat = conn->DescribeTable(TABLE_NAME, tb_schema); std::cout << "DescribeTable function call status: " << stat.message() << std::endl; PrintTableSchema(tb_schema); } - std::vector> search_record_array; + std::vector> search_record_array; {//insert vectors for (int i = 0; i < ADD_VECTOR_LOOP; i++) {//add vectors - std::vector record_array; + std::vector record_array; int64_t begin_index = i * BATCH_ROW_COUNT; BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array); @@ -268,21 +284,21 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::vector record_ids; //generate user defined ids - for(int k = 0; k < BATCH_ROW_COUNT; k++) { - record_ids.push_back(i*BATCH_ROW_COUNT+k); + for (int k = 0; k < BATCH_ROW_COUNT; k++) { + record_ids.push_back(i * BATCH_ROW_COUNT + k); } auto start = std::chrono::high_resolution_clock::now(); - Status stat = conn->Insert(TABLE_NAME, record_array, record_ids); + milvus::Status stat = conn->Insert(TABLE_NAME, record_array, record_ids); auto finish = std::chrono::high_resolution_clock::now(); - std::cout << "InsertVector cost: " << std::chrono::duration_cast>(finish - start).count() << "s\n"; - + std::cout << "InsertVector cost: " + << std::chrono::duration_cast>(finish - start).count() << "s\n"; std::cout << "InsertVector function call status: " << stat.message() << std::endl; std::cout << "Returned id array count: " << record_ids.size() << std::endl; - if(search_record_array.size() < NQ) { + if (search_record_array.size() < NQ) { search_record_array.push_back( std::make_pair(record_ids[SEARCH_TARGET], record_array[SEARCH_TARGET])); } @@ -293,27 +309,27 @@ ClientTest::Test(const std::string& address, const std::string& port) { Sleep(2); int64_t row_count = 0; - Status stat = conn->CountTable(TABLE_NAME, row_count); + milvus::Status stat = conn->CountTable(TABLE_NAME, row_count); std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; // DoSearch(conn, search_record_array, "Search without index"); } {//wait unit build index finish std::cout << "Wait until create all index done" << std::endl; - IndexParam index; + milvus::IndexParam index; index.table_name = TABLE_NAME; - index.index_type = IndexType::gpu_ivfsq8; + index.index_type = milvus::IndexType::gpu_ivfsq8; index.nlist = 16384; - Status stat = conn->CreateIndex(index); + milvus::Status stat = conn->CreateIndex(index); std::cout << "CreateIndex function call status: " << stat.message() << std::endl; - IndexParam index2; + milvus::IndexParam index2; stat = conn->DescribeIndex(TABLE_NAME, index2); std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; } {//preload table - Status stat = conn->PreloadTable(TABLE_NAME); + milvus::Status stat = conn->PreloadTable(TABLE_NAME); std::cout << "PreloadTable function call status: " << stat.message() << std::endl; } @@ -325,7 +341,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { } {//delete index - Status stat = conn->DropIndex(TABLE_NAME); + milvus::Status stat = conn->DropIndex(TABLE_NAME); std::cout << "DropIndex function call status: " << stat.message() << std::endl; int64_t row_count = 0; @@ -334,11 +350,11 @@ ClientTest::Test(const std::string& address, const std::string& port) { } {//delete by range - Range rg; + milvus::Range rg; rg.start_value = CurrentTmDate(-2); rg.end_value = CurrentTmDate(-3); - Status stat = conn->DeleteByRange(rg, TABLE_NAME); + milvus::Status stat = conn->DeleteByRange(rg, TABLE_NAME); std::cout << "DeleteByRange function call status: " << stat.message() << std::endl; } @@ -351,7 +367,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::string status = conn->ServerStatus(); std::cout << "Server status before disconnect: " << status << std::endl; } - Connection::Destroy(conn); + milvus::Connection::Destroy(conn); {//server status std::string status = conn->ServerStatus(); std::cout << "Server status after disconnect: " << status << std::endl; diff --git a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h index d95897927e..165c36a180 100644 --- a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h +++ b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h @@ -20,6 +20,6 @@ #include class ClientTest { -public: - void Test(const std::string& address, const std::string& port); -}; \ No newline at end of file + public: + void Test(const std::string &address, const std::string &port); +}; diff --git a/cpp/src/sdk/grpc/ClientProxy.cpp b/cpp/src/sdk/grpc/ClientProxy.cpp index 3bcfd699fe..f4cd1e9a38 100644 --- a/cpp/src/sdk/grpc/ClientProxy.cpp +++ b/cpp/src/sdk/grpc/ClientProxy.cpp @@ -15,16 +15,20 @@ // specific language governing permissions and limitations // under the License. -#include "ClientProxy.h" -#include "version.h" -#include "milvus.grpc.pb.h" +#include "sdk/grpc/ClientProxy.h" +#include "../../../version.h" +#include "grpc/gen-milvus/milvus.grpc.pb.h" + +#include +#include +#include + //#define GRPC_MULTIPLE_THREAD; namespace milvus { - bool UriCheck(const std::string &uri) { - size_t index = uri.find_first_of(':', 0); + size_t index = uri.find_first_of(':', 0); if (index == std::string::npos) { return false; } else { @@ -79,7 +83,7 @@ ClientProxy::Disconnect() { connected_ = false; channel_.reset(); return status; - }catch (std::exception &ex) { + } catch (std::exception &ex) { return Status(StatusCode::UnknownError, "failed to disconnect: " + std::string(ex.what())); } } @@ -96,7 +100,7 @@ ClientProxy::CreateTable(const TableSchema ¶m) { schema.set_table_name(param.table_name); schema.set_dimension(param.dimension); schema.set_index_file_size(param.index_file_size); - schema.set_metric_type((int32_t)param.metric_type); + schema.set_metric_type((int32_t) param.metric_type); return client_ptr_->CreateTable(schema); } catch (std::exception &ex) { @@ -127,13 +131,12 @@ ClientProxy::DropTable(const std::string &table_name) { Status ClientProxy::CreateIndex(const IndexParam &index_param) { try { - //TODO:add index params + //TODO: add index params ::milvus::grpc::IndexParam grpc_index_param; grpc_index_param.set_table_name(index_param.table_name); - grpc_index_param.mutable_index()->set_index_type((int32_t)index_param.index_type); + grpc_index_param.mutable_index()->set_index_type((int32_t) index_param.index_type); grpc_index_param.mutable_index()->set_nlist(index_param.nlist); return client_ptr_->CreateIndex(grpc_index_param); - } catch (std::exception &ex) { return Status(StatusCode::UnknownError, "failed to build index: " + std::string(ex.what())); } @@ -141,8 +144,8 @@ ClientProxy::CreateIndex(const IndexParam &index_param) { Status ClientProxy::Insert(const std::string &table_name, - const std::vector &record_array, - std::vector &id_array) { + const std::vector &record_array, + std::vector &id_array) { Status status = Status::OK(); try { //////////////////////////////////////////////////////////////////////////// @@ -181,7 +184,9 @@ ClientProxy::Insert(const std::string &table_name, } std::for_each(threads.begin(), threads.end(), std::mem_fn(&std::thread::join)); auto finish = std::chrono::high_resolution_clock::now(); - std::cout << "InsertVector cost: " << std::chrono::duration_cast>(finish - start).count() << "s\n"; + std::cout << + "InsertVector cost: " << std::chrono::duration_cast>(finish - start).count() + << "s\n"; std::cout << "*****************************************************\n"; for (size_t i = 0; i < thread_count; i++) { @@ -213,9 +218,7 @@ ClientProxy::Insert(const std::string &table_name, id_array.push_back(vector_ids.vector_id_array(i)); } } - #endif - } catch (std::exception &ex) { return Status(StatusCode::UnknownError, "fail to add vector: " + std::string(ex.what())); } @@ -225,11 +228,11 @@ ClientProxy::Insert(const std::string &table_name, Status ClientProxy::Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) { + const std::vector &query_record_array, + const std::vector &query_range_array, + int64_t topk, + int64_t nprobe, + std::vector &topk_query_result_array) { try { //step 1: convert vectors data ::milvus::grpc::SearchParam search_param; @@ -267,11 +270,9 @@ ClientProxy::Search(const std::string &table_name, topk_query_result_array.emplace_back(result); } return status; - } catch (std::exception &ex) { return Status(StatusCode::UnknownError, "fail to search vectors: " + std::string(ex.what())); } - } Status @@ -284,13 +285,12 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch table_schema.table_name = grpc_schema.table_name(); table_schema.dimension = grpc_schema.dimension(); table_schema.index_file_size = grpc_schema.index_file_size(); - table_schema.metric_type = (MetricType)grpc_schema.metric_type(); + table_schema.metric_type = (MetricType) grpc_schema.metric_type(); return status; } catch (std::exception &ex) { return Status(StatusCode::UnknownError, "fail to describe table: " + std::string(ex.what())); } - } Status @@ -316,7 +316,6 @@ ClientProxy::ShowTables(std::vector &table_array) { table_array[i] = table_name_list.table_names(i); } return status; - } catch (std::exception &ex) { return Status(StatusCode::UnknownError, "fail to show tables: " + std::string(ex.what())); } @@ -396,11 +395,10 @@ ClientProxy::DescribeIndex(const std::string &table_name, IndexParam &index_para grpc_table_name.set_table_name(table_name); ::milvus::grpc::IndexParam grpc_index_param; Status status = client_ptr_->DescribeIndex(grpc_table_name, grpc_index_param); - index_param.index_type = (IndexType)(grpc_index_param.mutable_index()->index_type()); + index_param.index_type = (IndexType) (grpc_index_param.mutable_index()->index_type()); index_param.nlist = grpc_index_param.mutable_index()->nlist(); return status; - } catch (std::exception &ex) { return Status(StatusCode::UnknownError, "fail to describe index: " + std::string(ex.what())); } @@ -418,4 +416,4 @@ ClientProxy::DropIndex(const std::string &table_name) const { } } -} +} // namespace milvus diff --git a/cpp/src/sdk/grpc/ClientProxy.h b/cpp/src/sdk/grpc/ClientProxy.h index c3f79d7209..044ce44847 100644 --- a/cpp/src/sdk/grpc/ClientProxy.h +++ b/cpp/src/sdk/grpc/ClientProxy.h @@ -20,88 +20,92 @@ #include "MilvusApi.h" #include "GrpcClient.h" +#include +#include +#include + namespace milvus { class ClientProxy : public Connection { -public: + public: // Implementations of the Connection interface - virtual Status + Status Connect(const ConnectParam ¶m) override; - virtual Status + Status Connect(const std::string &uri) override; - virtual Status + Status Connected() const override; - virtual Status + Status Disconnect() override; - virtual Status + Status CreateTable(const TableSchema ¶m) override; - virtual bool + bool HasTable(const std::string &table_name) override; - virtual Status + Status DropTable(const std::string &table_name) override; - virtual Status + Status CreateIndex(const IndexParam &index_param) override; - virtual Status + Status Insert(const std::string &table_name, - const std::vector &record_array, - std::vector &id_array) override; + const std::vector &record_array, + std::vector &id_array) override; - virtual Status + Status Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) override; + const std::vector &query_record_array, + const std::vector &query_range_array, + int64_t topk, + int64_t nprobe, + std::vector &topk_query_result_array) override; - virtual Status + Status DescribeTable(const std::string &table_name, TableSchema &table_schema) override; - virtual Status + Status CountTable(const std::string &table_name, int64_t &row_count) override; - virtual Status + Status ShowTables(std::vector &table_array) override; - virtual std::string + std::string ClientVersion() const override; - virtual std::string + std::string ServerVersion() const override; - virtual std::string + std::string ServerStatus() const override; - virtual std::string + std::string DumpTaskTables() const override; - virtual Status + Status DeleteByRange(Range &range, const std::string &table_name) override; - virtual Status + Status PreloadTable(const std::string &table_name) const override; - virtual Status + Status DescribeIndex(const std::string &table_name, IndexParam &index_param) const override; - virtual Status + Status DropIndex(const std::string &table_name) const override; -private: + private: std::shared_ptr<::grpc::Channel> channel_; -private: + private: std::shared_ptr client_ptr_; bool connected_ = false; }; -} +} // namespace milvus diff --git a/cpp/src/sdk/grpc/GrpcClient.cpp b/cpp/src/sdk/grpc/GrpcClient.cpp index 67bb85da8e..df86f0a3e7 100644 --- a/cpp/src/sdk/grpc/GrpcClient.cpp +++ b/cpp/src/sdk/grpc/GrpcClient.cpp @@ -15,13 +15,17 @@ // specific language governing permissions and limitations // under the License. +#include "sdk/grpc/GrpcClient.h" + #include #include #include #include #include -#include "GrpcClient.h" +#include +#include +#include using grpc::Channel; using grpc::ClientContext; @@ -31,15 +35,14 @@ using grpc::ClientWriter; using grpc::Status; namespace milvus { -GrpcClient::GrpcClient(std::shared_ptr<::grpc::Channel>& channel) - : stub_(::milvus::grpc::MilvusService::NewStub(channel)) { - +GrpcClient::GrpcClient(std::shared_ptr<::grpc::Channel> &channel) + : stub_(::milvus::grpc::MilvusService::NewStub(channel)) { } GrpcClient::~GrpcClient() = default; Status -GrpcClient::CreateTable(const ::milvus::grpc::TableSchema& table_schema) { +GrpcClient::CreateTable(const ::milvus::grpc::TableSchema &table_schema) { ClientContext context; grpc::Status response; ::grpc::Status grpc_status = stub_->CreateTable(&context, table_schema, &response); @@ -57,8 +60,8 @@ GrpcClient::CreateTable(const ::milvus::grpc::TableSchema& table_schema) { } bool -GrpcClient::HasTable(const ::milvus::grpc::TableName& table_name, - Status& status) { +GrpcClient::HasTable(const ::milvus::grpc::TableName &table_name, + Status &status) { ClientContext context; ::milvus::grpc::BoolReply response; ::grpc::Status grpc_status = stub_->HasTable(&context, table_name, &response); @@ -76,7 +79,7 @@ GrpcClient::HasTable(const ::milvus::grpc::TableName& table_name, } Status -GrpcClient::DropTable(const ::milvus::grpc::TableName& table_name) { +GrpcClient::DropTable(const ::milvus::grpc::TableName &table_name) { ClientContext context; grpc::Status response; ::grpc::Status grpc_status = stub_->DropTable(&context, table_name, &response); @@ -94,7 +97,7 @@ GrpcClient::DropTable(const ::milvus::grpc::TableName& table_name) { } Status -GrpcClient::CreateIndex(const ::milvus::grpc::IndexParam& index_param) { +GrpcClient::CreateIndex(const ::milvus::grpc::IndexParam &index_param) { ClientContext context; grpc::Status response; ::grpc::Status grpc_status = stub_->CreateIndex(&context, index_param, &response); @@ -112,9 +115,9 @@ GrpcClient::CreateIndex(const ::milvus::grpc::IndexParam& index_param) { } void -GrpcClient::Insert(::milvus::grpc::VectorIds& vector_ids, - const ::milvus::grpc::InsertParam& insert_param, - Status& status) { +GrpcClient::Insert(::milvus::grpc::VectorIds &vector_ids, + const ::milvus::grpc::InsertParam &insert_param, + Status &status) { ClientContext context; ::grpc::Status grpc_status = stub_->Insert(&context, insert_param, &vector_ids); @@ -133,7 +136,7 @@ GrpcClient::Insert(::milvus::grpc::VectorIds& vector_ids, } Status -GrpcClient::Search(::milvus::grpc::TopKQueryResultList& topk_query_result_list, +GrpcClient::Search(::milvus::grpc::TopKQueryResultList &topk_query_result_list, const ::milvus::grpc::SearchParam &search_param) { ::milvus::grpc::TopKQueryResult query_result; ClientContext context; @@ -154,8 +157,8 @@ GrpcClient::Search(::milvus::grpc::TopKQueryResultList& topk_query_result_list, } Status -GrpcClient::DescribeTable(::milvus::grpc::TableSchema& grpc_schema, - const std::string& table_name) { +GrpcClient::DescribeTable(::milvus::grpc::TableSchema &grpc_schema, + const std::string &table_name) { ClientContext context; ::milvus::grpc::TableName grpc_tablename; grpc_tablename.set_table_name(table_name); @@ -170,14 +173,14 @@ GrpcClient::DescribeTable(::milvus::grpc::TableSchema& grpc_schema, if (grpc_schema.status().error_code() != grpc::SUCCESS) { std::cerr << grpc_schema.status().reason() << std::endl; return Status(StatusCode::ServerFailed, - grpc_schema.status().reason()); + grpc_schema.status().reason()); } return Status::OK(); } int64_t -GrpcClient::CountTable(const std::string& table_name, Status& status) { +GrpcClient::CountTable(const std::string &table_name, Status &status) { ClientContext context; ::milvus::grpc::TableRowCount response; ::milvus::grpc::TableName grpc_tablename; @@ -186,7 +189,7 @@ GrpcClient::CountTable(const std::string& table_name, Status& status) { if (!grpc_status.ok()) { std::cerr << "DescribeTable rpc failed!" << std::endl; - status = Status(StatusCode::RPCFailed, grpc_status.error_message()); + status = Status(StatusCode::RPCFailed, grpc_status.error_message()); return -1; } @@ -223,7 +226,7 @@ GrpcClient::ShowTables(milvus::grpc::TableNameList &table_name_list) { Status GrpcClient::Cmd(std::string &result, - const std::string& cmd) { + const std::string &cmd) { ClientContext context; ::milvus::grpc::StringReply response; ::milvus::grpc::Command command; @@ -321,4 +324,4 @@ GrpcClient::DropIndex(grpc::TableName &table_name) { return Status::OK(); } -} \ No newline at end of file +} // namespace milvus diff --git a/cpp/src/sdk/grpc/GrpcClient.h b/cpp/src/sdk/grpc/GrpcClient.h index 08714d10e0..8f81e83ae8 100644 --- a/cpp/src/sdk/grpc/GrpcClient.h +++ b/cpp/src/sdk/grpc/GrpcClient.h @@ -16,6 +16,11 @@ // under the License. #pragma once + +#include "MilvusApi.h" +#include "grpc/gen-milvus/milvus.grpc.pb.h" +//#include "grpc/gen-status/status.grpc.pb.h" + #include #include #include @@ -28,55 +33,48 @@ #include #include #include -#include "MilvusApi.h" - -#include "milvus.grpc.pb.h" -//#include "status.grpc.pb.h" - -#include namespace milvus { class GrpcClient { -public: - explicit - GrpcClient(std::shared_ptr<::grpc::Channel>& channel); + public: + explicit GrpcClient(std::shared_ptr<::grpc::Channel> &channel); virtual ~GrpcClient(); Status - CreateTable(const grpc::TableSchema& table_schema); + CreateTable(const grpc::TableSchema &table_schema); bool - HasTable(const grpc::TableName& table_name, Status& status); + HasTable(const grpc::TableName &table_name, Status &status); Status - DropTable(const grpc::TableName& table_name); + DropTable(const grpc::TableName &table_name); Status - CreateIndex(const grpc::IndexParam& index_param); + CreateIndex(const grpc::IndexParam &index_param); void - Insert(grpc::VectorIds& vector_ids, - const grpc::InsertParam& insert_param, - Status& status); + Insert(grpc::VectorIds &vector_ids, + const grpc::InsertParam &insert_param, + Status &status); Status - Search(::milvus::grpc::TopKQueryResultList& topk_query_result_list, + Search(::milvus::grpc::TopKQueryResultList &topk_query_result_list, const grpc::SearchParam &search_param); Status - DescribeTable(grpc::TableSchema& grpc_schema, - const std::string& table_name); + DescribeTable(grpc::TableSchema &grpc_schema, + const std::string &table_name); int64_t - CountTable(const std::string& table_name, Status& status); + CountTable(const std::string &table_name, Status &status); Status ShowTables(milvus::grpc::TableNameList &table_name_list); Status - Cmd(std::string &result, const std::string& cmd); + Cmd(std::string &result, const std::string &cmd); Status DeleteByRange(grpc::DeleteByRangeParam &delete_by_range_param); @@ -93,8 +91,8 @@ public: Status Disconnect(); -private: + private: std::unique_ptr stub_; }; -} +} // namespace milvus diff --git a/cpp/src/sdk/include/MilvusApi.h b/cpp/src/sdk/include/MilvusApi.h index 05de495fbc..e6025fd52e 100644 --- a/cpp/src/sdk/include/MilvusApi.h +++ b/cpp/src/sdk/include/MilvusApi.h @@ -28,7 +28,6 @@ */ namespace milvus { - /** * @brief Index Type */ @@ -108,7 +107,6 @@ struct IndexParam { */ class Connection { public: - /** * @brief CreateConnection * @@ -131,7 +129,7 @@ class Connection { */ static Status - Destroy(std::shared_ptr& connection_ptr); + Destroy(std::shared_ptr &connection_ptr); /** * @brief Connect @@ -180,7 +178,6 @@ class Connection { virtual Status Disconnect() = 0; - /** * @brief Create table method * @@ -193,7 +190,6 @@ class Connection { virtual Status CreateTable(const TableSchema ¶m) = 0; - /** * @brief Test table existence method * @@ -206,7 +202,6 @@ class Connection { virtual bool HasTable(const std::string &table_name) = 0; - /** * @brief Delete table method * @@ -248,8 +243,8 @@ class Connection { */ virtual Status Insert(const std::string &table_name, - const std::vector &record_array, - std::vector &id_array) = 0; + const std::vector &record_array, + std::vector &id_array) = 0; /** * @brief Search vector @@ -266,11 +261,11 @@ class Connection { */ virtual Status Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) = 0; + const std::vector &query_record_array, + const std::vector &query_range_array, + int64_t topk, + int64_t nprobe, + std::vector &topk_query_result_array) = 0; /** * @brief Show table description @@ -297,7 +292,7 @@ class Connection { */ virtual Status CountTable(const std::string &table_name, - int64_t &row_count) = 0; + int64_t &row_count) = 0; /** * @brief Show all tables in database @@ -395,4 +390,4 @@ class Connection { DropIndex(const std::string &table_name) const = 0; }; -} \ No newline at end of file +} // namespace milvus diff --git a/cpp/src/sdk/include/Status.h b/cpp/src/sdk/include/Status.h index cdb5f54ede..670d9662b4 100644 --- a/cpp/src/sdk/include/Status.h +++ b/cpp/src/sdk/include/Status.h @@ -29,6 +29,7 @@ namespace milvus { */ enum class StatusCode { OK = 0, + // system error section UnknownError = 1, NotSupported, @@ -44,7 +45,7 @@ enum class StatusCode { * @brief Status for SDK interface return */ class Status { -public: + public: Status(StatusCode code, const std::string &msg); Status(); ~Status(); @@ -60,28 +61,32 @@ public: operator=(Status &&s); static Status - OK() { return Status(); } + OK() { + return Status(); + } bool - ok() const { return state_ == nullptr || code() == StatusCode::OK; } + ok() const { + return state_ == nullptr || code() == StatusCode::OK; + } StatusCode code() const { - return (state_ == nullptr) ? StatusCode::OK : *(StatusCode*)(state_); + return (state_ == nullptr) ? StatusCode::OK : *(StatusCode *) (state_); } std::string message() const; -private: + private: inline void CopyFrom(const Status &s); inline void MoveFrom(Status &s); -private: + private: const char *state_ = nullptr; }; // Status -} //Milvus \ No newline at end of file +} // namespace milvus diff --git a/cpp/src/sdk/interface/ConnectionImpl.cpp b/cpp/src/sdk/interface/ConnectionImpl.cpp index 3730cfd191..5a01cd1a79 100644 --- a/cpp/src/sdk/interface/ConnectionImpl.cpp +++ b/cpp/src/sdk/interface/ConnectionImpl.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "ConnectionImpl.h" +#include "sdk/interface/ConnectionImpl.h" namespace milvus { @@ -25,7 +25,7 @@ Connection::Create() { } Status -Connection::Destroy(std::shared_ptr& connection_ptr) { +Connection::Destroy(std::shared_ptr &connection_ptr) { if (connection_ptr != nullptr) { return connection_ptr->Disconnect(); } @@ -84,19 +84,18 @@ ConnectionImpl::CreateIndex(const IndexParam &index_param) { Status ConnectionImpl::Insert(const std::string &table_name, - const std::vector &record_array, - std::vector &id_array) { + const std::vector &record_array, + std::vector &id_array) { return client_proxy_->Insert(table_name, record_array, id_array); } - Status ConnectionImpl::Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) { + const std::vector &query_record_array, + const std::vector &query_range_array, + int64_t topk, + int64_t nprobe, + std::vector &topk_query_result_array) { return client_proxy_->Search(table_name, query_record_array, query_range_array, topk, nprobe, topk_query_result_array); } @@ -133,7 +132,7 @@ ConnectionImpl::DumpTaskTables() const { Status ConnectionImpl::DeleteByRange(Range &range, - const std::string &table_name) { + const std::string &table_name) { return client_proxy_->DeleteByRange(range, table_name); } @@ -143,7 +142,7 @@ ConnectionImpl::PreloadTable(const std::string &table_name) const { } Status -ConnectionImpl::DescribeIndex(const std::string &table_name, IndexParam& index_param) const { +ConnectionImpl::DescribeIndex(const std::string &table_name, IndexParam &index_param) const { return client_proxy_->DescribeIndex(table_name, index_param); } @@ -152,4 +151,4 @@ ConnectionImpl::DropIndex(const std::string &table_name) const { return client_proxy_->DropIndex(table_name); } -} +} // namespace milvus diff --git a/cpp/src/sdk/interface/ConnectionImpl.h b/cpp/src/sdk/interface/ConnectionImpl.h index 6a402fcb0d..7d5e090882 100644 --- a/cpp/src/sdk/interface/ConnectionImpl.h +++ b/cpp/src/sdk/interface/ConnectionImpl.h @@ -18,88 +18,92 @@ #pragma once #include "MilvusApi.h" -#include "src/sdk/grpc/ClientProxy.h" +#include "sdk/grpc/ClientProxy.h" + +#include +#include +#include namespace milvus { class ConnectionImpl : public Connection { -public: + public: ConnectionImpl(); // Implementations of the Connection interface - virtual Status + Status Connect(const ConnectParam ¶m) override; - virtual Status + Status Connect(const std::string &uri) override; - virtual Status + Status Connected() const override; - virtual Status + Status Disconnect() override; - virtual Status + Status CreateTable(const TableSchema ¶m) override; - virtual - bool HasTable(const std::string &table_name) override; + bool + HasTable(const std::string &table_name) override; - virtual Status + Status DropTable(const std::string &table_name) override; - virtual Status + Status CreateIndex(const IndexParam &index_param) override; - virtual Status + Status Insert(const std::string &table_name, const std::vector &record_array, std::vector &id_array) override; - virtual Status + Status Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) override; + const std::vector &query_record_array, + const std::vector &query_range_array, + int64_t topk, + int64_t nprobe, + std::vector &topk_query_result_array) override; - virtual Status + Status DescribeTable(const std::string &table_name, TableSchema &table_schema) override; - virtual Status + Status CountTable(const std::string &table_name, int64_t &row_count) override; - virtual Status + Status ShowTables(std::vector &table_array) override; - virtual std::string + std::string ClientVersion() const override; - virtual std::string + std::string ServerVersion() const override; - virtual std::string + std::string ServerStatus() const override; - virtual std::string + std::string DumpTaskTables() const override; - virtual Status + Status DeleteByRange(Range &range, const std::string &table_name) override; - virtual Status + Status PreloadTable(const std::string &table_name) const override; - virtual Status - DescribeIndex(const std::string &table_name, IndexParam& index_param) const override; + Status + DescribeIndex(const std::string &table_name, IndexParam &index_param) const override; - virtual Status + Status DropIndex(const std::string &table_name) const override; -private: + private: std::shared_ptr client_proxy_; }; -} +} // namespace milvus diff --git a/cpp/src/sdk/interface/Status.cpp b/cpp/src/sdk/interface/Status.cpp index e6fd06c72e..a8780f2ddd 100644 --- a/cpp/src/sdk/interface/Status.cpp +++ b/cpp/src/sdk/interface/Status.cpp @@ -23,12 +23,12 @@ namespace milvus { constexpr int CODE_WIDTH = sizeof(StatusCode); -Status::Status(StatusCode code, const std::string& msg) { +Status::Status(StatusCode code, const std::string &msg) { //4 bytes store code //4 bytes store message length //the left bytes store message string - const uint32_t length = (uint32_t)msg.size(); - char* result = new char[length + sizeof(length) + CODE_WIDTH]; + const uint32_t length = (uint32_t) msg.size(); + char *result = new char[length + sizeof(length) + CODE_WIDTH]; memcpy(result, &code, CODE_WIDTH); memcpy(result + CODE_WIDTH, &length, sizeof(length)); memcpy(result + sizeof(length) + CODE_WIDTH, msg.data(), length); @@ -37,8 +37,7 @@ Status::Status(StatusCode code, const std::string& msg) { } Status::Status() - : state_(nullptr) { - + : state_(nullptr) { } Status::~Status() { @@ -46,22 +45,22 @@ Status::~Status() { } Status::Status(const Status &s) - : state_(nullptr) { + : state_(nullptr) { CopyFrom(s); } -Status& +Status & Status::operator=(const Status &s) { CopyFrom(s); return *this; } Status::Status(Status &&s) - : state_(nullptr) { + : state_(nullptr) { MoveFrom(s); } -Status& +Status & Status::operator=(Status &&s) { MoveFrom(s); return *this; @@ -71,7 +70,7 @@ void Status::CopyFrom(const Status &s) { delete state_; state_ = nullptr; - if(s.state_ == nullptr) { + if (s.state_ == nullptr) { return; } @@ -79,7 +78,7 @@ Status::CopyFrom(const Status &s) { memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length)); int buff_len = length + sizeof(length) + CODE_WIDTH; state_ = new char[buff_len]; - memcpy((void*)state_, (void*)s.state_, buff_len); + memcpy((void *) state_, (void *) s.state_, buff_len); } void @@ -98,12 +97,13 @@ Status::message() const { std::string msg; uint32_t length = 0; memcpy(&length, state_ + CODE_WIDTH, sizeof(length)); - if(length > 0) { + if (length > 0) { msg.append(state_ + sizeof(length) + CODE_WIDTH, length); } return msg; } -} +} // namespace milvus + From 85d3c9d052932ecadea696015893c4d37bc71cb5 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Fri, 27 Sep 2019 11:25:13 +0800 Subject: [PATCH 128/217] MS-574 add API ResetDefaultConfig() Former-commit-id: 475b048bb6386ffd3bb09a229694e954f8fc67e7 --- cpp/src/server/Config.cpp | 79 +++++++++++++++++++++++++++++ cpp/src/server/Config.h | 1 + cpp/unittest/server/config_test.cpp | 3 ++ 3 files changed, 83 insertions(+) diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 34d1fab7b3..d0b2b84539 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -176,6 +176,85 @@ Config::ValidateConfig() { return Status::OK(); } +Status +Config::ResetDefaultConfig() { + Status s; + + /* server config */ + s = SetServerConfigAddress(CONFIG_SERVER_ADDRESS_DEFAULT); + if (!s.ok()) return s; + + s = SetServerConfigPort(CONFIG_SERVER_PORT_DEFAULT); + if (!s.ok()) return s; + + s = SetServerConfigDeployMode(CONFIG_SERVER_DEPLOY_MODE_DEFAULT); + if (!s.ok()) return s; + + s = SetServerConfigTimeZone(CONFIG_SERVER_TIME_ZONE_DEFAULT); + if (!s.ok()) return s; + + /* db config */ + s = SetDBConfigPrimaryPath(CONFIG_DB_PRIMARY_PATH_DEFAULT); + if (!s.ok()) return s; + + s = SetDBConfigSecondaryPath(CONFIG_DB_SECONDARY_PATH_DEFAULT); + if (!s.ok()) return s; + + s = SetDBConfigBackendUrl(CONFIG_DB_BACKEND_URL_DEFAULT); + if (!s.ok()) return s; + + s = SetDBConfigArchiveDiskThreshold(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); + if (!s.ok()) return s; + + s = SetDBConfigArchiveDaysThreshold(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); + if (!s.ok()) return s; + + s = SetDBConfigInsertBufferSize(CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); + if (!s.ok()) return s; + + s = SetDBConfigBuildIndexGPU(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); + if (!s.ok()) return s; + + /* metric config */ + s = SetMetricConfigEnableMonitor(CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); + if (!s.ok()) return s; + + s = SetMetricConfigCollector(CONFIG_METRIC_COLLECTOR_DEFAULT); + if (!s.ok()) return s; + + s = SetMetricConfigPrometheusPort(CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); + if (!s.ok()) return s; + + /* cache config */ + s = SetCacheConfigCpuMemCapacity(CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); + if (!s.ok()) return s; + + s = SetCacheConfigCpuMemThreshold(CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT); + if (!s.ok()) return s; + + s = SetCacheConfigGpuMemCapacity(CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT); + if (!s.ok()) return s; + + s = SetCacheConfigGpuMemThreshold(CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT); + if (!s.ok()) return s; + + s = SetCacheConfigCacheInsertData(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); + if (!s.ok()) return s; + + /* engine config */ + s = SetEngineConfigBlasThreshold(CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); + if (!s.ok()) return s; + + s = SetEngineConfigOmpThreadNum(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); + if (!s.ok()) return s; + + /* resource config */ + s = SetResourceConfigMode(CONFIG_RESOURCE_MODE_DEFAULT); + if (!s.ok()) return s; + + return Status::OK(); +} + void Config::PrintConfigSection(const std::string &config_node_name) { std::cout << std::endl; diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index b1de101bd1..af23dd67b6 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -99,6 +99,7 @@ class Config { static Config &GetInstance(); Status LoadConfigFile(const std::string &filename); Status ValidateConfig(); + Status ResetDefaultConfig(); void PrintAll(); private: diff --git a/cpp/unittest/server/config_test.cpp b/cpp/unittest/server/config_test.cpp index 61e9ce44ad..b27d1e60c8 100644 --- a/cpp/unittest/server/config_test.cpp +++ b/cpp/unittest/server/config_test.cpp @@ -110,4 +110,7 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) { ASSERT_TRUE(s.ok()); config.PrintAll(); + + s = config.ResetDefaultConfig(); + ASSERT_TRUE(s.ok()); } \ No newline at end of file From 965ca2c9c1666f5b90118676fdccae2e4b4e6b21 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 27 Sep 2019 11:50:01 +0800 Subject: [PATCH 129/217] format scheduler code Former-commit-id: c2ff1ac702af62b4968a192d12b258bc90dd0b50 --- cpp/src/scheduler/Algorithm.cpp | 16 +++--- cpp/src/scheduler/Algorithm.h | 8 +-- cpp/src/scheduler/Definition.h | 7 ++- cpp/src/scheduler/JobMgr.cpp | 18 ++++--- cpp/src/scheduler/JobMgr.h | 18 +++---- cpp/src/scheduler/ResourceFactory.cpp | 9 ++-- cpp/src/scheduler/ResourceFactory.h | 11 ++-- cpp/src/scheduler/ResourceMgr.cpp | 14 +++--- cpp/src/scheduler/ResourceMgr.h | 24 ++++----- cpp/src/scheduler/SchedInst.cpp | 13 +++-- cpp/src/scheduler/SchedInst.h | 19 ++++--- cpp/src/scheduler/Scheduler.cpp | 17 ++++--- cpp/src/scheduler/Scheduler.h | 21 ++++---- cpp/src/scheduler/TaskCreator.cpp | 13 ++--- cpp/src/scheduler/TaskCreator.h | 11 ++-- cpp/src/scheduler/TaskTable.cpp | 15 +++--- cpp/src/scheduler/TaskTable.h | 36 +++++++------ cpp/src/scheduler/Utils.cpp | 9 ++-- cpp/src/scheduler/Utils.h | 7 ++- cpp/src/scheduler/action/Action.h | 15 +++--- .../scheduler/action/PushTaskToNeighbour.cpp | 22 ++++---- cpp/src/scheduler/event/Event.h | 15 +++--- cpp/src/scheduler/event/EventDump.cpp | 22 ++++---- cpp/src/scheduler/event/FinishTaskEvent.h | 18 ++++--- cpp/src/scheduler/event/LoadCompletedEvent.h | 20 +++++--- cpp/src/scheduler/event/StartUpEvent.h | 19 ++++--- .../scheduler/event/TaskTableUpdatedEvent.h | 18 ++++--- cpp/src/scheduler/job/DeleteJob.cpp | 23 +++++---- cpp/src/scheduler/job/DeleteJob.h | 16 +++--- cpp/src/scheduler/job/Job.h | 17 +++---- cpp/src/scheduler/job/SearchJob.cpp | 23 ++++----- cpp/src/scheduler/job/SearchJob.h | 19 ++++--- cpp/src/scheduler/resource/Connection.h | 18 +++---- cpp/src/scheduler/resource/CpuResource.cpp | 21 +++++--- cpp/src/scheduler/resource/CpuResource.h | 11 ++-- cpp/src/scheduler/resource/DiskResource.cpp | 22 ++++---- cpp/src/scheduler/resource/DiskResource.h | 12 ++--- cpp/src/scheduler/resource/GpuResource.cpp | 21 ++++---- cpp/src/scheduler/resource/GpuResource.h | 13 ++--- cpp/src/scheduler/resource/Node.cpp | 19 ++++--- cpp/src/scheduler/resource/Node.h | 21 ++++---- cpp/src/scheduler/resource/Resource.cpp | 35 +++++++------ cpp/src/scheduler/resource/Resource.h | 11 ++-- cpp/src/scheduler/resource/TestResource.cpp | 19 ++++--- cpp/src/scheduler/resource/TestResource.h | 13 ++--- cpp/src/scheduler/task/DeleteTask.cpp | 13 +++-- cpp/src/scheduler/task/DeleteTask.h | 14 +++--- cpp/src/scheduler/task/Path.h | 20 +++++--- cpp/src/scheduler/task/SearchTask.cpp | 50 ++++++++++--------- cpp/src/scheduler/task/SearchTask.h | 18 +++---- cpp/src/scheduler/task/Task.h | 24 ++++----- cpp/src/scheduler/task/TestTask.cpp | 20 ++++---- cpp/src/scheduler/task/TestTask.h | 17 +++---- cpp/src/scheduler/tasklabel/BroadcastLabel.h | 14 +++--- cpp/src/scheduler/tasklabel/DefaultLabel.h | 14 +++--- cpp/src/scheduler/tasklabel/SpecResLabel.h | 17 +++---- cpp/src/scheduler/tasklabel/TaskLabel.h | 17 +++---- 57 files changed, 520 insertions(+), 487 deletions(-) diff --git a/cpp/src/scheduler/Algorithm.cpp b/cpp/src/scheduler/Algorithm.cpp index b388c17bdd..10a585304a 100644 --- a/cpp/src/scheduler/Algorithm.cpp +++ b/cpp/src/scheduler/Algorithm.cpp @@ -16,20 +16,23 @@ // under the License. -#include "Algorithm.h" +#include "scheduler/Algorithm.h" + +#include +#include +#include namespace zilliz { namespace milvus { namespace scheduler { -constexpr uint64_t MAXINT = std::numeric_limits::max(); +constexpr uint64_t MAXINT = std::numeric_limits::max(); uint64_t ShortestPath(const ResourcePtr &src, const ResourcePtr &dest, const ResourceMgrPtr &res_mgr, std::vector &path) { - std::vector> paths; uint64_t num_of_resources = res_mgr->GetAllResources().size(); @@ -53,7 +56,6 @@ ShortestPath(const ResourcePtr &src, std::vector vis(num_of_resources, false); std::vector dis(num_of_resources, MAXINT); for (auto &res : res_mgr->GetAllResources()) { - auto cur_node = std::static_pointer_cast(res); auto cur_neighbours = cur_node->GetNeighbours(); @@ -105,6 +107,6 @@ ShortestPath(const ResourcePtr &src, return dis[name_id_map.at(dest->name())]; } -} -} -} \ No newline at end of file +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Algorithm.h b/cpp/src/scheduler/Algorithm.h index be77cf351f..d7e0233ba0 100644 --- a/cpp/src/scheduler/Algorithm.h +++ b/cpp/src/scheduler/Algorithm.h @@ -30,8 +30,8 @@ uint64_t ShortestPath(const ResourcePtr &src, const ResourcePtr &dest, const ResourceMgrPtr &res_mgr, - std::vector& path); + std::vector &path); -} -} -} \ No newline at end of file +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Definition.h b/cpp/src/scheduler/Definition.h index 9ac647de88..ce41aca48d 100644 --- a/cpp/src/scheduler/Definition.h +++ b/cpp/src/scheduler/Definition.h @@ -30,7 +30,6 @@ #include "db/engine/EngineFactory.h" #include "db/engine/ExecutionEngine.h" - namespace zilliz { namespace milvus { namespace scheduler { @@ -43,6 +42,6 @@ using EngineFactory = engine::EngineFactory; using EngineType = engine::EngineType; using MetricType = engine::MetricType; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/JobMgr.cpp b/cpp/src/scheduler/JobMgr.cpp index 31db4b344e..0406e98c49 100644 --- a/cpp/src/scheduler/JobMgr.cpp +++ b/cpp/src/scheduler/JobMgr.cpp @@ -15,19 +15,19 @@ // specific language governing permissions and limitations // under the License. -#include "JobMgr.h" +#include "scheduler/JobMgr.h" #include "task/Task.h" #include "TaskCreator.h" +#include namespace zilliz { namespace milvus { namespace scheduler { -using namespace engine; - JobMgr::JobMgr(ResourceMgrPtr res_mgr) - : res_mgr_(std::move(res_mgr)) {} + : res_mgr_(std::move(res_mgr)) { +} void JobMgr::Start() { @@ -59,7 +59,9 @@ void JobMgr::worker_function() { while (running_) { std::unique_lock lock(mutex_); - cv_.wait(lock, [this] { return !queue_.empty(); }); + cv_.wait(lock, [this] { + return !queue_.empty(); + }); auto job = queue_.front(); queue_.pop(); lock.unlock(); @@ -84,6 +86,6 @@ JobMgr::build_task(const JobPtr &job) { return TaskCreator::Create(job); } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/JobMgr.h b/cpp/src/scheduler/JobMgr.h index 8d22b5d4f5..49ba9154e3 100644 --- a/cpp/src/scheduler/JobMgr.h +++ b/cpp/src/scheduler/JobMgr.h @@ -31,15 +31,13 @@ #include "task/Task.h" #include "ResourceMgr.h" - namespace zilliz { namespace milvus { namespace scheduler { class JobMgr { -public: - explicit - JobMgr(ResourceMgrPtr res_mgr); + public: + explicit JobMgr(ResourceMgrPtr res_mgr); void Start(); @@ -47,18 +45,18 @@ public: void Stop(); -public: + public: void Put(const JobPtr &job); -private: + private: void worker_function(); std::vector build_task(const JobPtr &job); -private: + private: bool running_ = false; std::queue queue_; @@ -72,6 +70,6 @@ private: using JobMgrPtr = std::shared_ptr; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/ResourceFactory.cpp b/cpp/src/scheduler/ResourceFactory.cpp index 066b47aad7..de9b5bc717 100644 --- a/cpp/src/scheduler/ResourceFactory.cpp +++ b/cpp/src/scheduler/ResourceFactory.cpp @@ -16,8 +16,7 @@ // under the License. -#include "ResourceFactory.h" - +#include "scheduler/ResourceFactory.h" namespace zilliz { namespace milvus { @@ -40,6 +39,6 @@ ResourceFactory::Create(const std::string &name, } } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/ResourceFactory.h b/cpp/src/scheduler/ResourceFactory.h index 39fb79154b..f7a47ef1e5 100644 --- a/cpp/src/scheduler/ResourceFactory.h +++ b/cpp/src/scheduler/ResourceFactory.h @@ -25,13 +25,12 @@ #include "resource/GpuResource.h" #include "resource/DiskResource.h" - namespace zilliz { namespace milvus { namespace scheduler { class ResourceFactory { -public: + public: static std::shared_ptr Create(const std::string &name, const std::string &type, @@ -40,8 +39,6 @@ public: bool enable_executor = true); }; - -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/ResourceMgr.cpp b/cpp/src/scheduler/ResourceMgr.cpp index 0f9fab8e8d..6067b2eb01 100644 --- a/cpp/src/scheduler/ResourceMgr.cpp +++ b/cpp/src/scheduler/ResourceMgr.cpp @@ -16,15 +16,13 @@ // specific language governing permissions and limitations // under the License. -#include "ResourceMgr.h" +#include "scheduler/ResourceMgr.h" #include "utils/Log.h" - namespace zilliz { namespace milvus { namespace scheduler { - void ResourceMgr::Start() { std::lock_guard lck(resources_mutex_); @@ -186,7 +184,9 @@ void ResourceMgr::event_process() { while (running_) { std::unique_lock lock(event_mutex_); - event_cv_.wait(lock, [this] { return !queue_.empty(); }); + event_cv_.wait(lock, [this] { + return !queue_.empty(); + }); auto event = queue_.front(); queue_.pop(); @@ -201,6 +201,6 @@ ResourceMgr::event_process() { } } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/ResourceMgr.h b/cpp/src/scheduler/ResourceMgr.h index 8bd045ab7d..d03408a7df 100644 --- a/cpp/src/scheduler/ResourceMgr.h +++ b/cpp/src/scheduler/ResourceMgr.h @@ -22,21 +22,21 @@ #include #include #include +#include #include #include "resource/Resource.h" #include "utils/Log.h" - namespace zilliz { namespace milvus { namespace scheduler { class ResourceMgr { -public: + public: ResourceMgr() = default; -public: + public: /******** Management Interface ********/ void Start(); @@ -58,7 +58,7 @@ public: subscriber_ = std::move(subscriber); } -public: + public: /******** Management Interface ********/ inline std::vector & GetDiskResources() { @@ -89,10 +89,10 @@ public: uint64_t GetNumGpuResource() const; -public: + public: // TODO: add stats interface(low) -public: + public: /******** Utility Functions ********/ std::string Dump(); @@ -100,14 +100,14 @@ public: std::string DumpTaskTables(); -private: + private: void post_event(const EventPtr &event); void event_process(); -private: + private: bool running_ = false; std::vector disk_resources_; @@ -120,13 +120,11 @@ private: std::condition_variable event_cv_; std::thread worker_thread_; - }; using ResourceMgrPtr = std::shared_ptr; using ResourceMgrWPtr = std::weak_ptr; -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index 5d65d35993..71b40de9ee 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -16,12 +16,16 @@ // under the License. -#include "SchedInst.h" +#include "scheduler/SchedInst.h" #include "server/Config.h" #include "ResourceFactory.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" #include "Utils.h" +#include +#include +#include +#include namespace zilliz { namespace milvus { @@ -165,6 +169,7 @@ StopSchedulerService() { SchedInst::GetInstance()->Stop(); ResMgrInst::GetInstance()->Stop(); } -} -} -} + +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/SchedInst.h b/cpp/src/scheduler/SchedInst.h index d23c200b05..4cca6ec5a9 100644 --- a/cpp/src/scheduler/SchedInst.h +++ b/cpp/src/scheduler/SchedInst.h @@ -24,13 +24,12 @@ #include #include - namespace zilliz { namespace milvus { namespace scheduler { class ResMgrInst { -public: + public: static ResourceMgrPtr GetInstance() { if (instance == nullptr) { @@ -42,13 +41,13 @@ public: return instance; } -private: + private: static ResourceMgrPtr instance; static std::mutex mutex_; }; class SchedInst { -public: + public: static SchedulerPtr GetInstance() { if (instance == nullptr) { @@ -60,13 +59,13 @@ public: return instance; } -private: + private: static SchedulerPtr instance; static std::mutex mutex_; }; class JobMgrInst { -public: + public: static scheduler::JobMgrPtr GetInstance() { if (instance == nullptr) { @@ -78,7 +77,7 @@ public: return instance; } -private: + private: static scheduler::JobMgrPtr instance; static std::mutex mutex_; }; @@ -89,6 +88,6 @@ StartSchedulerService(); void StopSchedulerService(); -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Scheduler.cpp b/cpp/src/scheduler/Scheduler.cpp index abaccd4fcb..24f7bfe73b 100644 --- a/cpp/src/scheduler/Scheduler.cpp +++ b/cpp/src/scheduler/Scheduler.cpp @@ -15,13 +15,13 @@ // specific language governing permissions and limitations // under the License. - -#include "src/cache/GpuCacheMgr.h" +#include "scheduler/Scheduler.h" +#include "cache/GpuCacheMgr.h" #include "event/LoadCompletedEvent.h" -#include "Scheduler.h" #include "action/Action.h" #include "Algorithm.h" +#include namespace zilliz { namespace milvus { @@ -43,7 +43,6 @@ Scheduler::Scheduler(ResourceMgrWPtr res_mgr) std::bind(&Scheduler::OnFinishTask, this, std::placeholders::_1))); } - void Scheduler::Start() { running_ = true; @@ -79,7 +78,9 @@ void Scheduler::worker_function() { while (running_) { std::unique_lock lock(event_mutex_); - event_cv_.wait(lock, [this] { return !event_queue_.empty(); }); + event_cv_.wait(lock, [this] { + return !event_queue_.empty(); + }); auto event = event_queue_.front(); event_queue_.pop(); if (event == nullptr) { @@ -142,6 +143,6 @@ Scheduler::OnTaskTableUpdated(const EventPtr &event) { } } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Scheduler.h b/cpp/src/scheduler/Scheduler.h index b591fdd256..073f7eeb0d 100644 --- a/cpp/src/scheduler/Scheduler.h +++ b/cpp/src/scheduler/Scheduler.h @@ -22,22 +22,20 @@ #include #include #include +#include #include "resource/Resource.h" #include "ResourceMgr.h" #include "utils/Log.h" - namespace zilliz { namespace milvus { namespace scheduler { - // TODO: refactor, not friendly to unittest, logical in framework code class Scheduler { -public: - explicit - Scheduler(ResourceMgrWPtr res_mgr); + public: + explicit Scheduler(ResourceMgrWPtr res_mgr); Scheduler(const Scheduler &) = delete; Scheduler(Scheduler &&) = delete; @@ -66,7 +64,7 @@ public: std::string Dump(); -private: + private: /******** Events ********/ /* @@ -106,7 +104,7 @@ private: void OnTaskTableUpdated(const EventPtr &event); -private: + private: /* * Dispatch event to event handler; */ @@ -119,7 +117,7 @@ private: void worker_function(); -private: + private: bool running_; std::unordered_map> event_register_; @@ -133,7 +131,6 @@ private: using SchedulerPtr = std::shared_ptr; -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/TaskCreator.cpp b/cpp/src/scheduler/TaskCreator.cpp index 0d3b2a8a96..8b0378c646 100644 --- a/cpp/src/scheduler/TaskCreator.cpp +++ b/cpp/src/scheduler/TaskCreator.cpp @@ -15,11 +15,10 @@ // specific language governing permissions and limitations // under the License. -#include -#include "TaskCreator.h" +#include "scheduler/TaskCreator.h" +#include "scheduler/tasklabel/BroadcastLabel.h" #include "tasklabel/DefaultLabel.h" - namespace zilliz { namespace milvus { namespace scheduler { @@ -64,8 +63,6 @@ TaskCreator::Create(const DeleteJobPtr &job) { return tasks; } - -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/TaskCreator.h b/cpp/src/scheduler/TaskCreator.h index d4e87bd0a2..81cb25010f 100644 --- a/cpp/src/scheduler/TaskCreator.h +++ b/cpp/src/scheduler/TaskCreator.h @@ -34,17 +34,16 @@ #include "task/SearchTask.h" #include "task/DeleteTask.h" - namespace zilliz { namespace milvus { namespace scheduler { class TaskCreator { -public: + public: static std::vector Create(const JobPtr &job); -public: + public: static std::vector Create(const SearchJobPtr &job); @@ -52,6 +51,6 @@ public: Create(const DeleteJobPtr &job); }; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/TaskTable.cpp b/cpp/src/scheduler/TaskTable.cpp index badcb46c2d..a7343ee509 100644 --- a/cpp/src/scheduler/TaskTable.cpp +++ b/cpp/src/scheduler/TaskTable.cpp @@ -16,7 +16,7 @@ // under the License. -#include "TaskTable.h" +#include "scheduler/TaskTable.h" #include "event/TaskTableUpdatedEvent.h" #include "Utils.h" @@ -24,7 +24,6 @@ #include #include - namespace zilliz { namespace milvus { namespace scheduler { @@ -75,6 +74,7 @@ TaskTableItem::Load() { } return false; } + bool TaskTableItem::Loaded() { std::unique_lock lock(mutex); @@ -86,6 +86,7 @@ TaskTableItem::Loaded() { } return false; } + bool TaskTableItem::Execute() { std::unique_lock lock(mutex); @@ -97,6 +98,7 @@ TaskTableItem::Execute() { } return false; } + bool TaskTableItem::Executed() { std::unique_lock lock(mutex); @@ -109,6 +111,7 @@ TaskTableItem::Executed() { } return false; } + bool TaskTableItem::Move() { std::unique_lock lock(mutex); @@ -120,6 +123,7 @@ TaskTableItem::Move() { } return false; } + bool TaskTableItem::Moved() { std::unique_lock lock(mutex); @@ -206,7 +210,6 @@ TaskTable::Put(std::vector &tasks) { } } - TaskTableItemPtr TaskTable::Get(uint64_t index) { return table_[index]; @@ -232,6 +235,6 @@ TaskTable::Dump() { return ss.str(); } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/TaskTable.h b/cpp/src/scheduler/TaskTable.h index c9557c8f9e..819197c1ad 100644 --- a/cpp/src/scheduler/TaskTable.h +++ b/cpp/src/scheduler/TaskTable.h @@ -20,11 +20,13 @@ #include #include #include +#include +#include +#include #include "task/SearchTask.h" #include "event/Event.h" - namespace zilliz { namespace milvus { namespace scheduler { @@ -52,7 +54,8 @@ struct TaskTimestamp { }; struct TaskTableItem { - TaskTableItem() : id(0), task(nullptr), state(TaskTableItemState::INVALID), mutex() {} + TaskTableItem() : id(0), task(nullptr), state(TaskTableItemState::INVALID), mutex() { + } TaskTableItem(const TaskTableItem &src) = delete; TaskTableItem(TaskTableItem &&) = delete; @@ -91,7 +94,7 @@ struct TaskTableItem { using TaskTableItemPtr = std::shared_ptr; class TaskTable { -public: + public: TaskTable() = default; TaskTable(const TaskTable &) = delete; @@ -145,24 +148,28 @@ public: return table_.size(); } -public: + public: TaskTableItemPtr & operator[](uint64_t index) { return table_[index]; } - std::deque::iterator begin() { return table_.begin(); } - std::deque::iterator end() { return table_.end(); } + std::deque::iterator begin() { + return table_.begin(); + } -public: + std::deque::iterator end() { + return table_.end(); + } + + public: std::vector PickToLoad(uint64_t limit); std::vector PickToExecute(uint64_t limit); -public: - + public: /******** Action ********/ // TODO: bool to Status @@ -227,14 +234,14 @@ public: return table_[index]->Moved(); } -public: + public: /* * Dump; */ std::string Dump(); -private: + private: std::uint64_t id_ = 0; mutable std::mutex id_mutex_; std::deque table_; @@ -246,7 +253,6 @@ private: uint64_t last_finish_ = -1; }; - -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Utils.cpp b/cpp/src/scheduler/Utils.cpp index f207845c20..bb19950ffe 100644 --- a/cpp/src/scheduler/Utils.cpp +++ b/cpp/src/scheduler/Utils.cpp @@ -16,12 +16,11 @@ // under the License. -#include "Utils.h" +#include "scheduler/Utils.h" #include #include - namespace zilliz { namespace milvus { namespace scheduler { @@ -41,6 +40,6 @@ get_num_gpu() { return n_devices; } -} -} -} \ No newline at end of file +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Utils.h b/cpp/src/scheduler/Utils.h index f1056f104a..c69028f0fa 100644 --- a/cpp/src/scheduler/Utils.h +++ b/cpp/src/scheduler/Utils.h @@ -18,7 +18,6 @@ #include - namespace zilliz { namespace milvus { namespace scheduler { @@ -29,6 +28,6 @@ get_current_timestamp(); uint64_t get_num_gpu(); -} -} -} \ No newline at end of file +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/action/Action.h b/cpp/src/scheduler/action/Action.h index 9f2fa9f6c5..a5f67aa98d 100644 --- a/cpp/src/scheduler/action/Action.h +++ b/cpp/src/scheduler/action/Action.h @@ -17,16 +17,17 @@ #pragma once -#include "../resource/Resource.h" -#include "../ResourceMgr.h" +#include "scheduler/resource/Resource.h" +#include "scheduler/ResourceMgr.h" +#include namespace zilliz { namespace milvus { namespace scheduler { class Action { -public: + public: static void PushTaskToNeighbourRandomly(const TaskPtr &task, const ResourcePtr &self); @@ -43,10 +44,8 @@ public: SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr resource, std::shared_ptr event); - }; - -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp index d53903ba2d..909112eb62 100644 --- a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp @@ -22,7 +22,6 @@ #include "src/cache/GpuCacheMgr.h" #include "Action.h" - namespace zilliz { namespace milvus { namespace scheduler { @@ -57,13 +56,12 @@ get_neighbours_with_connetion(const ResourcePtr &self) { return neighbours; } - void Action::PushTaskToNeighbourRandomly(const TaskPtr &task, const ResourcePtr &self) { auto neighbours = get_neighbours_with_connetion(self); if (not neighbours.empty()) { - std::vector speeds; + std::vector speeds; uint64_t total_speed = 0; for (auto &neighbour : neighbours) { uint64_t speed = neighbour.second.speed(); @@ -87,7 +85,6 @@ Action::PushTaskToNeighbourRandomly(const TaskPtr &task, } else { //TODO: process } - } void @@ -99,14 +96,14 @@ Action::PushTaskToAllNeighbour(const TaskPtr &task, const ResourcePtr &self) { } void -Action::PushTaskToResource(const TaskPtr& task, const ResourcePtr& dest) { +Action::PushTaskToResource(const TaskPtr &task, const ResourcePtr &dest) { dest->task_table().Put(task); } void Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr, - ResourcePtr resource, - std::shared_ptr event) { + ResourcePtr resource, + std::shared_ptr event) { if (not resource->HasExecutor() && event->task_table_item_->Move()) { auto task = event->task_table_item_->task; auto search_task = std::static_pointer_cast(task); @@ -135,8 +132,8 @@ Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr, void Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, - ResourcePtr resource, - std::shared_ptr event) { + ResourcePtr resource, + std::shared_ptr event) { auto task = event->task_table_item_->task; if (resource->type() == ResourceType::DISK) { // step 1: calculate shortest path per resource, from disk to compute resource @@ -181,7 +178,6 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, } } -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/Event.h b/cpp/src/scheduler/event/Event.h index b00bd7f902..860c60c5b7 100644 --- a/cpp/src/scheduler/event/Event.h +++ b/cpp/src/scheduler/event/Event.h @@ -18,6 +18,8 @@ #pragma once #include +#include +#include namespace zilliz { namespace milvus { @@ -33,11 +35,12 @@ enum class EventType { class Resource; class Event { -public: + public: explicit Event(EventType type, std::weak_ptr resource) : type_(type), - resource_(std::move(resource)) {} + resource_(std::move(resource)) { + } inline EventType Type() const { @@ -49,13 +52,13 @@ public: friend std::ostream &operator<<(std::ostream &out, const Event &event); -public: + public: EventType type_; std::weak_ptr resource_; }; using EventPtr = std::shared_ptr; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/EventDump.cpp b/cpp/src/scheduler/event/EventDump.cpp index 3d06dfd5f5..a9ed751d88 100644 --- a/cpp/src/scheduler/event/EventDump.cpp +++ b/cpp/src/scheduler/event/EventDump.cpp @@ -22,36 +22,40 @@ #include "FinishTaskEvent.h" #include "TaskTableUpdatedEvent.h" - namespace zilliz { namespace milvus { namespace scheduler { -std::ostream &operator<<(std::ostream &out, const Event &event) { +std::ostream & +operator<<(std::ostream &out, const Event &event) { out << event.Dump(); return out; } -std::ostream &operator<<(std::ostream &out, const StartUpEvent &event) { +std::ostream & +operator<<(std::ostream &out, const StartUpEvent &event) { out << event.Dump(); return out; } -std::ostream &operator<<(std::ostream &out, const LoadCompletedEvent &event) { +std::ostream & +operator<<(std::ostream &out, const LoadCompletedEvent &event) { out << event.Dump(); return out; } -std::ostream &operator<<(std::ostream &out, const FinishTaskEvent &event) { +std::ostream & +operator<<(std::ostream &out, const FinishTaskEvent &event) { out << event.Dump(); return out; } -std::ostream &operator<<(std::ostream &out, const TaskTableUpdatedEvent &event) { +std::ostream & +operator<<(std::ostream &out, const TaskTableUpdatedEvent &event) { out << event.Dump(); return out; } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/FinishTaskEvent.h b/cpp/src/scheduler/event/FinishTaskEvent.h index 3ffa855f30..f49acb16ad 100644 --- a/cpp/src/scheduler/event/FinishTaskEvent.h +++ b/cpp/src/scheduler/event/FinishTaskEvent.h @@ -17,18 +17,22 @@ #pragma once -#include "Event.h" +#include "scheduler/event/Event.h" +#include +#include +#include namespace zilliz { namespace milvus { namespace scheduler { class FinishTaskEvent : public Event { -public: + public: FinishTaskEvent(std::weak_ptr resource, TaskTableItemPtr task_table_item) : Event(EventType::FINISH_TASK, std::move(resource)), - task_table_item_(std::move(task_table_item)) {} + task_table_item_(std::move(task_table_item)) { + } inline std::string Dump() const override { @@ -37,10 +41,10 @@ public: friend std::ostream &operator<<(std::ostream &out, const FinishTaskEvent &event); -public: + public: TaskTableItemPtr task_table_item_; }; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/LoadCompletedEvent.h b/cpp/src/scheduler/event/LoadCompletedEvent.h index 286f26723f..8d727f7435 100644 --- a/cpp/src/scheduler/event/LoadCompletedEvent.h +++ b/cpp/src/scheduler/event/LoadCompletedEvent.h @@ -17,19 +17,23 @@ #pragma once -#include "Event.h" -#include "../TaskTable.h" +#include "scheduler/event/Event.h" +#include "scheduler/TaskTable.h" +#include +#include +#include namespace zilliz { namespace milvus { namespace scheduler { class LoadCompletedEvent : public Event { -public: + public: LoadCompletedEvent(std::weak_ptr resource, TaskTableItemPtr task_table_item) : Event(EventType::LOAD_COMPLETED, std::move(resource)), - task_table_item_(std::move(task_table_item)) {} + task_table_item_(std::move(task_table_item)) { + } inline std::string Dump() const override { @@ -38,10 +42,10 @@ public: friend std::ostream &operator<<(std::ostream &out, const LoadCompletedEvent &event); -public: + public: TaskTableItemPtr task_table_item_; }; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/StartUpEvent.h b/cpp/src/scheduler/event/StartUpEvent.h index 089eb1377b..8e4ad120de 100644 --- a/cpp/src/scheduler/event/StartUpEvent.h +++ b/cpp/src/scheduler/event/StartUpEvent.h @@ -17,18 +17,21 @@ #pragma once -#include "Event.h" +#include "scheduler/event/Event.h" +#include +#include +#include namespace zilliz { namespace milvus { namespace scheduler { class StartUpEvent : public Event { -public: - explicit - StartUpEvent(std::weak_ptr resource) - : Event(EventType::START_UP, std::move(resource)) {} + public: + explicit StartUpEvent(std::weak_ptr resource) + : Event(EventType::START_UP, std::move(resource)) { + } inline std::string Dump() const override { @@ -38,6 +41,6 @@ public: friend std::ostream &operator<<(std::ostream &out, const StartUpEvent &event); }; -} -} -} \ No newline at end of file +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/TaskTableUpdatedEvent.h b/cpp/src/scheduler/event/TaskTableUpdatedEvent.h index 70ee28fdab..ec579b31be 100644 --- a/cpp/src/scheduler/event/TaskTableUpdatedEvent.h +++ b/cpp/src/scheduler/event/TaskTableUpdatedEvent.h @@ -19,16 +19,19 @@ #include "Event.h" +#include +#include +#include namespace zilliz { namespace milvus { namespace scheduler { class TaskTableUpdatedEvent : public Event { -public: - explicit - TaskTableUpdatedEvent(std::weak_ptr resource) - : Event(EventType::TASK_TABLE_UPDATED, std::move(resource)) {} + public: + explicit TaskTableUpdatedEvent(std::weak_ptr resource) + : Event(EventType::TASK_TABLE_UPDATED, std::move(resource)) { + } inline std::string Dump() const override { @@ -38,7 +41,6 @@ public: friend std::ostream &operator<<(std::ostream &out, const TaskTableUpdatedEvent &event); }; - -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/DeleteJob.cpp b/cpp/src/scheduler/job/DeleteJob.cpp index 9066afafaa..9d917751c6 100644 --- a/cpp/src/scheduler/job/DeleteJob.cpp +++ b/cpp/src/scheduler/job/DeleteJob.cpp @@ -15,8 +15,9 @@ // specific language governing permissions and limitations // under the License. -#include "DeleteJob.h" +#include "scheduler/job/DeleteJob.h" +#include namespace zilliz { namespace milvus { @@ -29,15 +30,20 @@ DeleteJob::DeleteJob(JobId id, : Job(id, JobType::DELETE), table_id_(std::move(table_id)), meta_ptr_(std::move(meta_ptr)), - num_resource_(num_resource) {} + num_resource_(num_resource) { +} -void DeleteJob::WaitAndDelete() { +void +DeleteJob::WaitAndDelete() { std::unique_lock lock(mutex_); - cv_.wait(lock, [&] { return done_resource == num_resource_; }); + cv_.wait(lock, [&] { + return done_resource == num_resource_; + }); meta_ptr_->DeleteTableFiles(table_id_); } -void DeleteJob::ResourceDone() { +void +DeleteJob::ResourceDone() { { std::lock_guard lock(mutex_); ++done_resource; @@ -45,7 +51,6 @@ void DeleteJob::ResourceDone() { cv_.notify_one(); } -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/DeleteJob.h b/cpp/src/scheduler/job/DeleteJob.h index d82262b235..7d8b20e47c 100644 --- a/cpp/src/scheduler/job/DeleteJob.h +++ b/cpp/src/scheduler/job/DeleteJob.h @@ -30,26 +30,25 @@ #include "Job.h" #include "db/meta/Meta.h" - namespace zilliz { namespace milvus { namespace scheduler { class DeleteJob : public Job { -public: + public: DeleteJob(JobId id, std::string table_id, engine::meta::MetaPtr meta_ptr, uint64_t num_resource); -public: + public: void WaitAndDelete(); void ResourceDone(); -public: + public: std::string table_id() const { return table_id_; @@ -60,7 +59,7 @@ public: return meta_ptr_; } -private: + private: std::string table_id_; engine::meta::MetaPtr meta_ptr_; @@ -72,7 +71,6 @@ private: using DeleteJobPtr = std::shared_ptr; -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/Job.h b/cpp/src/scheduler/job/Job.h index 242c33a4e5..c646a4f034 100644 --- a/cpp/src/scheduler/job/Job.h +++ b/cpp/src/scheduler/job/Job.h @@ -27,7 +27,6 @@ #include #include - namespace zilliz { namespace milvus { namespace scheduler { @@ -42,7 +41,7 @@ enum class JobType { using JobId = std::uint64_t; class Job { -public: + public: inline JobId id() const { return id_; @@ -53,10 +52,11 @@ public: return type_; } -protected: - Job(JobId id, JobType type) : id_(id), type_(type) {} + protected: + Job(JobId id, JobType type) : id_(id), type_(type) { + } -private: + private: JobId id_; JobType type_; }; @@ -64,7 +64,6 @@ private: using JobPtr = std::shared_ptr; using JobWPtr = std::weak_ptr; -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/SearchJob.cpp b/cpp/src/scheduler/job/SearchJob.cpp index 65b6701b5f..dee7125fed 100644 --- a/cpp/src/scheduler/job/SearchJob.cpp +++ b/cpp/src/scheduler/job/SearchJob.cpp @@ -15,11 +15,9 @@ // specific language governing permissions and limitations // under the License. +#include "scheduler/job/SearchJob.h" #include "utils/Log.h" -#include "SearchJob.h" - - namespace zilliz { namespace milvus { namespace scheduler { @@ -33,7 +31,8 @@ SearchJob::SearchJob(zilliz::milvus::scheduler::JobId id, topk_(topk), nq_(nq), nprobe_(nprobe), - vectors_(vectors) {} + vectors_(vectors) { +} bool SearchJob::AddIndexFile(const TableFileSchemaPtr &index_file) { @@ -48,11 +47,12 @@ SearchJob::AddIndexFile(const TableFileSchemaPtr &index_file) { return true; } - void SearchJob::WaitResult() { std::unique_lock lock(mutex_); - cv_.wait(lock, [this] { return index_files_.empty(); }); + cv_.wait(lock, [this] { + return index_files_.empty(); + }); SERVER_LOG_DEBUG << "SearchJob " << id() << " all done"; } @@ -69,14 +69,11 @@ SearchJob::GetResult() { return result_; } -Status& +Status & SearchJob::GetStatus() { return status_; } - -} -} -} - - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/SearchJob.h b/cpp/src/scheduler/job/SearchJob.h index 12edeab199..7bb7fbefbf 100644 --- a/cpp/src/scheduler/job/SearchJob.h +++ b/cpp/src/scheduler/job/SearchJob.h @@ -26,16 +26,15 @@ #include #include #include +#include #include "Job.h" #include "db/meta/MetaTypes.h" - namespace zilliz { namespace milvus { namespace scheduler { - using engine::meta::TableFileSchemaPtr; using Id2IndexMap = std::unordered_map; @@ -43,10 +42,10 @@ using Id2DistanceMap = std::vector>; using ResultSet = std::vector; class SearchJob : public Job { -public: + public: SearchJob(JobId id, uint64_t topk, uint64_t nq, uint64_t nprobe, const float *vectors); -public: + public: bool AddIndexFile(const TableFileSchemaPtr &index_file); @@ -62,7 +61,7 @@ public: Status & GetStatus(); -public: + public: uint64_t topk() const { return topk_; @@ -77,6 +76,7 @@ public: nprobe() const { return nprobe_; } + const float * vectors() const { return vectors_; @@ -87,7 +87,7 @@ public: return index_files_; } -private: + private: uint64_t topk_ = 0; uint64_t nq_ = 0; uint64_t nprobe_ = 0; @@ -105,7 +105,6 @@ private: using SearchJobPtr = std::shared_ptr; -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Connection.h b/cpp/src/scheduler/resource/Connection.h index 38fbc65bb5..cf18b6c9a2 100644 --- a/cpp/src/scheduler/resource/Connection.h +++ b/cpp/src/scheduler/resource/Connection.h @@ -19,17 +19,18 @@ #include #include - +#include namespace zilliz { namespace milvus { namespace scheduler { class Connection { -public: + public: // TODO: update construct function, speed: double->uint64_t Connection(std::string name, double speed) - : name_(std::move(name)), speed_(speed) {} + : name_(std::move(name)), speed_(speed) { + } const std::string & name() const { @@ -46,7 +47,7 @@ public: return 1024 / speed_; } -public: + public: std::string Dump() const { std::stringstream ss; @@ -54,12 +55,11 @@ public: return ss.str(); } -private: + private: std::string name_; uint64_t speed_; }; - -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/CpuResource.cpp b/cpp/src/scheduler/resource/CpuResource.cpp index 3933bbb5a0..5859dfd0cd 100644 --- a/cpp/src/scheduler/resource/CpuResource.cpp +++ b/cpp/src/scheduler/resource/CpuResource.cpp @@ -16,29 +16,34 @@ // under the License. -#include "CpuResource.h" +#include "scheduler/resource/CpuResource.h" +#include namespace zilliz { namespace milvus { namespace scheduler { -std::ostream &operator<<(std::ostream &out, const CpuResource &resource) { +std::ostream & +operator<<(std::ostream &out, const CpuResource &resource) { out << resource.Dump(); return out; } CpuResource::CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor) - : Resource(std::move(name), ResourceType::CPU, device_id, enable_loader, enable_executor) {} + : Resource(std::move(name), ResourceType::CPU, device_id, enable_loader, enable_executor) { +} -void CpuResource::LoadFile(TaskPtr task) { +void +CpuResource::LoadFile(TaskPtr task) { task->Load(LoadType::DISK2CPU, 0); } -void CpuResource::Process(TaskPtr task) { +void +CpuResource::Process(TaskPtr task) { task->Execute(); } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/CpuResource.h b/cpp/src/scheduler/resource/CpuResource.h index 1f6f235621..2226523fdf 100644 --- a/cpp/src/scheduler/resource/CpuResource.h +++ b/cpp/src/scheduler/resource/CpuResource.h @@ -21,13 +21,12 @@ #include "Resource.h" - namespace zilliz { namespace milvus { namespace scheduler { class CpuResource : public Resource { -public: + public: explicit CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); @@ -38,7 +37,7 @@ public: friend std::ostream &operator<<(std::ostream &out, const CpuResource &resource); -protected: + protected: void LoadFile(TaskPtr task) override; @@ -46,6 +45,6 @@ protected: Process(TaskPtr task) override; }; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/DiskResource.cpp b/cpp/src/scheduler/resource/DiskResource.cpp index 6495396fdb..eee2424cc1 100644 --- a/cpp/src/scheduler/resource/DiskResource.cpp +++ b/cpp/src/scheduler/resource/DiskResource.cpp @@ -15,14 +15,17 @@ // specific language governing permissions and limitations // under the License. -#include "DiskResource.h" +#include "scheduler/resource/DiskResource.h" +#include +#include namespace zilliz { namespace milvus { namespace scheduler { -std::ostream &operator<<(std::ostream &out, const DiskResource &resource) { +std::ostream & +operator<<(std::ostream &out, const DiskResource &resource) { out << resource.Dump(); return out; } @@ -31,15 +34,14 @@ DiskResource::DiskResource(std::string name, uint64_t device_id, bool enable_loa : Resource(std::move(name), ResourceType::DISK, device_id, enable_loader, enable_executor) { } -void DiskResource::LoadFile(TaskPtr task) { - +void +DiskResource::LoadFile(TaskPtr task) { } -void DiskResource::Process(TaskPtr task) { - +void +DiskResource::Process(TaskPtr task) { } -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/DiskResource.h b/cpp/src/scheduler/resource/DiskResource.h index b9e7a4035c..a7caf5c662 100644 --- a/cpp/src/scheduler/resource/DiskResource.h +++ b/cpp/src/scheduler/resource/DiskResource.h @@ -17,16 +17,16 @@ #pragma once - #include "Resource.h" +#include namespace zilliz { namespace milvus { namespace scheduler { class DiskResource : public Resource { -public: + public: explicit DiskResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); @@ -37,7 +37,7 @@ public: friend std::ostream &operator<<(std::ostream &out, const DiskResource &resource); -protected: + protected: void LoadFile(TaskPtr task) override; @@ -45,6 +45,6 @@ protected: Process(TaskPtr task) override; }; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/GpuResource.cpp b/cpp/src/scheduler/resource/GpuResource.cpp index c3981e65cb..3c7abc0b29 100644 --- a/cpp/src/scheduler/resource/GpuResource.cpp +++ b/cpp/src/scheduler/resource/GpuResource.cpp @@ -16,29 +16,32 @@ // under the License. -#include "GpuResource.h" - +#include "scheduler/resource/GpuResource.h" namespace zilliz { namespace milvus { namespace scheduler { -std::ostream &operator<<(std::ostream &out, const GpuResource &resource) { +std::ostream & +operator<<(std::ostream &out, const GpuResource &resource) { out << resource.Dump(); return out; } GpuResource::GpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor) - : Resource(std::move(name), ResourceType::GPU, device_id, enable_loader, enable_executor) {} + : Resource(std::move(name), ResourceType::GPU, device_id, enable_loader, enable_executor) { +} -void GpuResource::LoadFile(TaskPtr task) { +void +GpuResource::LoadFile(TaskPtr task) { task->Load(LoadType::CPU2GPU, device_id_); } -void GpuResource::Process(TaskPtr task) { +void +GpuResource::Process(TaskPtr task) { task->Execute(); } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/GpuResource.h b/cpp/src/scheduler/resource/GpuResource.h index f08e767c51..9f19b07464 100644 --- a/cpp/src/scheduler/resource/GpuResource.h +++ b/cpp/src/scheduler/resource/GpuResource.h @@ -17,16 +17,17 @@ #pragma once - #include "Resource.h" +#include +#include namespace zilliz { namespace milvus { namespace scheduler { class GpuResource : public Resource { -public: + public: explicit GpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); @@ -37,7 +38,7 @@ public: friend std::ostream &operator<<(std::ostream &out, const GpuResource &resource); -protected: + protected: void LoadFile(TaskPtr task) override; @@ -45,6 +46,6 @@ protected: Process(TaskPtr task) override; }; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Node.cpp b/cpp/src/scheduler/resource/Node.cpp index 719bfb1427..cf652b8ba0 100644 --- a/cpp/src/scheduler/resource/Node.cpp +++ b/cpp/src/scheduler/resource/Node.cpp @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. +#include "scheduler/resource/Node.h" #include -#include "Node.h" - +#include namespace zilliz { namespace milvus { @@ -29,7 +29,8 @@ Node::Node() { id_ = counter++; } -std::vector Node::GetNeighbours() { +std::vector +Node::GetNeighbours() { std::lock_guard lk(mutex_); std::vector ret; for (auto &e : neighbours_) { @@ -38,7 +39,8 @@ std::vector Node::GetNeighbours() { return ret; } -std::string Node::Dump() { +std::string +Node::Dump() { std::stringstream ss; ss << "::neighbours:" << std::endl; for (auto &neighbour : neighbours_) { @@ -48,7 +50,8 @@ std::string Node::Dump() { return ss.str(); } -void Node::AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &connection) { +void +Node::AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &connection) { std::lock_guard lk(mutex_); if (auto s = neighbour_node.lock()) { neighbours_.emplace(std::make_pair(s->id_, Neighbour(neighbour_node, connection))); @@ -56,6 +59,6 @@ void Node::AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &conn // else do nothing, consider it.. } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Node.h b/cpp/src/scheduler/resource/Node.h index 46506da10d..00337a7009 100644 --- a/cpp/src/scheduler/resource/Node.h +++ b/cpp/src/scheduler/resource/Node.h @@ -20,11 +20,11 @@ #include #include #include +#include -#include "../TaskTable.h" +#include "scheduler/TaskTable.h" #include "Connection.h" - namespace zilliz { namespace milvus { namespace scheduler { @@ -34,8 +34,9 @@ class Node; using NeighbourNodePtr = std::weak_ptr; struct Neighbour { - Neighbour(NeighbourNodePtr nei, Connection conn) - : neighbour_node(nei), connection(conn) {} + Neighbour(NeighbourNodePtr nei, Connection conn) + : neighbour_node(nei), connection(conn) { + } NeighbourNodePtr neighbour_node; Connection connection; @@ -43,7 +44,7 @@ struct Neighbour { // TODO(linxj): return type void -> Status class Node { -public: + public: Node(); void @@ -52,11 +53,11 @@ public: std::vector GetNeighbours(); -public: + public: std::string Dump(); -private: + private: std::mutex mutex_; uint8_t id_; std::map neighbours_; @@ -65,6 +66,6 @@ private: using NodePtr = std::shared_ptr; using NodeWPtr = std::weak_ptr; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Resource.cpp b/cpp/src/scheduler/resource/Resource.cpp index 22e7eb8502..4eb71b8ac5 100644 --- a/cpp/src/scheduler/resource/Resource.cpp +++ b/cpp/src/scheduler/resource/Resource.cpp @@ -15,10 +15,11 @@ // specific language governing permissions and limitations // under the License. -#include -#include "../Utils.h" -#include "Resource.h" +#include "scheduler/resource/Resource.h" +#include "scheduler/Utils.h" +#include +#include namespace zilliz { namespace milvus { @@ -100,7 +101,8 @@ Resource::NumOfTaskToExec() { return count; } -TaskTableItemPtr Resource::pick_task_load() { +TaskTableItemPtr +Resource::pick_task_load() { auto indexes = task_table_.PickToLoad(10); for (auto index : indexes) { // try to set one task loading, then return @@ -111,7 +113,8 @@ TaskTableItemPtr Resource::pick_task_load() { return nullptr; } -TaskTableItemPtr Resource::pick_task_execute() { +TaskTableItemPtr +Resource::pick_task_execute() { auto indexes = task_table_.PickToExecute(3); for (auto index : indexes) { // try to set one task executing, then return @@ -122,10 +125,13 @@ TaskTableItemPtr Resource::pick_task_execute() { return nullptr; } -void Resource::loader_function() { +void +Resource::loader_function() { while (running_) { std::unique_lock lock(load_mutex_); - load_cv_.wait(lock, [&] { return load_flag_; }); + load_cv_.wait(lock, [&] { + return load_flag_; + }); load_flag_ = false; lock.unlock(); while (true) { @@ -140,18 +146,20 @@ void Resource::loader_function() { subscriber_(std::static_pointer_cast(event)); } } - } } -void Resource::executor_function() { +void +Resource::executor_function() { if (subscriber_) { auto event = std::make_shared(shared_from_this()); subscriber_(std::static_pointer_cast(event)); } while (running_) { std::unique_lock lock(exec_mutex_); - exec_cv_.wait(lock, [&] { return exec_flag_; }); + exec_cv_.wait(lock, [&] { + return exec_flag_; + }); exec_flag_ = false; lock.unlock(); while (true) { @@ -172,10 +180,9 @@ void Resource::executor_function() { subscriber_(std::static_pointer_cast(event)); } } - } } -} -} -} \ No newline at end of file +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Resource.h b/cpp/src/scheduler/resource/Resource.h index 0b0eaae304..1c18b1a2b2 100644 --- a/cpp/src/scheduler/resource/Resource.h +++ b/cpp/src/scheduler/resource/Resource.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -34,7 +35,6 @@ #include "Connection.h" #include "Node.h" - namespace zilliz { namespace milvus { namespace scheduler { @@ -104,7 +104,7 @@ class Resource : public Node, public std::enable_shared_from_this { return task_table_; } -public: + public: inline bool HasLoader() const { return enable_loader_; @@ -212,7 +212,6 @@ public: using ResourcePtr = std::shared_ptr; using ResourceWPtr = std::weak_ptr; -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/TestResource.cpp b/cpp/src/scheduler/resource/TestResource.cpp index b49ea12d77..25560cf7ee 100644 --- a/cpp/src/scheduler/resource/TestResource.cpp +++ b/cpp/src/scheduler/resource/TestResource.cpp @@ -15,14 +15,16 @@ // specific language governing permissions and limitations // under the License. -#include "TestResource.h" +#include "scheduler/resource/TestResource.h" +#include namespace zilliz { namespace milvus { namespace scheduler { -std::ostream &operator<<(std::ostream &out, const TestResource &resource) { +std::ostream & +operator<<(std::ostream &out, const TestResource &resource) { out << resource.Dump(); return out; } @@ -31,15 +33,16 @@ TestResource::TestResource(std::string name, uint64_t device_id, bool enable_loa : Resource(std::move(name), ResourceType::TEST, device_id, enable_loader, enable_executor) { } -void TestResource::LoadFile(TaskPtr task) { +void +TestResource::LoadFile(TaskPtr task) { task->Load(LoadType::TEST, 0); } -void TestResource::Process(TaskPtr task) { +void +TestResource::Process(TaskPtr task) { task->Execute(); } -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/TestResource.h b/cpp/src/scheduler/resource/TestResource.h index 861efbd825..ac83a42c60 100644 --- a/cpp/src/scheduler/resource/TestResource.h +++ b/cpp/src/scheduler/resource/TestResource.h @@ -17,16 +17,17 @@ #pragma once - #include "Resource.h" +#include +#include namespace zilliz { namespace milvus { namespace scheduler { class TestResource : public Resource { -public: + public: explicit TestResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); @@ -37,7 +38,7 @@ public: friend std::ostream &operator<<(std::ostream &out, const TestResource &resource); -protected: + protected: void LoadFile(TaskPtr task) override; @@ -45,6 +46,6 @@ protected: Process(TaskPtr task) override; }; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/DeleteTask.cpp b/cpp/src/scheduler/task/DeleteTask.cpp index 65a7a537fd..52579d67c6 100644 --- a/cpp/src/scheduler/task/DeleteTask.cpp +++ b/cpp/src/scheduler/task/DeleteTask.cpp @@ -16,19 +16,18 @@ // under the License. -#include "DeleteTask.h" - +#include "scheduler/task/DeleteTask.h" namespace zilliz { namespace milvus { namespace scheduler { XDeleteTask::XDeleteTask(const scheduler::DeleteJobPtr &delete_job) - : Task(TaskType::DeleteTask), delete_job_(delete_job) {} + : Task(TaskType::DeleteTask), delete_job_(delete_job) { +} void XDeleteTask::Load(LoadType type, uint8_t device_id) { - } void @@ -36,6 +35,6 @@ XDeleteTask::Execute() { delete_job_->ResourceDone(); } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/DeleteTask.h b/cpp/src/scheduler/task/DeleteTask.h index b64f9a977f..608960e7c8 100644 --- a/cpp/src/scheduler/task/DeleteTask.h +++ b/cpp/src/scheduler/task/DeleteTask.h @@ -20,15 +20,13 @@ #include "scheduler/job/DeleteJob.h" #include "Task.h" - namespace zilliz { namespace milvus { namespace scheduler { class XDeleteTask : public Task { -public: - explicit - XDeleteTask(const scheduler::DeleteJobPtr &job); + public: + explicit XDeleteTask(const scheduler::DeleteJobPtr &job); void Load(LoadType type, uint8_t device_id) override; @@ -36,10 +34,10 @@ public: void Execute() override; -public: + public: scheduler::DeleteJobPtr delete_job_; }; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/Path.h b/cpp/src/scheduler/task/Path.h index a463494b77..672dfff1b9 100644 --- a/cpp/src/scheduler/task/Path.h +++ b/cpp/src/scheduler/task/Path.h @@ -20,7 +20,6 @@ #include #include - namespace zilliz { namespace milvus { namespace scheduler { @@ -29,7 +28,8 @@ class Path { public: Path() = default; - Path(std::vector& path, uint64_t index) : path_(path), index_(index) {} + Path(std::vector &path, uint64_t index) : path_(path), index_(index) { + } void push_back(const std::string &str) { @@ -49,7 +49,6 @@ class Path { } else { return nullptr; } - } std::string @@ -67,14 +66,19 @@ class Path { return path_[index]; } - std::vector::iterator begin() { return path_.begin(); } - std::vector::iterator end() { return path_.end(); } + std::vector::iterator begin() { + return path_.begin(); + } + + std::vector::iterator end() { + return path_.end(); + } public: std::vector path_; uint64_t index_ = 0; }; -} -} -} \ No newline at end of file +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/SearchTask.cpp b/cpp/src/scheduler/task/SearchTask.cpp index 284152f14e..0c205fcafa 100644 --- a/cpp/src/scheduler/task/SearchTask.cpp +++ b/cpp/src/scheduler/task/SearchTask.cpp @@ -15,15 +15,16 @@ // specific language governing permissions and limitations // under the License. -#include "SearchTask.h" +#include "scheduler/task/SearchTask.h" +#include "scheduler/job/SearchJob.h" #include "metrics/Metrics.h" #include "db/engine/EngineFactory.h" #include "utils/TimeRecorder.h" #include "utils/Log.h" #include -#include "scheduler/job/SearchJob.h" - +#include +#include namespace zilliz { namespace milvus { @@ -104,7 +105,6 @@ XSearchTask::XSearchTask(TableFileSchemaPtr file) (MetricType) file_->metric_type_, file_->nlist_); } - } void @@ -144,7 +144,7 @@ XSearchTask::Load(LoadType type, uint8_t device_id) { s = Status(SERVER_UNEXPECTED_ERROR, error_msg); } - if (auto job = job_.lock()){ + if (auto job = job_.lock()) { auto search_job = std::static_pointer_cast(job); search_job->SearchDone(file_->id_); search_job->GetStatus() = s; @@ -183,7 +183,7 @@ XSearchTask::Execute() { server::CollectDurationMetrics metrics(index_type_); - std::vector output_ids; + std::vector output_ids; std::vector output_distance; if (auto job = job_.lock()) { @@ -192,7 +192,7 @@ XSearchTask::Execute() { uint64_t nq = search_job->nq(); uint64_t topk = search_job->topk(); uint64_t nprobe = search_job->nprobe(); - const float* vectors = search_job->vectors(); + const float *vectors = search_job->vectors(); output_ids.resize(topk * nq); output_distance.resize(topk * nq); @@ -236,11 +236,12 @@ XSearchTask::Execute() { index_engine_ = nullptr; } -Status XSearchTask::ClusterResult(const std::vector &output_ids, - const std::vector &output_distance, - uint64_t nq, - uint64_t topk, - scheduler::ResultSet &result_set) { +Status +XSearchTask::ClusterResult(const std::vector &output_ids, + const std::vector &output_distance, + uint64_t nq, + uint64_t topk, + scheduler::ResultSet &result_set) { if (output_ids.size() < nq * topk || output_distance.size() < nq * topk) { std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) + " distance array size: " + std::to_string(output_distance.size()); @@ -275,10 +276,11 @@ Status XSearchTask::ClusterResult(const std::vector &output_ids, return Status::OK(); } -Status XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, - scheduler::Id2DistanceMap &distance_target, - uint64_t topk, - bool ascending) { +Status +XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, + scheduler::Id2DistanceMap &distance_target, + uint64_t topk, + bool ascending) { //Note: the score_src and score_target are already arranged by score in ascending order if (distance_src.empty()) { ENGINE_LOG_WARNING << "Empty distance source array"; @@ -349,10 +351,11 @@ Status XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, return Status::OK(); } -Status XSearchTask::TopkResult(scheduler::ResultSet &result_src, - uint64_t topk, - bool ascending, - scheduler::ResultSet &result_target) { +Status +XSearchTask::TopkResult(scheduler::ResultSet &result_src, + uint64_t topk, + bool ascending, + scheduler::ResultSet &result_target) { if (result_target.empty()) { result_target.swap(result_src); return Status::OK(); @@ -381,7 +384,6 @@ Status XSearchTask::TopkResult(scheduler::ResultSet &result_src, return Status::OK(); } - -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/SearchTask.h b/cpp/src/scheduler/task/SearchTask.h index 6f0ec6bc53..7c19ba20f9 100644 --- a/cpp/src/scheduler/task/SearchTask.h +++ b/cpp/src/scheduler/task/SearchTask.h @@ -21,6 +21,7 @@ #include "scheduler/job/SearchJob.h" #include "scheduler/Definition.h" +#include namespace zilliz { namespace milvus { @@ -28,9 +29,8 @@ namespace scheduler { // TODO: rewrite class XSearchTask : public Task { -public: - explicit - XSearchTask(TableFileSchemaPtr file); + public: + explicit XSearchTask(TableFileSchemaPtr file); void Load(LoadType type, uint8_t device_id) override; @@ -38,8 +38,8 @@ public: void Execute() override; -public: - static Status ClusterResult(const std::vector &output_ids, + public: + static Status ClusterResult(const std::vector &output_ids, const std::vector &output_distence, uint64_t nq, uint64_t topk, @@ -55,7 +55,7 @@ public: bool ascending, scheduler::ResultSet &result_target); -public: + public: TableFileSchemaPtr file_; size_t index_id_ = 0; @@ -66,6 +66,6 @@ public: static std::mutex merge_mutex_; }; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/Task.h b/cpp/src/scheduler/task/Task.h index 785ef9cfb7..3600c10f03 100644 --- a/cpp/src/scheduler/task/Task.h +++ b/cpp/src/scheduler/task/Task.h @@ -25,7 +25,6 @@ #include #include - namespace zilliz { namespace milvus { namespace scheduler { @@ -49,20 +48,22 @@ using TaskPtr = std::shared_ptr; // TODO: re-design class Task { -public: - explicit - Task(TaskType type) : type_(type) {} + public: + explicit Task(TaskType type) : type_(type) { + } /* * Just Getter; */ inline TaskType - Type() const { return type_; } + Type() const { + return type_; + } /* * Transport path; */ - inline Path& + inline Path & path() { return task_path_; } @@ -75,14 +76,14 @@ public: return label_; } -public: + public: virtual void Load(LoadType type, uint8_t device_id) = 0; virtual void Execute() = 0; -public: + public: Path task_path_; // std::vector search_contexts_; scheduler::JobWPtr job_; @@ -90,7 +91,6 @@ public: TaskLabelPtr label_ = nullptr; }; - -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/TestTask.cpp b/cpp/src/scheduler/task/TestTask.cpp index 32e0a8587e..fc66e56269 100644 --- a/cpp/src/scheduler/task/TestTask.cpp +++ b/cpp/src/scheduler/task/TestTask.cpp @@ -15,16 +15,15 @@ // specific language governing permissions and limitations // under the License. - -#include -#include "TestTask.h" - +#include "scheduler/task/TestTask.h" +#include "cache/GpuCacheMgr.h" namespace zilliz { namespace milvus { namespace scheduler { -TestTask::TestTask(TableFileSchemaPtr &file) : XSearchTask(file) {} +TestTask::TestTask(TableFileSchemaPtr &file) : XSearchTask(file) { +} void TestTask::Load(LoadType type, uint8_t device_id) { @@ -44,10 +43,11 @@ TestTask::Execute() { void TestTask::Wait() { std::unique_lock lock(mutex_); - cv_.wait(lock, [&] { return done_; }); -} - -} -} + cv_.wait(lock, [&] { + return done_; + }); } +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/TestTask.h b/cpp/src/scheduler/task/TestTask.h index 3367506d88..7051080861 100644 --- a/cpp/src/scheduler/task/TestTask.h +++ b/cpp/src/scheduler/task/TestTask.h @@ -19,17 +19,15 @@ #include "SearchTask.h" - namespace zilliz { namespace milvus { namespace scheduler { class TestTask : public XSearchTask { -public: - explicit - TestTask(TableFileSchemaPtr& file); + public: + explicit TestTask(TableFileSchemaPtr &file); -public: + public: void Load(LoadType type, uint8_t device_id) override; @@ -39,7 +37,7 @@ public: void Wait(); -public: + public: uint64_t load_count_ = 0; uint64_t exec_count_ = 0; @@ -48,7 +46,6 @@ public: std::condition_variable cv_; }; - -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/tasklabel/BroadcastLabel.h b/cpp/src/scheduler/tasklabel/BroadcastLabel.h index 1c89460df0..6fca107864 100644 --- a/cpp/src/scheduler/tasklabel/BroadcastLabel.h +++ b/cpp/src/scheduler/tasklabel/BroadcastLabel.h @@ -21,19 +21,19 @@ #include - namespace zilliz { namespace milvus { namespace scheduler { - class BroadcastLabel : public TaskLabel { -public: - BroadcastLabel() : TaskLabel(TaskLabelType::BROADCAST) {} + public: + BroadcastLabel() : TaskLabel(TaskLabelType::BROADCAST) { + } }; using BroadcastLabelPtr = std::shared_ptr; -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz + diff --git a/cpp/src/scheduler/tasklabel/DefaultLabel.h b/cpp/src/scheduler/tasklabel/DefaultLabel.h index d61028117d..7943c4f7c1 100644 --- a/cpp/src/scheduler/tasklabel/DefaultLabel.h +++ b/cpp/src/scheduler/tasklabel/DefaultLabel.h @@ -21,20 +21,18 @@ #include - namespace zilliz { namespace milvus { namespace scheduler { class DefaultLabel : public TaskLabel { -public: - DefaultLabel() : TaskLabel(TaskLabelType::DEFAULT) {} + public: + DefaultLabel() : TaskLabel(TaskLabelType::DEFAULT) { + } }; using DefaultLabelPtr = std::shared_ptr; -} -} -} - - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/tasklabel/SpecResLabel.h b/cpp/src/scheduler/tasklabel/SpecResLabel.h index 8e94a7677b..cfc5aa9400 100644 --- a/cpp/src/scheduler/tasklabel/SpecResLabel.h +++ b/cpp/src/scheduler/tasklabel/SpecResLabel.h @@ -22,7 +22,6 @@ #include #include - class Resource; using ResourceWPtr = std::weak_ptr; @@ -32,9 +31,10 @@ namespace milvus { namespace scheduler { class SpecResLabel : public TaskLabel { -public: - SpecResLabel(const ResourceWPtr &resource) - : TaskLabel(TaskLabelType::SPECIFIED_RESOURCE), resource_(resource) {} + public: + explicit SpecResLabel(const ResourceWPtr &resource) + : TaskLabel(TaskLabelType::SPECIFIED_RESOURCE), resource_(resource) { + } inline ResourceWPtr & resource() { @@ -46,14 +46,13 @@ public: return resource_name_; } -private: + private: ResourceWPtr resource_; std::string resource_name_; }; using SpecResLabelPtr = std::shared_ptr(); -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/tasklabel/TaskLabel.h b/cpp/src/scheduler/tasklabel/TaskLabel.h index 5d43adb7db..d8d404e094 100644 --- a/cpp/src/scheduler/tasklabel/TaskLabel.h +++ b/cpp/src/scheduler/tasklabel/TaskLabel.h @@ -30,23 +30,22 @@ enum class TaskLabelType { }; class TaskLabel { -public: + public: inline TaskLabelType Type() const { return type_; } -protected: - explicit - TaskLabel(TaskLabelType type) : type_(type) {} + protected: + explicit TaskLabel(TaskLabelType type) : type_(type) { + } -private: + private: TaskLabelType type_; }; using TaskLabelPtr = std::shared_ptr; -} -} -} - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz From a0e4d19e4f1843f7c9de083ba41bbcfa66c03df0 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 27 Sep 2019 14:13:33 +0800 Subject: [PATCH 130/217] format db code Former-commit-id: 3b4781e4c000beea41baddf54635d7bbe7ea5f52 --- cpp/src/db/Constants.h | 4 +- cpp/src/db/DB.h | 48 +- cpp/src/db/DBFactory.cpp | 10 +- cpp/src/db/DBFactory.h | 11 +- cpp/src/db/DBImpl.cpp | 294 ++++++----- cpp/src/db/DBImpl.h | 12 +- cpp/src/db/IDGenerator.cpp | 22 +- cpp/src/db/IDGenerator.h | 2 - cpp/src/db/Options.cpp | 23 +- cpp/src/db/Options.h | 25 +- cpp/src/db/Types.h | 9 +- cpp/src/db/Utils.cpp | 74 +-- cpp/src/db/Utils.h | 33 +- cpp/src/db/engine/EngineFactory.cpp | 19 +- cpp/src/db/engine/EngineFactory.h | 13 +- cpp/src/db/engine/ExecutionEngine.h | 23 +- cpp/src/db/engine/ExecutionEngineImpl.cpp | 87 ++-- cpp/src/db/engine/ExecutionEngineImpl.h | 32 +- cpp/src/db/insert/MemManager.h | 5 +- cpp/src/db/insert/MemManagerImpl.cpp | 48 +- cpp/src/db/insert/MemManagerImpl.h | 9 +- cpp/src/db/insert/MemMenagerFactory.cpp | 6 +- cpp/src/db/insert/MemMenagerFactory.h | 10 +- cpp/src/db/insert/MemTable.cpp | 30 +- cpp/src/db/insert/MemTable.h | 7 +- cpp/src/db/insert/MemTableFile.cpp | 38 +- cpp/src/db/insert/MemTableFile.h | 14 +- cpp/src/db/insert/VectorSource.cpp | 35 +- cpp/src/db/insert/VectorSource.h | 5 +- cpp/src/db/meta/Meta.h | 13 +- cpp/src/db/meta/MetaConsts.h | 20 +- cpp/src/db/meta/MetaFactory.cpp | 67 +-- cpp/src/db/meta/MetaFactory.h | 11 +- cpp/src/db/meta/MetaTypes.h | 11 +- cpp/src/db/meta/MySQLConnectionPool.cpp | 105 ++-- cpp/src/db/meta/MySQLConnectionPool.h | 24 +- cpp/src/db/meta/MySQLMetaImpl.cpp | 590 +++++++++++----------- cpp/src/db/meta/MySQLMetaImpl.h | 21 +- cpp/src/db/meta/SqliteMetaImpl.cpp | 438 ++++++++-------- cpp/src/db/meta/SqliteMetaImpl.h | 17 +- 40 files changed, 1174 insertions(+), 1091 deletions(-) diff --git a/cpp/src/db/Constants.h b/cpp/src/db/Constants.h index aa254b0bd6..2beb3c3a97 100644 --- a/cpp/src/db/Constants.h +++ b/cpp/src/db/Constants.h @@ -33,8 +33,8 @@ constexpr uint64_t MAX_TABLE_FILE_MEM = 128 * M; constexpr int VECTOR_TYPE_SIZE = sizeof(float); static constexpr uint64_t ONE_KB = K; -static constexpr uint64_t ONE_MB = ONE_KB*ONE_KB; -static constexpr uint64_t ONE_GB = ONE_KB*ONE_MB; +static constexpr uint64_t ONE_MB = ONE_KB * ONE_KB; +static constexpr uint64_t ONE_GB = ONE_KB * ONE_MB; } // namespace engine } // namespace milvus diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index 5fe6df491f..127812fc30 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -25,6 +25,7 @@ #include #include +#include namespace zilliz { namespace milvus { @@ -33,46 +34,45 @@ namespace engine { class Env; class DB { -public: + public: DB() = default; - DB(const DB&) = delete; - DB& operator=(const DB&) = delete; + DB(const DB &) = delete; + DB &operator=(const DB &) = delete; virtual ~DB() = default; virtual Status Start() = 0; virtual Status Stop() = 0; - virtual Status CreateTable(meta::TableSchema& table_schema_) = 0; - virtual Status DeleteTable(const std::string& table_id, const meta::DatesT& dates) = 0; - virtual Status DescribeTable(meta::TableSchema& table_schema_) = 0; - virtual Status HasTable(const std::string& table_id, bool& has_or_not_) = 0; - virtual Status AllTables(std::vector& table_schema_array) = 0; - virtual Status GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0; - virtual Status PreloadTable(const std::string& table_id) = 0; + virtual Status CreateTable(meta::TableSchema &table_schema_) = 0; + virtual Status DeleteTable(const std::string &table_id, const meta::DatesT &dates) = 0; + virtual Status DescribeTable(meta::TableSchema &table_schema_) = 0; + virtual Status HasTable(const std::string &table_id, bool &has_or_not_) = 0; + virtual Status AllTables(std::vector &table_schema_array) = 0; + virtual Status GetTableRowCount(const std::string &table_id, uint64_t &row_count) = 0; + virtual Status PreloadTable(const std::string &table_id) = 0; virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0; - virtual Status InsertVectors(const std::string& table_id_, - uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0; + virtual Status InsertVectors(const std::string &table_id_, + uint64_t n, const float *vectors, IDNumbers &vector_ids_) = 0; - virtual Status Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, - const float* vectors, QueryResults& results) = 0; + virtual Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, + const float *vectors, QueryResults &results) = 0; - virtual Status Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, - const float* vectors, const meta::DatesT& dates, QueryResults& results) = 0; + virtual Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, + const float *vectors, const meta::DatesT &dates, QueryResults &results) = 0; - virtual Status Query(const std::string& table_id, const std::vector& file_ids, - uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, - const meta::DatesT& dates, QueryResults& results) = 0; + virtual Status Query(const std::string &table_id, const std::vector &file_ids, + uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors, + const meta::DatesT &dates, QueryResults &results) = 0; - virtual Status Size(uint64_t& result) = 0; + virtual Status Size(uint64_t &result) = 0; - virtual Status CreateIndex(const std::string& table_id, const TableIndex& index) = 0; - virtual Status DescribeIndex(const std::string& table_id, TableIndex& index) = 0; - virtual Status DropIndex(const std::string& table_id) = 0; + virtual Status CreateIndex(const std::string &table_id, const TableIndex &index) = 0; + virtual Status DescribeIndex(const std::string &table_id, TableIndex &index) = 0; + virtual Status DropIndex(const std::string &table_id) = 0; virtual Status DropAll() = 0; - }; // DB using DBPtr = std::shared_ptr; diff --git a/cpp/src/db/DBFactory.cpp b/cpp/src/db/DBFactory.cpp index 2eec49fc7a..edf70cccf8 100644 --- a/cpp/src/db/DBFactory.cpp +++ b/cpp/src/db/DBFactory.cpp @@ -16,7 +16,7 @@ // under the License. -#include "DBFactory.h" +#include "db/DBFactory.h" #include "DBImpl.h" #include "utils/Exception.h" #include "meta/MetaFactory.h" @@ -33,14 +33,16 @@ namespace zilliz { namespace milvus { namespace engine { -DBOptions DBFactory::BuildOption() { +DBOptions +DBFactory::BuildOption() { auto meta = MetaFactory::BuildOption(); - DBOptions options; + DBOptions options; options.meta_ = meta; return options; } -DBPtr DBFactory::Build(const DBOptions& options) { +DBPtr +DBFactory::Build(const DBOptions &options) { return std::make_shared(options); } diff --git a/cpp/src/db/DBFactory.h b/cpp/src/db/DBFactory.h index bb363118ed..1223a11c29 100644 --- a/cpp/src/db/DBFactory.h +++ b/cpp/src/db/DBFactory.h @@ -28,13 +28,12 @@ namespace milvus { namespace engine { class DBFactory { -public: + public: static DBOptions BuildOption(); - static DBPtr Build(const DBOptions& options); + static DBPtr Build(const DBOptions &options); }; - -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 4536e29e95..ab3f3104af 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "DBImpl.h" +#include "db/DBImpl.h" #include "cache/CpuCacheMgr.h" #include "cache/GpuCacheMgr.h" #include "engine/EngineFactory.h" @@ -36,6 +36,7 @@ #include #include #include +#include #include namespace zilliz { @@ -48,10 +49,9 @@ constexpr uint64_t METRIC_ACTION_INTERVAL = 1; constexpr uint64_t COMPACT_ACTION_INTERVAL = 1; constexpr uint64_t INDEX_ACTION_INTERVAL = 1; -} +} // namespace - -DBImpl::DBImpl(const DBOptions& options) +DBImpl::DBImpl(const DBOptions &options) : options_(options), shutting_down_(true), compact_thread_pool_(1, 1), @@ -68,8 +68,9 @@ DBImpl::~DBImpl() { /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //external api /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -Status DBImpl::Start() { - if (!shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::Start() { + if (!shutting_down_.load(std::memory_order_acquire)) { return Status::OK(); } @@ -85,8 +86,9 @@ Status DBImpl::Start() { return Status::OK(); } -Status DBImpl::Stop() { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::Stop() { + if (shutting_down_.load(std::memory_order_acquire)) { return Status::OK(); } @@ -106,12 +108,14 @@ Status DBImpl::Stop() { return Status::OK(); } -Status DBImpl::DropAll() { +Status +DBImpl::DropAll() { return meta_ptr_->DropAll(); } -Status DBImpl::CreateTable(meta::TableSchema& table_schema) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::CreateTable(meta::TableSchema &table_schema) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -120,8 +124,9 @@ Status DBImpl::CreateTable(meta::TableSchema& table_schema) { return meta_ptr_->CreateTable(temp_schema); } -Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::DeleteTable(const std::string &table_id, const meta::DatesT &dates) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -144,8 +149,9 @@ Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& date return Status::OK(); } -Status DBImpl::DescribeTable(meta::TableSchema& table_schema) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::DescribeTable(meta::TableSchema &table_schema) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -154,24 +160,27 @@ Status DBImpl::DescribeTable(meta::TableSchema& table_schema) { return stat; } -Status DBImpl::HasTable(const std::string& table_id, bool& has_or_not) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::HasTable(const std::string &table_id, bool &has_or_not) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } return meta_ptr_->HasTable(table_id, has_or_not); } -Status DBImpl::AllTables(std::vector& table_schema_array) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::AllTables(std::vector &table_schema_array) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } return meta_ptr_->AllTables(table_schema_array); } -Status DBImpl::PreloadTable(const std::string &table_id) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::PreloadTable(const std::string &table_id) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -189,10 +198,14 @@ Status DBImpl::PreloadTable(const std::string &table_id) { int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); int64_t available_size = cache_total - cache_usage; - for(auto &day_files : files) { + for (auto &day_files : files) { for (auto &file : day_files.second) { - ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, (MetricType)file.metric_type_, file.nlist_); - if(engine == nullptr) { + ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, + file.location_, + (EngineType) file.engine_type_, + (MetricType) file.metric_type_, + file.nlist_); + if (engine == nullptr) { ENGINE_LOG_ERROR << "Invalid engine type"; return Status(DB_ERROR, "Invalid engine type"); } @@ -215,33 +228,37 @@ Status DBImpl::PreloadTable(const std::string &table_id) { return Status::OK(); } -Status DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } return meta_ptr_->UpdateTableFlag(table_id, flag); } -Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::GetTableRowCount(const std::string &table_id, uint64_t &row_count) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } return meta_ptr_->Count(table_id, row_count); } -Status DBImpl::InsertVectors(const std::string& table_id_, - uint64_t n, const float* vectors, IDNumbers& vector_ids_) { +Status +DBImpl::InsertVectors(const std::string &table_id_, + uint64_t n, const float *vectors, IDNumbers &vector_ids_) { // ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache"; - if (shutting_down_.load(std::memory_order_acquire)){ + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } Status status; zilliz::milvus::server::CollectInsertMetrics metrics(n, status); status = mem_mgr_->InsertVectors(table_id_, n, vectors, vector_ids_); -// std::chrono::microseconds time_span = std::chrono::duration_cast(end_time - start_time); +// std::chrono::microseconds time_span = +// std::chrono::duration_cast(end_time - start_time); // double average_time = double(time_span.count()) / n; // ENGINE_LOG_DEBUG << "Insert vectors to cache finished"; @@ -249,14 +266,15 @@ Status DBImpl::InsertVectors(const std::string& table_id_, return status; } -Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) { +Status +DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) { { std::unique_lock lock(build_index_mutex_); //step 1: check index difference TableIndex old_index; auto status = DescribeIndex(table_id, old_index); - if(!status.ok()) { + if (!status.ok()) { ENGINE_LOG_ERROR << "Failed to get table index info for table: " << table_id; return status; } @@ -264,7 +282,7 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) //step 2: update index info TableIndex new_index = index; new_index.metric_type_ = old_index.metric_type_;//dont change metric type, it was defined by CreateTable - if(!utils::IsSameIndex(old_index, new_index)) { + if (!utils::IsSameIndex(old_index, new_index)) { DropIndex(table_id); status = meta_ptr_->UpdateTableIndex(table_id, new_index); @@ -283,18 +301,18 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) //for IDMAP type, only wait all NEW file converted to RAW file //for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files std::vector file_types; - if(index.engine_type_ == (int)EngineType::FAISS_IDMAP) { + if (index.engine_type_ == (int) EngineType::FAISS_IDMAP) { file_types = { - (int) meta::TableFileSchema::NEW, - (int) meta::TableFileSchema::NEW_MERGE, + (int) meta::TableFileSchema::NEW, + (int) meta::TableFileSchema::NEW_MERGE, }; } else { file_types = { - (int) meta::TableFileSchema::RAW, - (int) meta::TableFileSchema::NEW, - (int) meta::TableFileSchema::NEW_MERGE, - (int) meta::TableFileSchema::NEW_INDEX, - (int) meta::TableFileSchema::TO_INDEX, + (int) meta::TableFileSchema::RAW, + (int) meta::TableFileSchema::NEW, + (int) meta::TableFileSchema::NEW_MERGE, + (int) meta::TableFileSchema::NEW_INDEX, + (int) meta::TableFileSchema::TO_INDEX, }; } @@ -304,11 +322,11 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) while (!file_ids.empty()) { ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times; - if(index.engine_type_ != (int)EngineType::FAISS_IDMAP) { + if (index.engine_type_ != (int) EngineType::FAISS_IDMAP) { status = meta_ptr_->UpdateTableFilesToIndex(table_id); } - std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10*1000, times*100))); + std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10 * 1000, times * 100))); status = meta_ptr_->FilesByType(table_id, file_types, file_ids); times++; } @@ -316,18 +334,21 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) return Status::OK(); } -Status DBImpl::DescribeIndex(const std::string& table_id, TableIndex& index) { +Status +DBImpl::DescribeIndex(const std::string &table_id, TableIndex &index) { return meta_ptr_->DescribeTableIndex(table_id, index); } -Status DBImpl::DropIndex(const std::string& table_id) { +Status +DBImpl::DropIndex(const std::string &table_id) { ENGINE_LOG_DEBUG << "Drop index for table: " << table_id; return meta_ptr_->DropTableIndex(table_id); } -Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, - const float *vectors, QueryResults &results) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, + const float *vectors, QueryResults &results) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -337,9 +358,10 @@ Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint6 return result; } -Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, - const float* vectors, const meta::DatesT& dates, QueryResults& results) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, + const float *vectors, const meta::DatesT &dates, QueryResults &results) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -364,10 +386,11 @@ Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint6 return status; } -Status DBImpl::Query(const std::string& table_id, const std::vector& file_ids, - uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, - const meta::DatesT& dates, QueryResults& results) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::Query(const std::string &table_id, const std::vector &file_ids, + uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors, + const meta::DatesT &dates, QueryResults &results) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -395,7 +418,7 @@ Status DBImpl::Query(const std::string& table_id, const std::vector } } - if(file_id_array.empty()) { + if (file_id_array.empty()) { return Status(DB_ERROR, "Invalid file id"); } @@ -405,36 +428,37 @@ Status DBImpl::Query(const std::string& table_id, const std::vector return status; } -Status DBImpl::Size(uint64_t& result) { - if (shutting_down_.load(std::memory_order_acquire)){ +Status +DBImpl::Size(uint64_t &result) { + if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } - return meta_ptr_->Size(result); + return meta_ptr_->Size(result); } - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //internal methods /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, - uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, - const meta::DatesT& dates, QueryResults& results) { - using namespace scheduler; +Status +DBImpl::QueryAsync(const std::string &table_id, const meta::TableFilesSchema &files, + uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors, + const meta::DatesT &dates, QueryResults &results) { server::CollectQueryMetrics metrics(nq); TimeRecorder rc(""); //step 1: get files to search - ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size() << " date range count: " << dates.size(); - SearchJobPtr job = std::make_shared(0, k, nq, nprobe, vectors); - for (auto &file : files) { - TableFileSchemaPtr file_ptr = std::make_shared(file); + ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size() << " date range count: " + << dates.size(); + scheduler::SearchJobPtr job = std::make_shared(0, k, nq, nprobe, vectors); + for (auto &file : files) { + scheduler::TableFileSchemaPtr file_ptr = std::make_shared(file); job->AddIndexFile(file_ptr); } //step 2: put search task to scheduler - JobMgrInst::GetInstance()->Put(job); + scheduler::JobMgrInst::GetInstance()->Put(job); job->WaitResult(); if (!job->GetStatus().ok()) { return job->GetStatus(); @@ -453,9 +477,12 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch // double search_percent = search_cost/total_cost; // double reduce_percent = reduce_cost/total_cost; // -// ENGINE_LOG_DEBUG << "Engine load index totally cost: " << load_info << " percent: " << load_percent*100 << "%"; -// ENGINE_LOG_DEBUG << "Engine search index totally cost: " << search_info << " percent: " << search_percent*100 << "%"; -// ENGINE_LOG_DEBUG << "Engine reduce topk totally cost: " << reduce_info << " percent: " << reduce_percent*100 << "%"; +// ENGINE_LOG_DEBUG << "Engine load index totally cost: " << load_info +// << " percent: " << load_percent*100 << "%"; +// ENGINE_LOG_DEBUG << "Engine search index totally cost: " << search_info +// << " percent: " << search_percent*100 << "%"; +// ENGINE_LOG_DEBUG << "Engine reduce topk totally cost: " << reduce_info +// << " percent: " << reduce_percent*100 << "%"; // } else { // ENGINE_LOG_DEBUG << "Engine load cost: " << load_info // << " search cost: " << search_info @@ -469,11 +496,12 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch return Status::OK(); } -void DBImpl::BackgroundTimerTask() { +void +DBImpl::BackgroundTimerTask() { Status status; server::SystemInfo::GetInstance().Init(); while (true) { - if (shutting_down_.load(std::memory_order_acquire)){ + if (shutting_down_.load(std::memory_order_acquire)) { WaitMergeFileFinish(); WaitBuildIndexFinish(); @@ -489,24 +517,27 @@ void DBImpl::BackgroundTimerTask() { } } -void DBImpl::WaitMergeFileFinish() { +void +DBImpl::WaitMergeFileFinish() { std::lock_guard lck(compact_result_mutex_); - for(auto& iter : compact_thread_results_) { + for (auto &iter : compact_thread_results_) { iter.wait(); } } -void DBImpl::WaitBuildIndexFinish() { +void +DBImpl::WaitBuildIndexFinish() { std::lock_guard lck(index_result_mutex_); - for(auto& iter : index_thread_results_) { + for (auto &iter : index_thread_results_) { iter.wait(); } } -void DBImpl::StartMetricTask() { +void +DBImpl::StartMetricTask() { static uint64_t metric_clock_tick = 0; metric_clock_tick++; - if(metric_clock_tick%METRIC_ACTION_INTERVAL != 0) { + if (metric_clock_tick % METRIC_ACTION_INTERVAL != 0) { return; } @@ -515,7 +546,7 @@ void DBImpl::StartMetricTask() { server::Metrics::GetInstance().KeepingAliveCounterIncrement(METRIC_ACTION_INTERVAL); int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); int64_t cache_total = cache::CpuCacheMgr::GetInstance()->CacheCapacity(); - server::Metrics::GetInstance().CpuCacheUsageGaugeSet(cache_usage*100/cache_total); + server::Metrics::GetInstance().CpuCacheUsageGaugeSet(cache_usage * 100 / cache_total); server::Metrics::GetInstance().GpuCacheUsageGaugeSet(); uint64_t size; Size(size); @@ -533,25 +564,27 @@ void DBImpl::StartMetricTask() { ENGINE_LOG_TRACE << "Metric task finished"; } -Status DBImpl::MemSerialize() { +Status +DBImpl::MemSerialize() { std::lock_guard lck(mem_serialize_mutex_); std::set temp_table_ids; mem_mgr_->Serialize(temp_table_ids); - for(auto& id : temp_table_ids) { + for (auto &id : temp_table_ids) { compact_table_ids_.insert(id); } - if(!temp_table_ids.empty()) { + if (!temp_table_ids.empty()) { SERVER_LOG_DEBUG << "Insert cache serialized"; } return Status::OK(); } -void DBImpl::StartCompactionTask() { +void +DBImpl::StartCompactionTask() { static uint64_t compact_clock_tick = 0; compact_clock_tick++; - if(compact_clock_tick%COMPACT_ACTION_INTERVAL != 0) { + if (compact_clock_tick % COMPACT_ACTION_INTERVAL != 0) { return; } @@ -580,8 +613,9 @@ void DBImpl::StartCompactionTask() { } } -Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, - const meta::TableFilesSchema& files) { +Status +DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date, + const meta::TableFilesSchema &files) { ENGINE_LOG_DEBUG << "Merge files for table: " << table_id; //step 1: create table file @@ -598,13 +632,13 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, //step 2: merge files ExecutionEnginePtr index = - EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType)table_file.engine_type_, - (MetricType)table_file.metric_type_, table_file.nlist_); + EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType) table_file.engine_type_, + (MetricType) table_file.metric_type_, table_file.nlist_); meta::TableFilesSchema updated; - long index_size = 0; + int64_t index_size = 0; - for (auto& file : files) { + for (auto &file : files) { server::CollectMergeFilesMetrics metrics; index->Merge(file.location_); @@ -620,7 +654,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, //step 3: serialize to disk try { index->Serialize(); - } catch (std::exception& ex) { + } catch (std::exception &ex) { //typical error: out of disk space or permition denied std::string msg = "Serialize merged index encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; @@ -638,7 +672,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, //step 4: update table files state //if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size //else set file type to RAW, no need to build index - if (table_file.engine_type_ != (int)EngineType::FAISS_IDMAP) { + if (table_file.engine_type_ != (int) EngineType::FAISS_IDMAP) { table_file.file_type_ = (index->PhysicalSize() >= table_file.index_file_size_) ? meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; } else { @@ -649,16 +683,17 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, updated.push_back(table_file); status = meta_ptr_->UpdateTableFiles(updated); ENGINE_LOG_DEBUG << "New merged file " << table_file.file_id_ << - " of size " << index->PhysicalSize() << " bytes"; + " of size " << index->PhysicalSize() << " bytes"; - if(options_.insert_cache_immediately_) { + if (options_.insert_cache_immediately_) { index->Cache(); } return status; } -Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { +Status +DBImpl::BackgroundMergeFiles(const std::string &table_id) { meta::DatePartionedTableFilesSchema raw_files; auto status = meta_ptr_->FilesToMerge(table_id, raw_files); if (!status.ok()) { @@ -667,7 +702,7 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { } bool has_merge = false; - for (auto& kv : raw_files) { + for (auto &kv : raw_files) { auto files = kv.second; if (files.size() < options_.merge_trigger_number_) { ENGINE_LOG_DEBUG << "Files number not greater equal than merge trigger number, skip merge action"; @@ -676,7 +711,7 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { has_merge = true; MergeFiles(table_id, kv.first, kv.second); - if (shutting_down_.load(std::memory_order_acquire)){ + if (shutting_down_.load(std::memory_order_acquire)) { ENGINE_LOG_DEBUG << "Server will shutdown, skip merge action for table: " << table_id; break; } @@ -685,17 +720,18 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { return Status::OK(); } -void DBImpl::BackgroundCompaction(std::set table_ids) { +void +DBImpl::BackgroundCompaction(std::set table_ids) { ENGINE_LOG_TRACE << " Background compaction thread start"; Status status; - for (auto& table_id : table_ids) { + for (auto &table_id : table_ids) { status = BackgroundMergeFiles(table_id); if (!status.ok()) { ENGINE_LOG_ERROR << "Merge files for table " << table_id << " failed: " << status.ToString(); } - if (shutting_down_.load(std::memory_order_acquire)){ + if (shutting_down_.load(std::memory_order_acquire)) { ENGINE_LOG_DEBUG << "Server will shutdown, skip merge action"; break; } @@ -703,7 +739,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); - int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes + int ttl = 5 * meta::M_SEC;//default: file will be deleted after 5 minutes if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { ttl = meta::D_SEC; } @@ -712,10 +748,11 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { ENGINE_LOG_TRACE << " Background compaction thread exit"; } -void DBImpl::StartBuildIndexTask(bool force) { +void +DBImpl::StartBuildIndexTask(bool force) { static uint64_t index_clock_tick = 0; index_clock_tick++; - if(!force && (index_clock_tick%INDEX_ACTION_INTERVAL != 0)) { + if (!force && (index_clock_tick % INDEX_ACTION_INTERVAL != 0)) { return; } @@ -740,11 +777,12 @@ void DBImpl::StartBuildIndexTask(bool force) { } } -Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { +Status +DBImpl::BuildIndex(const meta::TableFileSchema &file) { ExecutionEnginePtr to_index = - EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, - (MetricType)file.metric_type_, file.nlist_); - if(to_index == nullptr) { + EngineFactory::Build(file.dimension_, file.location_, (EngineType) file.engine_type_, + (MetricType) file.metric_type_, file.nlist_); + if (to_index == nullptr) { ENGINE_LOG_ERROR << "Invalid engine type"; return Status(DB_ERROR, "Invalid engine type"); } @@ -761,7 +799,8 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { meta::TableFileSchema table_file; table_file.table_id_ = file.table_id_; table_file.date_ = file.date_; - table_file.file_type_ = meta::TableFileSchema::NEW_INDEX; //for multi-db-path, distribute index file averagely to each path + table_file.file_type_ = + meta::TableFileSchema::NEW_INDEX; //for multi-db-path, distribute index file averagely to each path status = meta_ptr_->CreateTableFile(table_file); if (!status.ok()) { ENGINE_LOG_ERROR << "Failed to create table file: " << status.ToString(); @@ -773,16 +812,16 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { try { server::CollectBuildIndexMetrics metrics; - index = to_index->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_); + index = to_index->BuildIndex(table_file.location_, (EngineType) table_file.engine_type_); if (index == nullptr) { table_file.file_type_ = meta::TableFileSchema::TO_DELETE; status = meta_ptr_->UpdateTableFile(table_file); - ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; + ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ + << " to to_delete"; return status; } - - } catch (std::exception& ex) { + } catch (std::exception &ex) { //typical error: out of gpu memory std::string msg = "BuildIndex encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; @@ -791,7 +830,8 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { status = meta_ptr_->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; - std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough" << std::endl; + std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough" + << std::endl; return Status(DB_ERROR, msg); } @@ -799,7 +839,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { //step 4: if table has been deleted, dont save index file bool has_table = false; meta_ptr_->HasTable(file.table_id_, has_table); - if(!has_table) { + if (!has_table) { meta_ptr_->DeleteTableFiles(file.table_id_); return Status::OK(); } @@ -807,7 +847,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { //step 5: save index file try { index->Serialize(); - } catch (std::exception& ex) { + } catch (std::exception &ex) { //typical error: out of disk space or permition denied std::string msg = "Serialize index encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; @@ -817,7 +857,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; std::cout << "ERROR: failed to persist index file: " << table_file.location_ - << ", possible out of disk space" << std::endl; + << ", possible out of disk space" << std::endl; return Status(DB_ERROR, msg); } @@ -832,12 +872,12 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { meta::TableFilesSchema update_files = {table_file, origin_file}; status = meta_ptr_->UpdateTableFiles(update_files); - if(status.ok()) { + if (status.ok()) { ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size " << index->PhysicalSize() << " bytes" << " from file " << origin_file.file_id_; - if(options_.insert_cache_immediately_) { + if (options_.insert_cache_immediately_) { index->Cache(); } } else { @@ -850,8 +890,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { status = meta_ptr_->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; } - - } catch (std::exception& ex) { + } catch (std::exception &ex) { std::string msg = "Build index encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; return Status(DB_ERROR, msg); @@ -860,20 +899,21 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { return Status::OK(); } -void DBImpl::BackgroundBuildIndex() { +void +DBImpl::BackgroundBuildIndex() { ENGINE_LOG_TRACE << "Background build index thread start"; std::unique_lock lock(build_index_mutex_); meta::TableFilesSchema to_index_files; meta_ptr_->FilesToIndex(to_index_files); Status status; - for (auto& file : to_index_files) { + for (auto &file : to_index_files) { status = BuildIndex(file); if (!status.ok()) { ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString(); } - if (shutting_down_.load(std::memory_order_acquire)){ + if (shutting_down_.load(std::memory_order_acquire)) { ENGINE_LOG_DEBUG << "Server will shutdown, skip build index action"; break; } diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index bc9e453e5c..344e290445 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -29,7 +29,8 @@ #include #include #include - +#include +#include namespace zilliz { namespace milvus { @@ -68,11 +69,11 @@ class DBImpl : public DB { Status InsertVectors(const std::string &table_id, uint64_t n, const float *vectors, IDNumbers &vector_ids) override; - Status CreateIndex(const std::string& table_id, const TableIndex& index) override; + Status CreateIndex(const std::string &table_id, const TableIndex &index) override; - Status DescribeIndex(const std::string& table_id, TableIndex& index) override; + Status DescribeIndex(const std::string &table_id, TableIndex &index) override; - Status DropIndex(const std::string& table_id) override; + Status DropIndex(const std::string &table_id) override; Status Query(const std::string &table_id, uint64_t k, @@ -123,7 +124,7 @@ class DBImpl : public DB { Status BackgroundMergeFiles(const std::string &table_id); void BackgroundCompaction(std::set table_ids); - void StartBuildIndexTask(bool force=false); + void StartBuildIndexTask(bool force = false); void BackgroundBuildIndex(); Status BuildIndex(const meta::TableFileSchema &); @@ -151,7 +152,6 @@ class DBImpl : public DB { std::list> index_thread_results_; std::mutex build_index_mutex_; - }; // DBImpl diff --git a/cpp/src/db/IDGenerator.cpp b/cpp/src/db/IDGenerator.cpp index a82ccee629..78c88b2c03 100644 --- a/cpp/src/db/IDGenerator.cpp +++ b/cpp/src/db/IDGenerator.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "IDGenerator.h" +#include "db/IDGenerator.h" #include #include @@ -29,16 +29,18 @@ IDGenerator::~IDGenerator() = default; constexpr size_t SimpleIDGenerator::MAX_IDS_PER_MICRO; -IDNumber SimpleIDGenerator::GetNextIDNumber() { +IDNumber +SimpleIDGenerator::GetNextIDNumber() { auto now = std::chrono::system_clock::now(); auto micros = std::chrono::duration_cast( - now.time_since_epoch()).count(); + now.time_since_epoch()).count(); return micros * MAX_IDS_PER_MICRO; } -void SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers& ids) { +void +SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers &ids) { if (n > MAX_IDS_PER_MICRO) { - NextIDNumbers(n-MAX_IDS_PER_MICRO, ids); + NextIDNumbers(n - MAX_IDS_PER_MICRO, ids); NextIDNumbers(MAX_IDS_PER_MICRO, ids); return; } @@ -48,20 +50,20 @@ void SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers& ids) { auto now = std::chrono::system_clock::now(); auto micros = std::chrono::duration_cast( - now.time_since_epoch()).count(); + now.time_since_epoch()).count(); micros *= MAX_IDS_PER_MICRO; - for (int pos=0; pos #include - namespace zilliz { namespace milvus { namespace engine { @@ -55,7 +54,6 @@ class SimpleIDGenerator : public IDGenerator { NextIDNumbers(size_t n, IDNumbers &ids); static constexpr size_t MAX_IDS_PER_MICRO = 1000; - }; // SimpleIDGenerator diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index fd52b851e2..4c82f3f736 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "Options.h" +#include "db/Options.h" #include "utils/Exception.h" #include "utils/Log.h" @@ -27,18 +27,20 @@ namespace zilliz { namespace milvus { namespace engine { -ArchiveConf::ArchiveConf(const std::string& type, const std::string& criterias) { +ArchiveConf::ArchiveConf(const std::string &type, const std::string &criterias) { ParseType(type); ParseCritirias(criterias); } -void ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT& criterial) { - for(auto& pair : criterial) { +void +ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT &criterial) { + for (auto &pair : criterial) { criterias_[pair.first] = pair.second; } } -void ArchiveConf::ParseCritirias(const std::string& criterias) { +void +ArchiveConf::ParseCritirias(const std::string &criterias) { std::stringstream ss(criterias); std::vector tokens; @@ -48,8 +50,8 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) { return; } - for (auto& token : tokens) { - if(token.empty()) { + for (auto &token : tokens) { + if (token.empty()) { continue; } @@ -67,12 +69,12 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) { auto value = std::stoi(kv[1]); criterias_[kv[0]] = value; } - catch (std::out_of_range&){ + catch (std::out_of_range &) { std::string msg = "Out of range: '" + kv[1] + "'"; ENGINE_LOG_ERROR << msg; throw InvalidArgumentException(msg); } - catch (...){ + catch (...) { std::string msg = "Invalid argument: '" + kv[1] + "'"; ENGINE_LOG_ERROR << msg; throw InvalidArgumentException(msg); @@ -80,7 +82,8 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) { } } -void ArchiveConf::ParseType(const std::string& type) { +void +ArchiveConf::ParseType(const std::string &type) { if (type != "delete" && type != "swap") { std::string msg = "Invalid argument: type='" + type + "'"; throw InvalidArgumentException(msg); diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index dbaf34b065..d8be2767df 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -30,22 +30,27 @@ namespace engine { class Env; -static const char* ARCHIVE_CONF_DISK = "disk"; -static const char* ARCHIVE_CONF_DAYS = "days"; +static const char *ARCHIVE_CONF_DISK = "disk"; +static const char *ARCHIVE_CONF_DAYS = "days"; struct ArchiveConf { using CriteriaT = std::map; - ArchiveConf(const std::string& type, const std::string& criterias = std::string()); + explicit ArchiveConf(const std::string &type, const std::string &criterias = std::string()); - const std::string& GetType() const { return type_; } - const CriteriaT GetCriterias() const { return criterias_; } + const std::string &GetType() const { + return type_; + } - void SetCriterias(const ArchiveConf::CriteriaT& criterial); + const CriteriaT GetCriterias() const { + return criterias_; + } -private: - void ParseCritirias(const std::string& type); - void ParseType(const std::string& criterias); + void SetCriterias(const ArchiveConf::CriteriaT &criterial); + + private: + void ParseCritirias(const std::string &type); + void ParseType(const std::string &criterias); std::string type_; CriteriaT criterias_; @@ -65,7 +70,7 @@ struct DBOptions { CLUSTER_WRITABLE } MODE; - uint16_t merge_trigger_number_ = 2; + uint16_t merge_trigger_number_ = 2; DBMetaOptions meta_; int mode_ = MODE::SINGLE; diff --git a/cpp/src/db/Types.h b/cpp/src/db/Types.h index d927ecab5f..04bf680937 100644 --- a/cpp/src/db/Types.h +++ b/cpp/src/db/Types.h @@ -21,22 +21,23 @@ #include #include +#include namespace zilliz { namespace milvus { namespace engine { -typedef long IDNumber; -typedef IDNumber* IDNumberPtr; +typedef int64_t IDNumber; +typedef IDNumber *IDNumberPtr; typedef std::vector IDNumbers; typedef std::vector> QueryResult; typedef std::vector QueryResults; struct TableIndex { - int32_t engine_type_ = (int)EngineType::FAISS_IDMAP; + int32_t engine_type_ = (int) EngineType::FAISS_IDMAP; int32_t nlist_ = 16384; - int32_t metric_type_ = (int)MetricType::L2; + int32_t metric_type_ = (int) MetricType::L2; }; } // namespace engine diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index cf61bf6b3d..b3d72202cc 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -15,13 +15,14 @@ // specific language governing permissions and limitations // under the License. -#include "Utils.h" +#include "db/Utils.h" #include "utils/CommonUtil.h" #include "utils/Log.h" #include #include #include +#include #include namespace zilliz { @@ -31,23 +32,25 @@ namespace utils { namespace { -const char* TABLES_FOLDER = "/tables/"; +const char *TABLES_FOLDER = "/tables/"; uint64_t index_file_counter = 0; std::mutex index_file_counter_mutex; -std::string ConstructParentFolder(const std::string& db_path, const meta::TableFileSchema& table_file) { +std::string +ConstructParentFolder(const std::string &db_path, const meta::TableFileSchema &table_file) { std::string table_path = db_path + TABLES_FOLDER + table_file.table_id_; std::string partition_path = table_path + "/" + std::to_string(table_file.date_); return partition_path; } -std::string GetTableFileParentFolder(const DBMetaOptions& options, const meta::TableFileSchema& table_file) { +std::string +GetTableFileParentFolder(const DBMetaOptions &options, const meta::TableFileSchema &table_file) { uint64_t path_count = options.slave_paths_.size() + 1; std::string target_path = options.path_; uint64_t index = 0; - if(meta::TableFileSchema::NEW_INDEX == table_file.file_type_) { + if (meta::TableFileSchema::NEW_INDEX == table_file.file_type_) { // index file is large file and to be persisted permanently // we need to distribute index files to each db_path averagely // round robin according to a file counter @@ -67,17 +70,19 @@ std::string GetTableFileParentFolder(const DBMetaOptions& options, const meta::T return ConstructParentFolder(target_path, table_file); } -} +} // namespace -long GetMicroSecTimeStamp() { +int64_t +GetMicroSecTimeStamp() { auto now = std::chrono::system_clock::now(); auto micros = std::chrono::duration_cast( - now.time_since_epoch()).count(); + now.time_since_epoch()).count(); return micros; } -Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id) { +Status +CreateTablePath(const DBMetaOptions &options, const std::string &table_id) { std::string db_path = options.path_; std::string table_path = db_path + TABLES_FOLDER + table_id; auto status = server::CommonUtil::CreateDirectory(table_path); @@ -86,7 +91,7 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id return status; } - for(auto& path : options.slave_paths_) { + for (auto &path : options.slave_paths_) { table_path = path + TABLES_FOLDER + table_id; status = server::CommonUtil::CreateDirectory(table_path); if (!status.ok()) { @@ -98,17 +103,18 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id return Status::OK(); } -Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force) { +Status +DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool force) { std::vector paths = options.slave_paths_; paths.push_back(options.path_); - for(auto& path : paths) { + for (auto &path : paths) { std::string table_path = path + TABLES_FOLDER + table_id; - if(force) { + if (force) { boost::filesystem::remove_all(table_path); ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; - } else if(boost::filesystem::exists(table_path) && - boost::filesystem::is_empty(table_path)) { + } else if (boost::filesystem::exists(table_path) && + boost::filesystem::is_empty(table_path)) { boost::filesystem::remove_all(table_path); ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; } @@ -117,7 +123,8 @@ Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id return Status::OK(); } -Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { +Status +CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) { std::string parent_path = GetTableFileParentFolder(options, table_file); auto status = server::CommonUtil::CreateDirectory(parent_path); @@ -131,17 +138,18 @@ Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& return Status::OK(); } -Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { +Status +GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) { std::string parent_path = ConstructParentFolder(options.path_, table_file); std::string file_path = parent_path + "/" + table_file.file_id_; - if(boost::filesystem::exists(file_path)) { + if (boost::filesystem::exists(file_path)) { table_file.location_ = file_path; return Status::OK(); } else { - for(auto& path : options.slave_paths_) { + for (auto &path : options.slave_paths_) { parent_path = ConstructParentFolder(path, table_file); file_path = parent_path + "/" + table_file.file_id_; - if(boost::filesystem::exists(file_path)) { + if (boost::filesystem::exists(file_path)) { table_file.location_ = file_path; return Status::OK(); } @@ -155,49 +163,55 @@ Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& tab return Status(DB_ERROR, msg); } -Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { +Status +DeleteTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) { utils::GetTableFilePath(options, table_file); boost::filesystem::remove(table_file.location_); return Status::OK(); } -bool IsSameIndex(const TableIndex& index1, const TableIndex& index2) { +bool +IsSameIndex(const TableIndex &index1, const TableIndex &index2) { return index1.engine_type_ == index2.engine_type_ && index1.nlist_ == index2.nlist_ && index1.metric_type_ == index2.metric_type_; } -meta::DateT GetDate(const std::time_t& t, int day_delta) { +meta::DateT +GetDate(const std::time_t &t, int day_delta) { struct tm ltm; localtime_r(&t, <m); if (day_delta > 0) { do { ++ltm.tm_mday; --day_delta; - } while(day_delta > 0); + } while (day_delta > 0); mktime(<m); } else if (day_delta < 0) { do { --ltm.tm_mday; ++day_delta; - } while(day_delta < 0); + } while (day_delta < 0); mktime(<m); } else { ltm.tm_mday; } - return ltm.tm_year*10000 + ltm.tm_mon*100 + ltm.tm_mday; + return ltm.tm_year * 10000 + ltm.tm_mon * 100 + ltm.tm_mday; } -meta::DateT GetDateWithDelta(int day_delta) { +meta::DateT +GetDateWithDelta(int day_delta) { return GetDate(std::time(nullptr), day_delta); } -meta::DateT GetDate() { +meta::DateT +GetDate() { return GetDate(std::time(nullptr), 0); } // URI format: dialect://username:password@host:port/database -Status ParseMetaUri(const std::string& uri, MetaUriInfo& info) { +Status +ParseMetaUri(const std::string &uri, MetaUriInfo &info) { std::string dialect_regex = "(.*)"; std::string username_tegex = "(.*)"; std::string password_regex = "(.*)"; @@ -205,7 +219,7 @@ Status ParseMetaUri(const std::string& uri, MetaUriInfo& info) { std::string port_regex = "(.*)"; std::string db_name_regex = "(.*)"; std::string uri_regex_str = - dialect_regex + "\\:\\/\\/" + + dialect_regex + "\\:\\/\\/" + username_tegex + "\\:" + password_regex + "\\@" + host_regex + "\\:" + diff --git a/cpp/src/db/Utils.h b/cpp/src/db/Utils.h index 39bdfbfbc1..fbeca84686 100644 --- a/cpp/src/db/Utils.h +++ b/cpp/src/db/Utils.h @@ -29,20 +29,30 @@ namespace milvus { namespace engine { namespace utils { -long GetMicroSecTimeStamp(); +int64_t +GetMicroSecTimeStamp(); -Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id); -Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force = true); +Status +CreateTablePath(const DBMetaOptions &options, const std::string &table_id); +Status +DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool force = true); -Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); -Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); -Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); +Status +CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file); +Status +GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file); +Status +DeleteTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file); -bool IsSameIndex(const TableIndex& index1, const TableIndex& index2); +bool +IsSameIndex(const TableIndex &index1, const TableIndex &index2); -meta::DateT GetDate(const std::time_t &t, int day_delta = 0); -meta::DateT GetDate(); -meta::DateT GetDateWithDelta(int day_delta); +meta::DateT +GetDate(const std::time_t &t, int day_delta = 0); +meta::DateT +GetDate(); +meta::DateT +GetDateWithDelta(int day_delta); struct MetaUriInfo { std::string dialect_; @@ -53,7 +63,8 @@ struct MetaUriInfo { std::string db_name_; }; -Status ParseMetaUri(const std::string& uri, MetaUriInfo& info); +Status +ParseMetaUri(const std::string &uri, MetaUriInfo &info); } // namespace utils } // namespace engine diff --git a/cpp/src/db/engine/EngineFactory.cpp b/cpp/src/db/engine/EngineFactory.cpp index 28cfa97511..043744f803 100644 --- a/cpp/src/db/engine/EngineFactory.cpp +++ b/cpp/src/db/engine/EngineFactory.cpp @@ -15,10 +15,12 @@ // specific language governing permissions and limitations // under the License. -#include "EngineFactory.h" -#include "ExecutionEngineImpl.h" +#include "db/engine/EngineFactory.h" +#include "db/engine/ExecutionEngineImpl.h" #include "utils/Log.h" +#include + namespace zilliz { namespace milvus { namespace engine { @@ -29,20 +31,19 @@ EngineFactory::Build(uint16_t dimension, EngineType index_type, MetricType metric_type, int32_t nlist) { - - if(index_type == EngineType::INVALID) { + if (index_type == EngineType::INVALID) { ENGINE_LOG_ERROR << "Unsupported engine type"; return nullptr; } - ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int)index_type; + ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int) index_type; ExecutionEnginePtr execution_engine_ptr = - std::make_shared(dimension, location, index_type, metric_type, nlist); + std::make_shared(dimension, location, index_type, metric_type, nlist); execution_engine_ptr->Init(); return execution_engine_ptr; } -} -} -} \ No newline at end of file +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/engine/EngineFactory.h b/cpp/src/db/engine/EngineFactory.h index c3fb41adde..105e48f885 100644 --- a/cpp/src/db/engine/EngineFactory.h +++ b/cpp/src/db/engine/EngineFactory.h @@ -21,19 +21,22 @@ #include "ExecutionEngine.h" #include "utils/Status.h" +#include + namespace zilliz { namespace milvus { namespace engine { class EngineFactory { -public: + public: static ExecutionEnginePtr Build(uint16_t dimension, - const std::string& location, + const std::string &location, EngineType index_type, MetricType metric_type, int32_t nlist); }; -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz + diff --git a/cpp/src/db/engine/ExecutionEngine.h b/cpp/src/db/engine/ExecutionEngine.h index 5868f15d33..c63760686f 100644 --- a/cpp/src/db/engine/ExecutionEngine.h +++ b/cpp/src/db/engine/ExecutionEngine.h @@ -21,6 +21,7 @@ #include #include +#include namespace zilliz { namespace milvus { @@ -41,9 +42,8 @@ enum class MetricType { }; class ExecutionEngine { -public: - - virtual Status AddWithIds(long n, const float *xdata, const long *xids) = 0; + public: + virtual Status AddWithIds(int64_t n, const float *xdata, const int64_t *xids) = 0; virtual size_t Count() const = 0; @@ -63,16 +63,16 @@ public: virtual std::shared_ptr Clone() = 0; - virtual Status Merge(const std::string& location) = 0; + virtual Status Merge(const std::string &location) = 0; - virtual Status Search(long n, - const float *data, - long k, - long nprobe, - float *distances, - long *labels) const = 0; + virtual Status Search(int64_t n, + const float *data, + int64_t k, + int64_t nprobe, + float *distances, + int64_t *labels) const = 0; - virtual std::shared_ptr BuildIndex(const std::string& location, EngineType engine_type) = 0; + virtual std::shared_ptr BuildIndex(const std::string &location, EngineType engine_type) = 0; virtual Status Cache() = 0; @@ -89,7 +89,6 @@ public: using ExecutionEnginePtr = std::shared_ptr; - } // namespace engine } // namespace milvus } // namespace zilliz diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index b9d18b4a2f..75b8b5d865 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "ExecutionEngineImpl.h" +#include "db/engine/ExecutionEngineImpl.h" #include "cache/GpuCacheMgr.h" #include "cache/CpuCacheMgr.h" #include "metrics/Metrics.h" @@ -29,6 +29,7 @@ #include "server/Config.h" #include +#include namespace zilliz { namespace milvus { @@ -72,7 +73,8 @@ ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index, nlist_(nlist) { } -VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) { +VecIndexPtr +ExecutionEngineImpl::CreatetVecIndex(EngineType type) { std::shared_ptr index; switch (type) { case EngineType::FAISS_IDMAP: { @@ -99,41 +101,48 @@ VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) { return index; } -Status ExecutionEngineImpl::AddWithIds(long n, const float *xdata, const long *xids) { +Status +ExecutionEngineImpl::AddWithIds(int64_t n, const float *xdata, const int64_t *xids) { auto status = index_->Add(n, xdata, xids); return status; } -size_t ExecutionEngineImpl::Count() const { - if(index_ == nullptr) { +size_t +ExecutionEngineImpl::Count() const { + if (index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, return count 0"; return 0; } return index_->Count(); } -size_t ExecutionEngineImpl::Size() const { +size_t +ExecutionEngineImpl::Size() const { return (size_t) (Count() * Dimension()) * sizeof(float); } -size_t ExecutionEngineImpl::Dimension() const { - if(index_ == nullptr) { +size_t +ExecutionEngineImpl::Dimension() const { + if (index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, return dimension " << dim_; return dim_; } return index_->Dimension(); } -size_t ExecutionEngineImpl::PhysicalSize() const { +size_t +ExecutionEngineImpl::PhysicalSize() const { return server::CommonUtil::GetFileSize(location_); } -Status ExecutionEngineImpl::Serialize() { +Status +ExecutionEngineImpl::Serialize() { auto status = write_index(index_, location_); return status; } -Status ExecutionEngineImpl::Load(bool to_cache) { +Status +ExecutionEngineImpl::Load(bool to_cache) { index_ = cache::CpuCacheMgr::GetInstance()->GetIndex(location_); bool already_in_cache = (index_ != nullptr); if (!already_in_cache) { @@ -141,7 +150,7 @@ Status ExecutionEngineImpl::Load(bool to_cache) { double physical_size = PhysicalSize(); server::CollectExecutionEngineMetrics metrics(physical_size); index_ = read_index(location_); - if(index_ == nullptr) { + if (index_ == nullptr) { std::string msg = "Failed to load index from " + location_; ENGINE_LOG_ERROR << msg; return Status(DB_ERROR, msg); @@ -160,13 +169,14 @@ Status ExecutionEngineImpl::Load(bool to_cache) { return Status::OK(); } -Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) { +Status +ExecutionEngineImpl::CopyToGpu(uint64_t device_id) { auto index = cache::GpuCacheMgr::GetInstance(device_id)->GetIndex(location_); bool already_in_cache = (index != nullptr); if (already_in_cache) { index_ = index; } else { - if(index_ == nullptr) { + if (index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to gpu"; return Status(DB_ERROR, "index is null"); } @@ -187,13 +197,14 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) { return Status::OK(); } -Status ExecutionEngineImpl::CopyToCpu() { +Status +ExecutionEngineImpl::CopyToCpu() { auto index = cache::CpuCacheMgr::GetInstance()->GetIndex(location_); bool already_in_cache = (index != nullptr); if (already_in_cache) { index_ = index; } else { - if(index_ == nullptr) { + if (index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to cpu"; return Status(DB_ERROR, "index is null"); } @@ -213,8 +224,9 @@ Status ExecutionEngineImpl::CopyToCpu() { return Status::OK(); } -ExecutionEnginePtr ExecutionEngineImpl::Clone() { - if(index_ == nullptr) { +ExecutionEnginePtr +ExecutionEngineImpl::Clone() { + if (index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to clone"; return nullptr; } @@ -225,7 +237,8 @@ ExecutionEnginePtr ExecutionEngineImpl::Clone() { return ret; } -Status ExecutionEngineImpl::Merge(const std::string &location) { +Status +ExecutionEngineImpl::Merge(const std::string &location) { if (location == location_) { return Status(DB_ERROR, "Cannot Merge Self"); } @@ -243,7 +256,7 @@ Status ExecutionEngineImpl::Merge(const std::string &location) { } } - if(index_ == nullptr) { + if (index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to merge"; return Status(DB_ERROR, "index is null"); } @@ -264,7 +277,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_; auto from_index = std::dynamic_pointer_cast(index_); - if(from_index == nullptr) { + if (from_index == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: from_index is null, failed to build index"; return nullptr; } @@ -282,21 +295,22 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t AutoGenParams(to_index->GetType(), Count(), build_cfg); auto status = to_index->BuildAll(Count(), - from_index->GetRawVectors(), - from_index->GetRawIds(), - build_cfg); + from_index->GetRawVectors(), + from_index->GetRawIds(), + build_cfg); if (!status.ok()) { throw Exception(DB_ERROR, status.message()); } return std::make_shared(to_index, location, engine_type, metric_type_, nlist_); } -Status ExecutionEngineImpl::Search(long n, - const float *data, - long k, - long nprobe, - float *distances, - long *labels) const { - if(index_ == nullptr) { +Status +ExecutionEngineImpl::Search(int64_t n, + const float *data, + int64_t k, + int64_t nprobe, + float *distances, + int64_t *labels) const { + if (index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search"; return Status(DB_ERROR, "index is null"); } @@ -310,14 +324,16 @@ Status ExecutionEngineImpl::Search(long n, return status; } -Status ExecutionEngineImpl::Cache() { +Status +ExecutionEngineImpl::Cache() { cache::DataObjPtr obj = std::make_shared(index_, PhysicalSize()); zilliz::milvus::cache::CpuCacheMgr::GetInstance()->InsertItem(location_, obj); return Status::OK(); } -Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { +Status +ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { cache::DataObjPtr obj = std::make_shared(index_, PhysicalSize()); zilliz::milvus::cache::GpuCacheMgr::GetInstance(gpu_id)->InsertItem(location_, obj); @@ -325,8 +341,8 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { } // TODO(linxj): remove. -Status ExecutionEngineImpl::Init() { - using namespace zilliz::milvus::server; +Status +ExecutionEngineImpl::Init() { server::Config &config = server::Config::GetInstance(); Status s = config.GetDBConfigBuildIndexGPU(gpu_num_); if (!s.ok()) return s; @@ -334,7 +350,6 @@ Status ExecutionEngineImpl::Init() { return Status::OK(); } - } // namespace engine } // namespace milvus } // namespace zilliz diff --git a/cpp/src/db/engine/ExecutionEngineImpl.h b/cpp/src/db/engine/ExecutionEngineImpl.h index 1a3c6c83be..6ea09ddb29 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.h +++ b/cpp/src/db/engine/ExecutionEngineImpl.h @@ -23,15 +23,12 @@ #include #include - namespace zilliz { namespace milvus { namespace engine { - class ExecutionEngineImpl : public ExecutionEngine { -public: - + public: ExecutionEngineImpl(uint16_t dimension, const std::string &location, EngineType index_type, @@ -44,7 +41,7 @@ public: MetricType metric_type, int32_t nlist); - Status AddWithIds(long n, const float *xdata, const long *xids) override; + Status AddWithIds(int64_t n, const float *xdata, const int64_t *xids) override; size_t Count() const override; @@ -66,12 +63,12 @@ public: Status Merge(const std::string &location) override; - Status Search(long n, + Status Search(int64_t n, const float *data, - long k, - long nprobe, + int64_t k, + int64_t nprobe, float *distances, - long *labels) const override; + int64_t *labels) const override; ExecutionEnginePtr BuildIndex(const std::string &location, EngineType engine_type) override; @@ -81,18 +78,24 @@ public: Status Init() override; - EngineType IndexEngineType() const override { return index_type_; } + EngineType IndexEngineType() const override { + return index_type_; + } - MetricType IndexMetricType() const override { return metric_type_; } + MetricType IndexMetricType() const override { + return metric_type_; + } - std::string GetLocation() const override { return location_; } + std::string GetLocation() const override { + return location_; + } -private: + private: VecIndexPtr CreatetVecIndex(EngineType type); VecIndexPtr Load(const std::string &location); -protected: + protected: VecIndexPtr index_ = nullptr; EngineType index_type_; MetricType metric_type_; @@ -104,7 +107,6 @@ protected: int32_t gpu_num_ = 0; }; - } // namespace engine } // namespace milvus } // namespace zilliz diff --git a/cpp/src/db/insert/MemManager.h b/cpp/src/db/insert/MemManager.h index 37f0548646..751633bf67 100644 --- a/cpp/src/db/insert/MemManager.h +++ b/cpp/src/db/insert/MemManager.h @@ -23,6 +23,7 @@ #include #include +#include namespace zilliz { namespace milvus { @@ -30,7 +31,6 @@ namespace engine { class MemManager { public: - virtual Status InsertVectors(const std::string &table_id, size_t n, const float *vectors, IDNumbers &vector_ids) = 0; @@ -43,11 +43,10 @@ class MemManager { virtual size_t GetCurrentImmutableMem() = 0; virtual size_t GetCurrentMem() = 0; - }; // MemManagerAbstract using MemManagerPtr = std::shared_ptr; } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/insert/MemManagerImpl.cpp b/cpp/src/db/insert/MemManagerImpl.cpp index ecc14583f9..e555acd28d 100644 --- a/cpp/src/db/insert/MemManagerImpl.cpp +++ b/cpp/src/db/insert/MemManagerImpl.cpp @@ -16,19 +16,19 @@ // under the License. -#include "MemManagerImpl.h" +#include "db/insert/MemManagerImpl.h" #include "VectorSource.h" #include "utils/Log.h" #include "db/Constants.h" #include - namespace zilliz { namespace milvus { namespace engine { -MemTablePtr MemManagerImpl::GetMemByTable(const std::string &table_id) { +MemTablePtr +MemManagerImpl::GetMemByTable(const std::string &table_id) { auto memIt = mem_id_map_.find(table_id); if (memIt != mem_id_map_.end()) { return memIt->second; @@ -38,11 +38,11 @@ MemTablePtr MemManagerImpl::GetMemByTable(const std::string &table_id) { return mem_id_map_[table_id]; } -Status MemManagerImpl::InsertVectors(const std::string &table_id_, - size_t n_, - const float *vectors_, - IDNumbers &vector_ids_) { - +Status +MemManagerImpl::InsertVectors(const std::string &table_id_, + size_t n_, + const float *vectors_, + IDNumbers &vector_ids_) { while (GetCurrentMem() > options_.insert_buffer_size_) { std::this_thread::sleep_for(std::chrono::milliseconds(1)); } @@ -52,11 +52,11 @@ Status MemManagerImpl::InsertVectors(const std::string &table_id_, return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_); } -Status MemManagerImpl::InsertVectorsNoLock(const std::string &table_id, - size_t n, - const float *vectors, - IDNumbers &vector_ids) { - +Status +MemManagerImpl::InsertVectorsNoLock(const std::string &table_id, + size_t n, + const float *vectors, + IDNumbers &vector_ids) { MemTablePtr mem = GetMemByTable(table_id); VectorSourcePtr source = std::make_shared(n, vectors); @@ -69,10 +69,11 @@ Status MemManagerImpl::InsertVectorsNoLock(const std::string &table_id, return status; } -Status MemManagerImpl::ToImmutable() { +Status +MemManagerImpl::ToImmutable() { std::unique_lock lock(mutex_); MemIdMap temp_map; - for (auto &kv: mem_id_map_) { + for (auto &kv : mem_id_map_) { if (kv.second->Empty()) { //empty table, no need to serialize temp_map.insert(kv); @@ -85,7 +86,8 @@ Status MemManagerImpl::ToImmutable() { return Status::OK(); } -Status MemManagerImpl::Serialize(std::set &table_ids) { +Status +MemManagerImpl::Serialize(std::set &table_ids) { ToImmutable(); std::unique_lock lock(serialization_mtx_); table_ids.clear(); @@ -97,7 +99,8 @@ Status MemManagerImpl::Serialize(std::set &table_ids) { return Status::OK(); } -Status MemManagerImpl::EraseMemVector(const std::string &table_id) { +Status +MemManagerImpl::EraseMemVector(const std::string &table_id) { {//erase MemVector from rapid-insert cache std::unique_lock lock(mutex_); mem_id_map_.erase(table_id); @@ -117,7 +120,8 @@ Status MemManagerImpl::EraseMemVector(const std::string &table_id) { return Status::OK(); } -size_t MemManagerImpl::GetCurrentMutableMem() { +size_t +MemManagerImpl::GetCurrentMutableMem() { size_t total_mem = 0; for (auto &kv : mem_id_map_) { auto memTable = kv.second; @@ -126,7 +130,8 @@ size_t MemManagerImpl::GetCurrentMutableMem() { return total_mem; } -size_t MemManagerImpl::GetCurrentImmutableMem() { +size_t +MemManagerImpl::GetCurrentImmutableMem() { size_t total_mem = 0; for (auto &mem_table : immu_mem_list_) { total_mem += mem_table->GetCurrentMem(); @@ -134,10 +139,11 @@ size_t MemManagerImpl::GetCurrentImmutableMem() { return total_mem; } -size_t MemManagerImpl::GetCurrentMem() { +size_t +MemManagerImpl::GetCurrentMem() { return GetCurrentMutableMem() + GetCurrentImmutableMem(); } } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/insert/MemManagerImpl.h b/cpp/src/db/insert/MemManagerImpl.h index 767dc0eef0..1783adec18 100644 --- a/cpp/src/db/insert/MemManagerImpl.h +++ b/cpp/src/db/insert/MemManagerImpl.h @@ -24,12 +24,13 @@ #include "utils/Status.h" #include +#include +#include #include #include #include #include - namespace zilliz { namespace milvus { namespace engine { @@ -39,7 +40,8 @@ class MemManagerImpl : public MemManager { using Ptr = std::shared_ptr; MemManagerImpl(const meta::MetaPtr &meta, const DBOptions &options) - : meta_(meta), options_(options) {} + : meta_(meta), options_(options) { + } Status InsertVectors(const std::string &table_id, size_t n, const float *vectors, IDNumbers &vector_ids) override; @@ -71,7 +73,6 @@ class MemManagerImpl : public MemManager { std::mutex serialization_mtx_; }; // NewMemManager - } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/insert/MemMenagerFactory.cpp b/cpp/src/db/insert/MemMenagerFactory.cpp index d0ef51f61f..76e521e168 100644 --- a/cpp/src/db/insert/MemMenagerFactory.cpp +++ b/cpp/src/db/insert/MemMenagerFactory.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "MemMenagerFactory.h" +#include "db/insert/MemMenagerFactory.h" #include "MemManagerImpl.h" #include "utils/Log.h" #include "utils/Exception.h" @@ -26,12 +26,14 @@ #include #include #include +#include namespace zilliz { namespace milvus { namespace engine { -MemManagerPtr MemManagerFactory::Build(const std::shared_ptr& meta, const DBOptions& options) { +MemManagerPtr +MemManagerFactory::Build(const std::shared_ptr &meta, const DBOptions &options) { return std::make_shared(meta, options); } diff --git a/cpp/src/db/insert/MemMenagerFactory.h b/cpp/src/db/insert/MemMenagerFactory.h index d38404f5c0..fa0cb5930a 100644 --- a/cpp/src/db/insert/MemMenagerFactory.h +++ b/cpp/src/db/insert/MemMenagerFactory.h @@ -20,15 +20,17 @@ #include "MemManager.h" #include "db/meta/Meta.h" +#include + namespace zilliz { namespace milvus { namespace engine { class MemManagerFactory { -public: + public: static MemManagerPtr Build(const std::shared_ptr &meta, const DBOptions &options); }; -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemTable.cpp b/cpp/src/db/insert/MemTable.cpp index 4be0695572..8db6a9d141 100644 --- a/cpp/src/db/insert/MemTable.cpp +++ b/cpp/src/db/insert/MemTable.cpp @@ -16,9 +16,11 @@ // under the License. -#include "MemTable.h" +#include "db/insert/MemTable.h" #include "utils/Log.h" +#include +#include namespace zilliz { namespace milvus { @@ -30,13 +32,11 @@ MemTable::MemTable(const std::string &table_id, table_id_(table_id), meta_(meta), options_(options) { - } -Status MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) { - +Status +MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) { while (!source->AllAdded()) { - MemTableFilePtr current_mem_table_file; if (!mem_table_file_list_.empty()) { current_mem_table_file = mem_table_file_list_.back(); @@ -62,15 +62,18 @@ Status MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) { return Status::OK(); } -void MemTable::GetCurrentMemTableFile(MemTableFilePtr &mem_table_file) { +void +MemTable::GetCurrentMemTableFile(MemTableFilePtr &mem_table_file) { mem_table_file = mem_table_file_list_.back(); } -size_t MemTable::GetTableFileCount() { +size_t +MemTable::GetTableFileCount() { return mem_table_file_list_.size(); } -Status MemTable::Serialize() { +Status +MemTable::Serialize() { for (auto mem_table_file = mem_table_file_list_.begin(); mem_table_file != mem_table_file_list_.end();) { auto status = (*mem_table_file)->Serialize(); if (!status.ok()) { @@ -84,15 +87,18 @@ Status MemTable::Serialize() { return Status::OK(); } -bool MemTable::Empty() { +bool +MemTable::Empty() { return mem_table_file_list_.empty(); } -const std::string &MemTable::GetTableId() const { +const std::string & +MemTable::GetTableId() const { return table_id_; } -size_t MemTable::GetCurrentMem() { +size_t +MemTable::GetCurrentMem() { std::lock_guard lock(mutex_); size_t total_mem = 0; for (auto &mem_table_file : mem_table_file_list_) { @@ -103,4 +109,4 @@ size_t MemTable::GetCurrentMem() { } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/insert/MemTable.h b/cpp/src/db/insert/MemTable.h index c1b4125331..da7d914b41 100644 --- a/cpp/src/db/insert/MemTable.h +++ b/cpp/src/db/insert/MemTable.h @@ -23,7 +23,9 @@ #include "utils/Status.h" #include - +#include +#include +#include namespace zilliz { namespace milvus { @@ -59,11 +61,10 @@ class MemTable { DBOptions options_; std::mutex mutex_; - }; //MemTable using MemTablePtr = std::shared_ptr; } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index 12dbbcf49f..fc6c2b319a 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -16,14 +16,14 @@ // under the License. -#include "MemTableFile.h" +#include "db/insert/MemTableFile.h" #include "db/Constants.h" #include "db/engine/EngineFactory.h" #include "metrics/Metrics.h" #include "utils/Log.h" #include - +#include namespace zilliz { namespace milvus { @@ -35,20 +35,19 @@ MemTableFile::MemTableFile(const std::string &table_id, table_id_(table_id), meta_(meta), options_(options) { - current_mem_ = 0; auto status = CreateTableFile(); if (status.ok()) { execution_engine_ = EngineFactory::Build(table_file_schema_.dimension_, table_file_schema_.location_, (EngineType) table_file_schema_.engine_type_, - (MetricType)table_file_schema_.metric_type_, + (MetricType) table_file_schema_.metric_type_, table_file_schema_.nlist_); } } -Status MemTableFile::CreateTableFile() { - +Status +MemTableFile::CreateTableFile() { meta::TableFileSchema table_file_schema; table_file_schema.table_id_ = table_id_; auto status = meta_->CreateTableFile(table_file_schema); @@ -61,8 +60,8 @@ Status MemTableFile::CreateTableFile() { return status; } -Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) { - +Status +MemTableFile::Add(const VectorSourcePtr &source, IDNumbers &vector_ids) { if (table_file_schema_.dimension_ <= 0) { std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " + std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_; @@ -75,7 +74,8 @@ Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) { if (mem_left >= single_vector_mem_size) { size_t num_vectors_to_add = std::ceil(mem_left / single_vector_mem_size); size_t num_vectors_added; - auto status = source->Add(execution_engine_, table_file_schema_, num_vectors_to_add, num_vectors_added, vector_ids); + auto status = + source->Add(execution_engine_, table_file_schema_, num_vectors_to_add, num_vectors_added, vector_ids); if (status.ok()) { current_mem_ += (num_vectors_added * single_vector_mem_size); } @@ -84,20 +84,24 @@ Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) { return Status::OK(); } -size_t MemTableFile::GetCurrentMem() { +size_t +MemTableFile::GetCurrentMem() { return current_mem_; } -size_t MemTableFile::GetMemLeft() { +size_t +MemTableFile::GetMemLeft() { return (MAX_TABLE_FILE_MEM - current_mem_); } -bool MemTableFile::IsFull() { +bool +MemTableFile::IsFull() { size_t single_vector_mem_size = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE; return (GetMemLeft() < single_vector_mem_size); } -Status MemTableFile::Serialize() { +Status +MemTableFile::Serialize() { size_t size = GetCurrentMem(); server::CollectSerializeMetrics metrics(size); @@ -107,7 +111,7 @@ Status MemTableFile::Serialize() { //if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size //else set file type to RAW, no need to build index - if (table_file_schema_.engine_type_ != (int)EngineType::FAISS_IDMAP) { + if (table_file_schema_.engine_type_ != (int) EngineType::FAISS_IDMAP) { table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ? meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; } else { @@ -117,9 +121,9 @@ Status MemTableFile::Serialize() { auto status = meta_->UpdateTableFile(table_file_schema_); ENGINE_LOG_DEBUG << "New " << ((table_file_schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index") - << " file " << table_file_schema_.file_id_ << " of size " << size << " bytes"; + << " file " << table_file_schema_.file_id_ << " of size " << size << " bytes"; - if(options_.insert_cache_immediately_) { + if (options_.insert_cache_immediately_) { execution_engine_->Cache(); } @@ -128,4 +132,4 @@ Status MemTableFile::Serialize() { } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/insert/MemTableFile.h b/cpp/src/db/insert/MemTableFile.h index 168fabe7e9..1d7019a42d 100644 --- a/cpp/src/db/insert/MemTableFile.h +++ b/cpp/src/db/insert/MemTableFile.h @@ -23,17 +23,18 @@ #include "db/engine/ExecutionEngine.h" #include "utils/Status.h" +#include +#include namespace zilliz { namespace milvus { namespace engine { class MemTableFile { - public: MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options); - Status Add(const VectorSourcePtr &source, IDNumbers& vector_ids); + Status Add(const VectorSourcePtr &source, IDNumbers &vector_ids); size_t GetCurrentMem(); @@ -44,25 +45,20 @@ class MemTableFile { Status Serialize(); private: - Status CreateTableFile(); + private: const std::string table_id_; - meta::TableFileSchema table_file_schema_; - meta::MetaPtr meta_; - DBOptions options_; - size_t current_mem_; ExecutionEnginePtr execution_engine_; - }; //MemTableFile using MemTableFilePtr = std::shared_ptr; } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/insert/VectorSource.cpp b/cpp/src/db/insert/VectorSource.cpp index d0aeb5f1ef..330ef08b98 100644 --- a/cpp/src/db/insert/VectorSource.cpp +++ b/cpp/src/db/insert/VectorSource.cpp @@ -16,32 +16,30 @@ // under the License. -#include "VectorSource.h" +#include "db/insert/VectorSource.h" #include "db/engine/ExecutionEngine.h" #include "db/engine/EngineFactory.h" #include "utils/Log.h" #include "metrics/Metrics.h" - namespace zilliz { namespace milvus { namespace engine { - VectorSource::VectorSource(const size_t &n, const float *vectors) : - n_(n), - vectors_(vectors), - id_generator_(std::make_shared()) { + n_(n), + vectors_(vectors), + id_generator_(std::make_shared()) { current_num_vectors_added = 0; } -Status VectorSource::Add(const ExecutionEnginePtr &execution_engine, - const meta::TableFileSchema &table_file_schema, - const size_t &num_vectors_to_add, - size_t &num_vectors_added, - IDNumbers &vector_ids) { - +Status +VectorSource::Add(const ExecutionEnginePtr &execution_engine, + const meta::TableFileSchema &table_file_schema, + const size_t &num_vectors_to_add, + size_t &num_vectors_added, + IDNumbers &vector_ids) { server::CollectAddMetrics metrics(n_, table_file_schema.dimension_); num_vectors_added = current_num_vectors_added + num_vectors_to_add <= n_ ? @@ -52,7 +50,7 @@ Status VectorSource::Add(const ExecutionEnginePtr &execution_engine, } else { vector_ids_to_add.resize(num_vectors_added); for (int pos = current_num_vectors_added; pos < current_num_vectors_added + num_vectors_added; pos++) { - vector_ids_to_add[pos-current_num_vectors_added] = vector_ids[pos]; + vector_ids_to_add[pos - current_num_vectors_added] = vector_ids[pos]; } } Status status = execution_engine->AddWithIds(num_vectors_added, @@ -70,18 +68,21 @@ Status VectorSource::Add(const ExecutionEnginePtr &execution_engine, return status; } -size_t VectorSource::GetNumVectorsAdded() { +size_t +VectorSource::GetNumVectorsAdded() { return current_num_vectors_added; } -bool VectorSource::AllAdded() { +bool +VectorSource::AllAdded() { return (current_num_vectors_added == n_); } -IDNumbers VectorSource::GetVectorIds() { +IDNumbers +VectorSource::GetVectorIds() { return vector_ids_; } } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/insert/VectorSource.h b/cpp/src/db/insert/VectorSource.h index a0292181ce..fd31a14fa6 100644 --- a/cpp/src/db/insert/VectorSource.h +++ b/cpp/src/db/insert/VectorSource.h @@ -23,6 +23,7 @@ #include "db/engine/ExecutionEngine.h" #include "utils/Status.h" +#include namespace zilliz { namespace milvus { @@ -45,7 +46,6 @@ class VectorSource { IDNumbers GetVectorIds(); private: - const size_t n_; const float *vectors_; IDNumbers vector_ids_; @@ -53,11 +53,10 @@ class VectorSource { size_t current_num_vectors_added; std::shared_ptr id_generator_; - }; //VectorSource using VectorSourcePtr = std::shared_ptr; } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/meta/Meta.h b/cpp/src/db/meta/Meta.h index 58168e5fee..42fe6a6dd5 100644 --- a/cpp/src/db/meta/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -25,14 +25,16 @@ #include #include +#include +#include namespace zilliz { namespace milvus { namespace engine { namespace meta { -static const char* META_TABLES = "Tables"; -static const char* META_TABLEFILES = "TableFiles"; +static const char *META_TABLES = "Tables"; +static const char *META_TABLEFILES = "TableFiles"; class Meta { public: @@ -46,7 +48,7 @@ class Meta { virtual Status AllTables(std::vector &table_schema_array) = 0; - virtual Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) = 0; + virtual Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) = 0; virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0; @@ -83,9 +85,9 @@ class Meta { virtual Status FilesByType(const std::string &table_id, const std::vector &file_types, - std::vector& file_ids) = 0; + std::vector &file_ids) = 0; - virtual Status DescribeTableIndex(const std::string &table_id, TableIndex& index) = 0; + virtual Status DescribeTableIndex(const std::string &table_id, TableIndex &index) = 0; virtual Status DropTableIndex(const std::string &table_id) = 0; @@ -96,7 +98,6 @@ class Meta { virtual Status DropAll() = 0; virtual Status Count(const std::string &table_id, uint64_t &result) = 0; - }; // MetaData using MetaPtr = std::shared_ptr; diff --git a/cpp/src/db/meta/MetaConsts.h b/cpp/src/db/meta/MetaConsts.h index 04b548009b..6abae564e8 100644 --- a/cpp/src/db/meta/MetaConsts.h +++ b/cpp/src/db/meta/MetaConsts.h @@ -23,20 +23,20 @@ namespace engine { namespace meta { const size_t K = 1024UL; -const size_t M = K*K; -const size_t G = K*M; -const size_t T = K*G; +const size_t M = K * K; +const size_t G = K * M; +const size_t T = K * G; const size_t S_PS = 1UL; -const size_t MS_PS = 1000*S_PS; -const size_t US_PS = 1000*MS_PS; -const size_t NS_PS = 1000*US_PS; +const size_t MS_PS = 1000 * S_PS; +const size_t US_PS = 1000 * MS_PS; +const size_t NS_PS = 1000 * US_PS; const size_t SECOND = 1UL; -const size_t M_SEC = 60*SECOND; -const size_t H_SEC = 60*M_SEC; -const size_t D_SEC = 24*H_SEC; -const size_t W_SEC = 7*D_SEC; +const size_t M_SEC = 60 * SECOND; +const size_t H_SEC = 60 * M_SEC; +const size_t D_SEC = 24 * H_SEC; +const size_t W_SEC = 7 * D_SEC; } // namespace meta } // namespace engine diff --git a/cpp/src/db/meta/MetaFactory.cpp b/cpp/src/db/meta/MetaFactory.cpp index 8837d54722..dd0b2ae7ea 100644 --- a/cpp/src/db/meta/MetaFactory.cpp +++ b/cpp/src/db/meta/MetaFactory.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "MetaFactory.h" +#include "db/meta/MetaFactory.h" #include "SqliteMetaImpl.h" #include "MySQLMetaImpl.h" #include "utils/Log.h" @@ -28,47 +28,52 @@ #include #include #include +#include namespace zilliz { namespace milvus { namespace engine { - DBMetaOptions MetaFactory::BuildOption(const std::string &path) { - auto p = path; - if(p == "") { - srand(time(nullptr)); - std::stringstream ss; - ss << "/tmp/" << rand(); - p = ss.str(); - } - - DBMetaOptions meta; - meta.path_ = p; - return meta; +DBMetaOptions +MetaFactory::BuildOption(const std::string &path) { + auto p = path; + if (p == "") { + srand(time(nullptr)); + std::stringstream ss; + uint32_t rd = 0; + rand_r(&rd); + ss << "/tmp/" << rd; + p = ss.str(); } - meta::MetaPtr MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) { - std::string uri = metaOptions.backend_uri_; + DBMetaOptions meta; + meta.path_ = p; + return meta; +} - utils::MetaUriInfo uri_info; - auto status = utils::ParseMetaUri(uri, uri_info); - if(!status.ok()) { - ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri; - throw InvalidArgumentException("Wrong URI format "); - } +meta::MetaPtr +MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) { + std::string uri = metaOptions.backend_uri_; - if (strcasecmp(uri_info.dialect_.c_str(), "mysql") == 0) { - ENGINE_LOG_INFO << "Using MySQL"; - return std::make_shared(metaOptions, mode); - } else if (strcasecmp(uri_info.dialect_.c_str(), "sqlite") == 0) { - ENGINE_LOG_INFO << "Using SQLite"; - return std::make_shared(metaOptions); - } else { - ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << uri_info.dialect_; - throw InvalidArgumentException("URI dialect is not mysql / sqlite"); - } + utils::MetaUriInfo uri_info; + auto status = utils::ParseMetaUri(uri, uri_info); + if (!status.ok()) { + ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri; + throw InvalidArgumentException("Wrong URI format "); } + if (strcasecmp(uri_info.dialect_.c_str(), "mysql") == 0) { + ENGINE_LOG_INFO << "Using MySQL"; + return std::make_shared(metaOptions, mode); + } else if (strcasecmp(uri_info.dialect_.c_str(), "sqlite") == 0) { + ENGINE_LOG_INFO << "Using SQLite"; + return std::make_shared(metaOptions); + } else { + ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << uri_info.dialect_; + throw InvalidArgumentException("URI dialect is not mysql / sqlite"); + } +} + } // namespace engine } // namespace milvus } // namespace zilliz diff --git a/cpp/src/db/meta/MetaFactory.h b/cpp/src/db/meta/MetaFactory.h index edc9706649..a13a09715a 100644 --- a/cpp/src/db/meta/MetaFactory.h +++ b/cpp/src/db/meta/MetaFactory.h @@ -20,18 +20,19 @@ #include "Meta.h" #include "db/Options.h" +#include + namespace zilliz { namespace milvus { namespace engine { class MetaFactory { -public: + public: static DBMetaOptions BuildOption(const std::string &path = ""); static meta::MetaPtr Build(const DBMetaOptions &metaOptions, const int &mode); }; - -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/MetaTypes.h b/cpp/src/db/meta/MetaTypes.h index cc6dd52057..6fd649bc07 100644 --- a/cpp/src/db/meta/MetaTypes.h +++ b/cpp/src/db/meta/MetaTypes.h @@ -23,21 +23,22 @@ #include #include #include +#include namespace zilliz { namespace milvus { namespace engine { namespace meta { -constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP; +constexpr int32_t DEFAULT_ENGINE_TYPE = (int) EngineType::FAISS_IDMAP; constexpr int32_t DEFAULT_NLIST = 16384; -constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2; +constexpr int32_t DEFAULT_METRIC_TYPE = (int) MetricType::L2; constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB; constexpr int64_t FLAG_MASK_NO_USERID = 0x1; -constexpr int64_t FLAG_MASK_HAS_USERID = 0x1<<1; +constexpr int64_t FLAG_MASK_HAS_USERID = 0x1 << 1; -using DateT = int ; +using DateT = int; const DateT EmptyDate = -1; using DatesT = std::vector; @@ -49,7 +50,7 @@ struct TableSchema { size_t id_ = 0; std::string table_id_; - int32_t state_ = (int)NORMAL; + int32_t state_ = (int) NORMAL; uint16_t dimension_ = 0; int64_t created_on_ = 0; int64_t flag_ = 0; diff --git a/cpp/src/db/meta/MySQLConnectionPool.cpp b/cpp/src/db/meta/MySQLConnectionPool.cpp index 87998122f4..9d612d8b54 100644 --- a/cpp/src/db/meta/MySQLConnectionPool.cpp +++ b/cpp/src/db/meta/MySQLConnectionPool.cpp @@ -16,37 +16,39 @@ // under the License. -#include "MySQLConnectionPool.h" +#include "db/meta/MySQLConnectionPool.h" namespace zilliz { namespace milvus { namespace engine { namespace meta { - // Do a simple form of in-use connection limiting: wait to return - // a connection until there are a reasonably low number in use - // already. Can't do this in create() because we're interested in - // connections actually in use, not those created. Also note that - // we keep our own count; ConnectionPool::size() isn't the same! - mysqlpp::Connection *MySQLConnectionPool::grab() { - while (conns_in_use_ > max_pool_size_) { - sleep(1); - } - - ++conns_in_use_; - return mysqlpp::ConnectionPool::grab(); +// Do a simple form of in-use connection limiting: wait to return +// a connection until there are a reasonably low number in use +// already. Can't do this in create() because we're interested in +// connections actually in use, not those created. Also note that +// we keep our own count; ConnectionPool::size() isn't the same! +mysqlpp::Connection * +MySQLConnectionPool::grab() { + while (conns_in_use_ > max_pool_size_) { + sleep(1); } - // Other half of in-use conn count limit - void MySQLConnectionPool::release(const mysqlpp::Connection *pc) { - mysqlpp::ConnectionPool::release(pc); + ++conns_in_use_; + return mysqlpp::ConnectionPool::grab(); +} - if (conns_in_use_ <= 0) { - ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = " << conns_in_use_; - } else { - --conns_in_use_; - } +// Other half of in-use conn count limit +void +MySQLConnectionPool::release(const mysqlpp::Connection *pc) { + mysqlpp::ConnectionPool::release(pc); + if (conns_in_use_ <= 0) { + ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = " + << conns_in_use_; + } else { + --conns_in_use_; } +} // int MySQLConnectionPool::getConnectionsInUse() { // return conns_in_use_; @@ -56,39 +58,42 @@ namespace meta { // max_idle_time_ = max_idle; // } - std::string MySQLConnectionPool::getDB() { - return db_; - } +std::string +MySQLConnectionPool::getDB() { + return db_; +} - // Superclass overrides - mysqlpp::Connection *MySQLConnectionPool::create() { - - try { - // Create connection using the parameters we were passed upon - // creation. - mysqlpp::Connection *conn = new mysqlpp::Connection(); - conn->set_option(new mysqlpp::ReconnectOption(true)); - conn->connect(db_.empty() ? 0 : db_.c_str(), - server_.empty() ? 0 : server_.c_str(), - user_.empty() ? 0 : user_.c_str(), - password_.empty() ? 0 : password_.c_str(), - port_); - return conn; - } catch (const mysqlpp::ConnectionFailed& er) { - ENGINE_LOG_ERROR << "Failed to connect to database server" << ": " << er.what(); - return nullptr; - } +// Superclass overrides +mysqlpp::Connection * +MySQLConnectionPool::create() { + try { + // Create connection using the parameters we were passed upon + // creation. + mysqlpp::Connection *conn = new mysqlpp::Connection(); + conn->set_option(new mysqlpp::ReconnectOption(true)); + conn->connect(db_.empty() ? 0 : db_.c_str(), + server_.empty() ? 0 : server_.c_str(), + user_.empty() ? 0 : user_.c_str(), + password_.empty() ? 0 : password_.c_str(), + port_); + return conn; + } catch (const mysqlpp::ConnectionFailed &er) { + ENGINE_LOG_ERROR << "Failed to connect to database server" << ": " << er.what(); + return nullptr; } +} - void MySQLConnectionPool::destroy(mysqlpp::Connection *cp) { - // Our superclass can't know how we created the Connection, so - // it delegates destruction to us, to be safe. - delete cp; - } +void +MySQLConnectionPool::destroy(mysqlpp::Connection *cp) { + // Our superclass can't know how we created the Connection, so + // it delegates destruction to us, to be safe. + delete cp; +} - unsigned int MySQLConnectionPool::max_idle_time() { - return max_idle_time_; - } +unsigned int +MySQLConnectionPool::max_idle_time() { + return max_idle_time_; +} } // namespace meta } // namespace engine diff --git a/cpp/src/db/meta/MySQLConnectionPool.h b/cpp/src/db/meta/MySQLConnectionPool.h index 4aadafd714..761f272bc1 100644 --- a/cpp/src/db/meta/MySQLConnectionPool.h +++ b/cpp/src/db/meta/MySQLConnectionPool.h @@ -30,8 +30,7 @@ namespace engine { namespace meta { class MySQLConnectionPool : public mysqlpp::ConnectionPool { - -public: + public: // The object's only constructor MySQLConnectionPool(std::string dbName, std::string userName, @@ -39,15 +38,13 @@ public: std::string serverIp, int port = 0, int maxPoolSize = 8) : - db_(dbName), - user_(userName), - password_(passWord), - server_(serverIp), - port_(port), - max_pool_size_(maxPoolSize) { - + db_(dbName), + user_(userName), + password_(passWord), + server_(serverIp), + port_(port), + max_pool_size_(maxPoolSize) { conns_in_use_ = 0; - max_idle_time_ = 10; //10 seconds } @@ -68,8 +65,7 @@ public: std::string getDB(); -protected: - + protected: // Superclass overrides mysqlpp::Connection *create() override; @@ -77,7 +73,7 @@ protected: unsigned int max_idle_time() override; -private: + private: // Number of connections currently in use std::atomic conns_in_use_; @@ -93,4 +89,4 @@ private: } // namespace meta } // namespace engine } // namespace milvus -} // namespace zilliz \ No newline at end of file +} // namespace zilliz diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index c1c55febf0..490eda1916 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "MySQLMetaImpl.h" +#include "db/meta/MySQLMetaImpl.h" #include "db/IDGenerator.h" #include "db/Utils.h" #include "utils/Log.h" @@ -33,21 +33,21 @@ #include #include #include +#include +#include #include #include - namespace zilliz { namespace milvus { namespace engine { namespace meta { -using namespace mysqlpp; - namespace { -Status HandleException(const std::string &desc, const char* what = nullptr) { - if(what == nullptr) { +Status +HandleException(const std::string &desc, const char *what = nullptr) { + if (what == nullptr) { ENGINE_LOG_ERROR << desc; return Status(DB_META_TRANSACTION_FAILED, desc); } else { @@ -58,8 +58,8 @@ Status HandleException(const std::string &desc, const char* what = nullptr) { } class MetaField { -public: - MetaField(const std::string& name, const std::string& type, const std::string& setting) + public: + MetaField(const std::string &name, const std::string &type, const std::string &setting) : name_(name), type_(type), setting_(setting) { @@ -75,14 +75,14 @@ public: // mysql field type has additional information. for instance, a filed type is defined as 'BIGINT' // we get the type from sql is 'bigint(20)', so we need to ignore the '(20)' - bool IsEqual(const MetaField& field) const { + bool IsEqual(const MetaField &field) const { size_t name_len_min = field.name_.length() > name_.length() ? name_.length() : field.name_.length(); size_t type_len_min = field.type_.length() > type_.length() ? type_.length() : field.type_.length(); return strncasecmp(field.name_.c_str(), name_.c_str(), name_len_min) == 0 && - strncasecmp(field.type_.c_str(), type_.c_str(), type_len_min) == 0; + strncasecmp(field.type_.c_str(), type_.c_str(), type_len_min) == 0; } -private: + private: std::string name_; std::string type_; std::string setting_; @@ -90,8 +90,8 @@ private: using MetaFields = std::vector; class MetaSchema { -public: - MetaSchema(const std::string& name, const MetaFields& fields) + public: + MetaSchema(const std::string &name, const MetaFields &fields) : name_(name), fields_(fields) { } @@ -102,8 +102,8 @@ public: std::string ToString() const { std::string result; - for(auto& field : fields_) { - if(!result.empty()) { + for (auto &field : fields_) { + if (!result.empty()) { result += ","; } result += field.ToString(); @@ -113,11 +113,11 @@ public: //if the outer fields contains all this MetaSchema fields, return true //otherwise return false - bool IsEqual(const MetaFields& fields) const { + bool IsEqual(const MetaFields &fields) const { std::vector found_field; - for(const auto& this_field : fields_) { - for(const auto& outer_field : fields) { - if(this_field.IsEqual(outer_field)) { + for (const auto &this_field : fields_) { + for (const auto &outer_field : fields) { + if (this_field.IsEqual(outer_field)) { found_field.push_back(this_field.name()); break; } @@ -127,7 +127,7 @@ public: return found_field.size() == fields_.size(); } -private: + private: std::string name_; MetaFields fields_; }; @@ -160,20 +160,20 @@ static const MetaSchema TABLEFILES_SCHEMA(META_TABLEFILES, { MetaField("date", "INT", "DEFAULT -1 NOT NULL"), }); -} +} // namespace //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_, const int &mode) - : options_(options_), +MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options, const int &mode) + : options_(options), mode_(mode) { Initialize(); } MySQLMetaImpl::~MySQLMetaImpl() { - } -Status MySQLMetaImpl::NextTableId(std::string &table_id) { +Status +MySQLMetaImpl::NextTableId(std::string &table_id) { std::stringstream ss; SimpleIDGenerator g; ss << g.GetNextIDNumber(); @@ -181,7 +181,8 @@ Status MySQLMetaImpl::NextTableId(std::string &table_id) { return Status::OK(); } -Status MySQLMetaImpl::NextFileId(std::string &file_id) { +Status +MySQLMetaImpl::NextFileId(std::string &file_id) { std::stringstream ss; SimpleIDGenerator g; ss << g.GetNextIDNumber(); @@ -189,26 +190,27 @@ Status MySQLMetaImpl::NextFileId(std::string &file_id) { return Status::OK(); } -void MySQLMetaImpl::ValidateMetaSchema() { - if(nullptr == mysql_connection_pool_) { +void +MySQLMetaImpl::ValidateMetaSchema() { + if (nullptr == mysql_connection_pool_) { return; } - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return; } - auto validate_func = [&](const MetaSchema& schema) { - Query query_statement = connectionPtr->query(); + auto validate_func = [&](const MetaSchema &schema) { + mysqlpp::Query query_statement = connectionPtr->query(); query_statement << "DESC " << schema.name() << ";"; MetaFields exist_fields; try { - StoreQueryResult res = query_statement.store(); + mysqlpp::StoreQueryResult res = query_statement.store(); for (size_t i = 0; i < res.num_rows(); i++) { - const Row &row = res[i]; + const mysqlpp::Row &row = res[i]; std::string name, type; row["Field"].to_string(name); row["Type"].to_string(type); @@ -219,7 +221,7 @@ void MySQLMetaImpl::ValidateMetaSchema() { ENGINE_LOG_DEBUG << "Meta table '" << schema.name() << "' not exist and will be created"; } - if(exist_fields.empty()) { + if (exist_fields.empty()) { return true; } @@ -237,7 +239,8 @@ void MySQLMetaImpl::ValidateMetaSchema() { } } -Status MySQLMetaImpl::Initialize() { +Status +MySQLMetaImpl::Initialize() { //step 1: create db root path if (!boost::filesystem::is_directory(options_.path_)) { auto ret = boost::filesystem::create_directory(options_.path_); @@ -253,7 +256,7 @@ Status MySQLMetaImpl::Initialize() { //step 2: parse and check meta uri utils::MetaUriInfo uri_info; auto status = utils::ParseMetaUri(uri, uri_info); - if(!status.ok()) { + if (!status.ok()) { std::string msg = "Wrong URI format: " + uri; ENGINE_LOG_ERROR << msg; throw Exception(DB_INVALID_META_URI, msg); @@ -274,8 +277,8 @@ Status MySQLMetaImpl::Initialize() { } mysql_connection_pool_ = - std::make_shared(uri_info.db_name_, uri_info.username_, - uri_info.password_, uri_info.host_, port, max_pool_size); + std::make_shared(uri_info.db_name_, uri_info.username_, + uri_info.password_, uri_info.host_, port, max_pool_size); ENGINE_LOG_DEBUG << "MySQL connection pool: maximum pool size = " << std::to_string(max_pool_size); //step 4: validate to avoid open old version schema @@ -283,24 +286,22 @@ Status MySQLMetaImpl::Initialize() { //step 5: create meta tables try { - if (mode_ != DBOptions::MODE::CLUSTER_READONLY) { CleanUp(); } { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - if (!connectionPtr->thread_aware()) { ENGINE_LOG_ERROR << "MySQL++ wasn't built with thread awareness! Can't run without it."; return Status(DB_ERROR, "MySQL++ wasn't built with thread awareness! Can't run without it."); } - Query InitializeQuery = connectionPtr->query(); + mysqlpp::Query InitializeQuery = connectionPtr->query(); InitializeQuery << "CREATE TABLE IF NOT EXISTS " << TABLES_SCHEMA.name() << " (" << TABLES_SCHEMA.ToString() + ");"; @@ -320,7 +321,6 @@ Status MySQLMetaImpl::Initialize() { return HandleException("Initialization Error", InitializeQuery.error()); } } //Scoped Connection - } catch (std::exception &e) { return HandleException("GENERAL ERROR DURING INITIALIZATION", e.what()); } @@ -329,8 +329,9 @@ Status MySQLMetaImpl::Initialize() { } // PXU TODO: Temp solution. Will fix later -Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, - const DatesT &dates) { +Status +MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, + const DatesT &dates) { if (dates.empty()) { return Status::OK(); } @@ -351,59 +352,59 @@ Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, dateListStr = dateListStr.substr(0, dateListStr.size() - 2); //remove the last ", " { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - - Query dropPartitionsByDatesQuery = connectionPtr->query(); + mysqlpp::Query dropPartitionsByDatesQuery = connectionPtr->query(); dropPartitionsByDatesQuery << "UPDATE " << META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," << "updated_time = " << utils::GetMicroSecTimeStamp() << " " << - "WHERE table_id = " << quote << table_id << " AND " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << "date in (" << dateListStr << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str(); if (!dropPartitionsByDatesQuery.exec()) { - return HandleException("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", dropPartitionsByDatesQuery.error()); + return HandleException("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", + dropPartitionsByDatesQuery.error()); } } //Scoped Connection ENGINE_LOG_DEBUG << "Successfully drop partitions, table id = " << table_schema.table_id_; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES", e.what()); } return Status::OK(); } -Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { +Status +MySQLMetaImpl::CreateTable(TableSchema &table_schema) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query createTableQuery = connectionPtr->query(); + mysqlpp::Query createTableQuery = connectionPtr->query(); if (table_schema.table_id_.empty()) { NextTableId(table_schema.table_id_); } else { createTableQuery << "SELECT state FROM " << META_TABLES << " " << - "WHERE table_id = " << quote << table_schema.table_id_ << ";"; + "WHERE table_id = " << mysqlpp::quote << table_schema.table_id_ << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); - StoreQueryResult res = createTableQuery.store(); + mysqlpp::StoreQueryResult res = createTableQuery.store(); if (res.num_rows() == 1) { int state = res[0]["state"]; @@ -431,13 +432,14 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { createTableQuery << "INSERT INTO " << META_TABLES << " " << - "VALUES(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << - created_on << ", " << flag << ", " << index_file_size << ", " << engine_type << ", " << + "VALUES(" << id << ", " << mysqlpp::quote << table_id << ", " << + state << ", " << dimension << ", " << created_on << ", " << + flag << ", " << index_file_size << ", " << engine_type << ", " << nlist << ", " << metric_type << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); - if (SimpleResult res = createTableQuery.execute()) { + if (mysqlpp::SimpleResult res = createTableQuery.execute()) { table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? //Consume all results to avoid "Commands out of sync" error @@ -448,43 +450,43 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { ENGINE_LOG_DEBUG << "Successfully create table: " << table_schema.table_id_; return utils::CreateTablePath(options_, table_schema.table_id_); - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN CREATING TABLE", e.what()); } } -Status MySQLMetaImpl::FilesByType(const std::string &table_id, - const std::vector &file_types, - std::vector &file_ids) { - if(file_types.empty()) { +Status +MySQLMetaImpl::FilesByType(const std::string &table_id, + const std::vector &file_types, + std::vector &file_ids) { + if (file_types.empty()) { return Status(DB_ERROR, "file types array is empty"); } try { file_ids.clear(); - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } std::string types; - for(auto type : file_types) { - if(!types.empty()) { + for (auto type : file_types) { + if (!types.empty()) { types += ","; } types += std::to_string(type); } - Query hasNonIndexFilesQuery = connectionPtr->query(); + mysqlpp::Query hasNonIndexFilesQuery = connectionPtr->query(); //since table_id is a unique column we just need to check whether it exists or not hasNonIndexFilesQuery << "SELECT file_id, file_type FROM " << META_TABLEFILES << " " << - "WHERE table_id = " << quote << table_id << " AND " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << "file_type in (" << types << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesByType: " << hasNonIndexFilesQuery.str(); @@ -502,29 +504,21 @@ Status MySQLMetaImpl::FilesByType(const std::string &table_id, int32_t file_type = resRow["file_type"]; switch (file_type) { - case (int) TableFileSchema::RAW: - raw_count++; + case (int) TableFileSchema::RAW:raw_count++; break; - case (int) TableFileSchema::NEW: - new_count++; + case (int) TableFileSchema::NEW:new_count++; break; - case (int) TableFileSchema::NEW_MERGE: - new_merge_count++; + case (int) TableFileSchema::NEW_MERGE:new_merge_count++; break; - case (int) TableFileSchema::NEW_INDEX: - new_index_count++; + case (int) TableFileSchema::NEW_INDEX:new_index_count++; break; - case (int) TableFileSchema::TO_INDEX: - to_index_count++; + case (int) TableFileSchema::TO_INDEX:to_index_count++; break; - case (int) TableFileSchema::INDEX: - index_count++; + case (int) TableFileSchema::INDEX:index_count++; break; - case (int) TableFileSchema::BACKUP: - backup_count++; - break; - default: + case (int) TableFileSchema::BACKUP:backup_count++; break; + default:break; } } @@ -533,7 +527,6 @@ Status MySQLMetaImpl::FilesByType(const std::string &table_id, << " new_index files:" << new_index_count << " to_index files:" << to_index_count << " index files:" << index_count << " backup files:" << backup_count; } - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN GET FILE BY TYPE", e.what()); } @@ -541,29 +534,30 @@ Status MySQLMetaImpl::FilesByType(const std::string &table_id, return Status::OK(); } -Status MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex& index) { +Status +MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex &index) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query updateTableIndexParamQuery = connectionPtr->query(); + mysqlpp::Query updateTableIndexParamQuery = connectionPtr->query(); updateTableIndexParamQuery << "SELECT id, state, dimension, created_on FROM " << META_TABLES << " " << - "WHERE table_id = " << quote << table_id << " AND " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableIndex: " << updateTableIndexParamQuery.str(); - StoreQueryResult res = updateTableIndexParamQuery.store(); + mysqlpp::StoreQueryResult res = updateTableIndexParamQuery.store(); if (res.num_rows() == 1) { - const Row &resRow = res[0]; + const mysqlpp::Row &resRow = res[0]; size_t id = resRow["id"]; int32_t state = resRow["state"]; @@ -579,22 +573,20 @@ Status MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableI "engine_type = " << index.engine_type_ << ", " << "nlist = " << index.nlist_ << ", " << "metric_type = " << index.metric_type_ << " " << - "WHERE table_id = " << quote << table_id << ";"; + "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableIndex: " << updateTableIndexParamQuery.str(); - if (!updateTableIndexParamQuery.exec()) { - return HandleException("QUERY ERROR WHEN UPDATING TABLE INDEX PARAM", updateTableIndexParamQuery.error()); + return HandleException("QUERY ERROR WHEN UPDATING TABLE INDEX PARAM", + updateTableIndexParamQuery.error()); } } else { return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); } - } //Scoped Connection ENGINE_LOG_DEBUG << "Successfully update table index, table id = " << table_id; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE INDEX PARAM", e.what()); } @@ -602,33 +594,32 @@ Status MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableI return Status::OK(); } -Status MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { +Status +MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query updateTableFlagQuery = connectionPtr->query(); + mysqlpp::Query updateTableFlagQuery = connectionPtr->query(); updateTableFlagQuery << "UPDATE " << META_TABLES << " " << "SET flag = " << flag << " " << - "WHERE table_id = " << quote << table_id << ";"; + "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFlag: " << updateTableFlagQuery.str(); if (!updateTableFlagQuery.exec()) { return HandleException("QUERY ERROR WHEN UPDATING TABLE FLAG", updateTableFlagQuery.error()); } - } //Scoped Connection ENGINE_LOG_DEBUG << "Successfully update table flag, table id = " << table_id; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FLAG", e.what()); } @@ -636,29 +627,30 @@ Status MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) return Status::OK(); } -Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex& index) { +Status +MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex &index) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query describeTableIndexQuery = connectionPtr->query(); + mysqlpp::Query describeTableIndexQuery = connectionPtr->query(); describeTableIndexQuery << "SELECT engine_type, nlist, index_file_size, metric_type FROM " << - META_TABLES << " " << - "WHERE table_id = " << quote << table_id << " AND " << - "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + META_TABLES << " " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << + "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTableIndex: " << describeTableIndexQuery.str(); - StoreQueryResult res = describeTableIndexQuery.store(); + mysqlpp::StoreQueryResult res = describeTableIndexQuery.store(); if (res.num_rows() == 1) { - const Row &resRow = res[0]; + const mysqlpp::Row &resRow = res[0]; index.engine_type_ = resRow["engine_type"]; index.nlist_ = resRow["nlist"]; @@ -666,9 +658,7 @@ Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex } else { return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); } - } //Scoped Connection - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FLAG", e.what()); } @@ -676,25 +666,26 @@ Status MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex return Status::OK(); } -Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { +Status +MySQLMetaImpl::DropTableIndex(const std::string &table_id) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query dropTableIndexQuery = connectionPtr->query(); + mysqlpp::Query dropTableIndexQuery = connectionPtr->query(); //soft delete index files dropTableIndexQuery << "UPDATE " << META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," << "updated_time = " << utils::GetMicroSecTimeStamp() << " " << - "WHERE table_id = " << quote << table_id << " AND " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); @@ -708,7 +699,7 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::RAW) << "," << "updated_time = " << utils::GetMicroSecTimeStamp() << " " << - "WHERE table_id = " << quote << table_id << " AND " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << "file_type = " << std::to_string(TableFileSchema::BACKUP) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); @@ -723,18 +714,16 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { "SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) << "," << "nlist = " << std::to_string(DEFAULT_NLIST) << ", " << "metric_type = " << std::to_string(DEFAULT_METRIC_TYPE) << " " << - "WHERE table_id = " << quote << table_id << ";"; + "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); if (!dropTableIndexQuery.exec()) { return HandleException("QUERY ERROR WHEN DROPPING TABLE INDEX", dropTableIndexQuery.error()); } - } //Scoped Connection ENGINE_LOG_DEBUG << "Successfully drop table index, table id = " << table_id; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN DROPPING TABLE INDEX", e.what()); } @@ -742,30 +731,30 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { return Status::OK(); } -Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { +Status +MySQLMetaImpl::DeleteTable(const std::string &table_id) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } //soft delete table - Query deleteTableQuery = connectionPtr->query(); + mysqlpp::Query deleteTableQuery = connectionPtr->query(); // deleteTableQuery << "UPDATE " << META_TABLES << " " << "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " << - "WHERE table_id = " << quote << table_id << ";"; + "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str(); if (!deleteTableQuery.exec()) { return HandleException("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error()); } - } //Scoped Connection if (mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { @@ -773,7 +762,6 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { } ENGINE_LOG_DEBUG << "Successfully delete table, table id = " << table_id; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN DELETING TABLE", e.what()); } @@ -781,24 +769,25 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { return Status::OK(); } -Status MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { +Status +MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } //soft delete table files - Query deleteTableFilesQuery = connectionPtr->query(); + mysqlpp::Query deleteTableFilesQuery = connectionPtr->query(); // deleteTableFilesQuery << "UPDATE " << META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << - "WHERE table_id = " << quote << table_id << " AND " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str(); @@ -809,7 +798,6 @@ Status MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { } //Scoped Connection ENGINE_LOG_DEBUG << "Successfully delete table files, table id = " << table_id; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN DELETING TABLE FILES", e.what()); } @@ -817,22 +805,24 @@ Status MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { return Status::OK(); } -Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { +Status +MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { try { server::MetricCollector metric; - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query describeTableQuery = connectionPtr->query(); - describeTableQuery << "SELECT id, state, dimension, created_on, flag, index_file_size, engine_type, nlist, metric_type FROM " << - META_TABLES << " " << - "WHERE table_id = " << quote << table_schema.table_id_ << " " << - "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + mysqlpp::Query describeTableQuery = connectionPtr->query(); + describeTableQuery + << "SELECT id, state, dimension, created_on, flag, index_file_size, engine_type, nlist, metric_type " + << " FROM " << META_TABLES << " " << + "WHERE table_id = " << mysqlpp::quote << table_schema.table_id_ << " " << + "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str(); @@ -840,7 +830,7 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { } //Scoped Connection if (res.num_rows() == 1) { - const Row &resRow = res[0]; + const mysqlpp::Row &resRow = res[0]; table_schema.id_ = resRow["id"]; //implicit conversion @@ -862,7 +852,6 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { } else { return Status(DB_NOT_FOUND, "Table " + table_schema.table_id_ + " not found"); } - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN DESCRIBING TABLE", e.what()); } @@ -870,25 +859,26 @@ Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { return Status::OK(); } -Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { +Status +MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { try { server::MetricCollector metric; - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query hasTableQuery = connectionPtr->query(); + mysqlpp::Query hasTableQuery = connectionPtr->query(); //since table_id is a unique column we just need to check whether it exists or not hasTableQuery << "SELECT EXISTS " << "(SELECT 1 FROM " << META_TABLES << " " << - "WHERE table_id = " << quote << table_id << " " << + "WHERE table_id = " << mysqlpp::quote << table_id << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << - "AS " << quote << "check" << ";"; + "AS " << mysqlpp::quote << "check" << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str(); @@ -897,7 +887,6 @@ Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { int check = res[0]["check"]; has_or_not = (check == 1); - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN CHECKING IF TABLE EXISTS", e.what()); } @@ -905,19 +894,21 @@ Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { return Status::OK(); } -Status MySQLMetaImpl::AllTables(std::vector &table_schema_array) { +Status +MySQLMetaImpl::AllTables(std::vector &table_schema_array) { try { server::MetricCollector metric; - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query allTablesQuery = connectionPtr->query(); - allTablesQuery << "SELECT id, table_id, dimension, engine_type, nlist, index_file_size, metric_type FROM " << + mysqlpp::Query allTablesQuery = connectionPtr->query(); + allTablesQuery << "SELECT id, table_id, dimension, engine_type, nlist, index_file_size, metric_type FROM " + << META_TABLES << " " << "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; @@ -954,7 +945,8 @@ Status MySQLMetaImpl::AllTables(std::vector &table_schema_array) { return Status::OK(); } -Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { +Status +MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { if (file_schema.date_ == EmptyDate) { file_schema.date_ = utils::GetDate(); } @@ -991,23 +983,24 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { std::string date = std::to_string(file_schema.date_); { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query createTableFileQuery = connectionPtr->query(); + mysqlpp::Query createTableFileQuery = connectionPtr->query(); createTableFileQuery << "INSERT INTO " << META_TABLEFILES << " " << - "VALUES(" << id << ", " << quote << table_id << ", " << engine_type << ", " << - quote << file_id << ", " << file_type << ", " << file_size << ", " << + "VALUES(" << id << ", " << mysqlpp::quote << table_id << + ", " << engine_type << ", " << + mysqlpp::quote << file_id << ", " << file_type << ", " << file_size << ", " << row_count << ", " << updated_time << ", " << created_on << ", " << date << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); - if (SimpleResult res = createTableFileQuery.execute()) { + if (mysqlpp::SimpleResult res = createTableFileQuery.execute()) { file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? //Consume all results to avoid "Commands out of sync" error @@ -1018,29 +1011,31 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { ENGINE_LOG_DEBUG << "Successfully create table file, file id = " << file_schema.file_id_; return utils::CreateTableFilePath(options_, file_schema); - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN CREATING TABLE FILE", e.what()); } } -Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { +Status +MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { files.clear(); try { server::MetricCollector metric; - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query filesToIndexQuery = connectionPtr->query(); - filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " << - META_TABLEFILES << " " << - "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; + mysqlpp::Query filesToIndexQuery = connectionPtr->query(); + filesToIndexQuery + << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " + << + META_TABLEFILES << " " << + "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); @@ -1051,7 +1046,6 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { std::map groups; TableFileSchema table_file; for (auto &resRow : res) { - table_file.id_ = resRow["id"]; //implicit conversion std::string table_id; @@ -1083,7 +1077,6 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { return status; } groups[table_file.table_id_] = table_schema; - } table_file.dimension_ = groups[table_file.table_id_].dimension_; table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_; @@ -1091,43 +1084,44 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { table_file.metric_type_ = groups[table_file.table_id_].metric_type_; auto status = utils::GetTableFilePath(options_, table_file); - if(!status.ok()) { + if (!status.ok()) { ret = status; } files.push_back(table_file); } - if(res.size() > 0) { + if (res.size() > 0) { ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-index files"; } return ret; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", e.what()); } } -Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, - const std::vector &ids, - const DatesT &partition, - DatePartionedTableFilesSchema &files) { +Status +MySQLMetaImpl::FilesToSearch(const std::string &table_id, + const std::vector &ids, + const DatesT &partition, + DatePartionedTableFilesSchema &files) { files.clear(); try { server::MetricCollector metric; - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query filesToSearchQuery = connectionPtr->query(); - filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date FROM " << - META_TABLEFILES << " " << - "WHERE table_id = " << quote << table_id; + mysqlpp::Query filesToSearchQuery = connectionPtr->query(); + filesToSearchQuery + << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date FROM " << + META_TABLEFILES << " " << + "WHERE table_id = " << mysqlpp::quote << table_id; if (!partition.empty()) { std::stringstream partitionListSS; @@ -1148,8 +1142,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, std::string idStr = idSS.str(); idStr = idStr.substr(0, idStr.size() - 4); //remove the last " OR " - filesToSearchQuery << " AND " << "(" << idStr << ")"; - + filesToSearchQuery << " AND " << "(" << idStr << ")"; } // End filesToSearchQuery << " AND " << @@ -1172,7 +1165,6 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, Status ret; TableFileSchema table_file; for (auto &resRow : res) { - table_file.id_ = resRow["id"]; //implicit conversion std::string table_id_str; @@ -1202,7 +1194,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, table_file.dimension_ = table_schema.dimension_; auto status = utils::GetTableFilePath(options_, table_file); - if(!status.ok()) { + if (!status.ok()) { ret = status; } @@ -1214,7 +1206,7 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, files[table_file.date_].push_back(table_file); } - if(res.size() > 0) { + if (res.size() > 0) { ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-search files"; } return ret; @@ -1223,8 +1215,9 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, } } -Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, - DatePartionedTableFilesSchema &files) { +Status +MySQLMetaImpl::FilesToMerge(const std::string &table_id, + DatePartionedTableFilesSchema &files) { files.clear(); try { @@ -1238,20 +1231,22 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, return status; } - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query filesToMergeQuery = connectionPtr->query(); - filesToMergeQuery << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, created_on FROM " << - META_TABLEFILES << " " << - "WHERE table_id = " << quote << table_id << " AND " << - "file_type = " << std::to_string(TableFileSchema::RAW) << " " << - "ORDER BY row_count DESC" << ";"; + mysqlpp::Query filesToMergeQuery = connectionPtr->query(); + filesToMergeQuery + << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, created_on FROM " + << + META_TABLEFILES << " " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << + "file_type = " << std::to_string(TableFileSchema::RAW) << " " << + "ORDER BY row_count DESC" << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str(); @@ -1262,7 +1257,7 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, for (auto &resRow : res) { TableFileSchema table_file; table_file.file_size_ = resRow["file_size"]; - if(table_file.file_size_ >= table_schema.index_file_size_) { + if (table_file.file_size_ >= table_schema.index_file_size_) { continue;//skip large file } @@ -1295,7 +1290,7 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, table_file.dimension_ = table_schema.dimension_; auto status = utils::GetTableFilePath(options_, table_file); - if(!status.ok()) { + if (!status.ok()) { ret = status; } @@ -1307,19 +1302,19 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, files[table_file.date_].push_back(table_file); } - if(res.size() > 0) { + if (res.size() > 0) { ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-merge files"; } return ret; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", e.what()); } } -Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, - const std::vector &ids, - TableFilesSchema &table_files) { +Status +MySQLMetaImpl::GetTableFiles(const std::string &table_id, + const std::vector &ids, + TableFilesSchema &table_files) { if (ids.empty()) { return Status::OK(); } @@ -1332,20 +1327,21 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, idStr = idStr.substr(0, idStr.size() - 4); //remove the last " OR " try { - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query getTableFileQuery = connectionPtr->query(); - getTableFileQuery << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " << - META_TABLEFILES << " " << - "WHERE table_id = " << quote << table_id << " AND " << - "(" << idStr << ") AND " << - "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + mysqlpp::Query getTableFileQuery = connectionPtr->query(); + getTableFileQuery + << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " << + META_TABLEFILES << " " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << + "(" << idStr << ") AND " << + "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); @@ -1358,7 +1354,6 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, Status ret; for (auto &resRow : res) { - TableFileSchema file_schema; file_schema.id_ = resRow["id"]; @@ -1396,14 +1391,14 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, ENGINE_LOG_DEBUG << "Get table files by id"; return ret; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN RETRIEVING TABLE FILES", e.what()); } } // PXU TODO: Support Swap -Status MySQLMetaImpl::Archive() { +Status +MySQLMetaImpl::Archive() { auto &criterias = options_.archive_conf_.GetCriterias(); if (criterias.empty()) { return Status::OK(); @@ -1414,16 +1409,16 @@ Status MySQLMetaImpl::Archive() { auto &limit = kv.second; if (criteria == engine::ARCHIVE_CONF_DAYS) { size_t usecs = limit * D_SEC * US_PS; - long now = utils::GetMicroSecTimeStamp(); + int64_t now = utils::GetMicroSecTimeStamp(); try { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query archiveQuery = connectionPtr->query(); + mysqlpp::Query archiveQuery = connectionPtr->query(); archiveQuery << "UPDATE " << META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << @@ -1437,7 +1432,6 @@ Status MySQLMetaImpl::Archive() { } ENGINE_LOG_DEBUG << "Archive old files"; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN DURING ARCHIVE", e.what()); } @@ -1456,19 +1450,20 @@ Status MySQLMetaImpl::Archive() { return Status::OK(); } -Status MySQLMetaImpl::Size(uint64_t &result) { +Status +MySQLMetaImpl::Size(uint64_t &result) { result = 0; try { - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query getSizeQuery = connectionPtr->query(); + mysqlpp::Query getSizeQuery = connectionPtr->query(); getSizeQuery << "SELECT IFNULL(SUM(file_size),0) AS sum FROM " << META_TABLEFILES << " " << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; @@ -1483,7 +1478,6 @@ Status MySQLMetaImpl::Size(uint64_t &result) { } else { result = res[0]["sum"]; } - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN RETRIEVING SIZE", e.what()); } @@ -1491,9 +1485,9 @@ Status MySQLMetaImpl::Size(uint64_t &result) { return Status::OK(); } -Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { +Status +MySQLMetaImpl::DiscardFiles(int64_t to_discard_size) { if (to_discard_size <= 0) { - return Status::OK(); } ENGINE_LOG_DEBUG << "About to discard size=" << to_discard_size; @@ -1502,13 +1496,13 @@ Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { server::MetricCollector metric; bool status; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query discardFilesQuery = connectionPtr->query(); + mysqlpp::Query discardFilesQuery = connectionPtr->query(); discardFilesQuery << "SELECT id, file_size FROM " << META_TABLEFILES << " " << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " << @@ -1517,7 +1511,7 @@ Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); - StoreQueryResult res = discardFilesQuery.store(); + mysqlpp::StoreQueryResult res = discardFilesQuery.store(); if (res.num_rows() == 0) { return Status::OK(); } @@ -1554,36 +1548,36 @@ Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { } //Scoped Connection return DiscardFiles(to_discard_size); - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN DISCARDING FILES", e.what()); } } //ZR: this function assumes all fields in file_schema have value -Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { +Status +MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query updateTableFileQuery = connectionPtr->query(); + mysqlpp::Query updateTableFileQuery = connectionPtr->query(); //if the table has been deleted, just mark the table file as TO_DELETE //clean thread will delete the file later updateTableFileQuery << "SELECT state FROM " << META_TABLES << " " << - "WHERE table_id = " << quote << file_schema.table_id_ << ";"; + "WHERE table_id = " << mysqlpp::quote << file_schema.table_id_ << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); - StoreQueryResult res = updateTableFileQuery.store(); + mysqlpp::StoreQueryResult res = updateTableFileQuery.store(); if (res.num_rows() == 1) { int state = res[0]["state"]; @@ -1607,9 +1601,9 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { updateTableFileQuery << "UPDATE " << META_TABLEFILES << " " << - "SET table_id = " << quote << table_id << ", " << + "SET table_id = " << mysqlpp::quote << table_id << ", " << "engine_type = " << engine_type << ", " << - "file_id = " << quote << file_id << ", " << + "file_id = " << mysqlpp::quote << file_id << ", " << "file_type = " << file_type << ", " << "file_size = " << file_size << ", " << "row_count = " << row_count << ", " << @@ -1627,7 +1621,6 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { } //Scoped Connection ENGINE_LOG_DEBUG << "Update single table file, file id = " << file_schema.file_id_; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILE", e.what()); } @@ -1635,30 +1628,31 @@ Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { return Status::OK(); } -Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { +Status +MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { try { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query updateTableFilesToIndexQuery = connectionPtr->query(); + mysqlpp::Query updateTableFilesToIndexQuery = connectionPtr->query(); updateTableFilesToIndexQuery << "UPDATE " << META_TABLEFILES << " " << "SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " " << - "WHERE table_id = " << quote << table_id << " AND " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << "file_type = " << std::to_string(TableFileSchema::RAW) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFilesToIndex: " << updateTableFilesToIndexQuery.str(); if (!updateTableFilesToIndexQuery.exec()) { - return HandleException("QUERY ERROR WHEN UPDATING TABLE FILE TO INDEX", updateTableFilesToIndexQuery.error()); + return HandleException("QUERY ERROR WHEN UPDATING TABLE FILE TO INDEX", + updateTableFilesToIndexQuery.error()); } ENGINE_LOG_DEBUG << "Update files to to_index, table id = " << table_id; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES TO INDEX", e.what()); } @@ -1666,21 +1660,21 @@ Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { return Status::OK(); } -Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { +Status +MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { try { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query updateTableFilesQuery = connectionPtr->query(); + mysqlpp::Query updateTableFilesQuery = connectionPtr->query(); std::map has_tables; for (auto &file_schema : files) { - if (has_tables.find(file_schema.table_id_) != has_tables.end()) { continue; } @@ -1688,20 +1682,19 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { updateTableFilesQuery << "SELECT EXISTS " << "(SELECT 1 FROM " << META_TABLES << " " << - "WHERE table_id = " << quote << file_schema.table_id_ << " " << + "WHERE table_id = " << mysqlpp::quote << file_schema.table_id_ << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << - "AS " << quote << "check" << ";"; + "AS " << mysqlpp::quote << "check" << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); - StoreQueryResult res = updateTableFilesQuery.store(); + mysqlpp::StoreQueryResult res = updateTableFilesQuery.store(); int check = res[0]["check"]; has_tables[file_schema.table_id_] = (check == 1); } for (auto &file_schema : files) { - if (!has_tables[file_schema.table_id_]) { file_schema.file_type_ = TableFileSchema::TO_DELETE; } @@ -1720,9 +1713,9 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { updateTableFilesQuery << "UPDATE " << META_TABLEFILES << " " << - "SET table_id = " << quote << table_id << ", " << + "SET table_id = " << mysqlpp::quote << table_id << ", " << "engine_type = " << engine_type << ", " << - "file_id = " << quote << file_id << ", " << + "file_id = " << mysqlpp::quote << file_id << ", " << "file_type = " << file_type << ", " << "file_size = " << file_size << ", " << "row_count = " << row_count << ", " << @@ -1740,7 +1733,6 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { } //Scoped Connection ENGINE_LOG_DEBUG << "Update " << files.size() << " table files"; - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES", e.what()); } @@ -1748,7 +1740,8 @@ Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { return Status::OK(); } -Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { +Status +MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { auto now = utils::GetMicroSecTimeStamp(); std::set table_ids; @@ -1757,13 +1750,13 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query cleanUpFilesWithTTLQuery = connectionPtr->query(); + mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date FROM " << META_TABLEFILES << " " << "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << @@ -1771,13 +1764,12 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + mysqlpp::StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); TableFileSchema table_file; std::vector idsToDelete; for (auto &resRow : res) { - table_file.id_ = resRow["id"]; //implicit conversion std::string table_id; @@ -1800,7 +1792,6 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } if (!idsToDelete.empty()) { - std::stringstream idsToDeleteSS; for (auto &id : idsToDelete) { idsToDeleteSS << "id = " << id << " OR "; @@ -1815,16 +1806,15 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); if (!cleanUpFilesWithTTLQuery.exec()) { - return HandleException("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", cleanUpFilesWithTTLQuery.error()); + return HandleException("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", + cleanUpFilesWithTTLQuery.error()); } } - if(res.size() > 0) { + if (res.size() > 0) { ENGINE_LOG_DEBUG << "Clean " << res.size() << " files deleted in " << seconds << " seconds"; } - } //Scoped Connection - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", e.what()); } @@ -1834,23 +1824,22 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query cleanUpFilesWithTTLQuery = connectionPtr->query(); + mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); cleanUpFilesWithTTLQuery << "SELECT id, table_id FROM " << META_TABLES << " " << "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + mysqlpp::StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); if (!res.empty()) { - std::stringstream idsToDeleteSS; for (auto &resRow : res) { size_t id = resRow["id"]; @@ -1870,15 +1859,15 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); if (!cleanUpFilesWithTTLQuery.exec()) { - return HandleException("QUERY ERROR WHEN CLEANING UP TABLES WITH TTL", cleanUpFilesWithTTLQuery.error()); + return HandleException("QUERY ERROR WHEN CLEANING UP TABLES WITH TTL", + cleanUpFilesWithTTLQuery.error()); } } - if(res.size() > 0) { + if (res.size() > 0) { ENGINE_LOG_DEBUG << "Remove " << res.size() << " tables from meta"; } } //Scoped Connection - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what()); } @@ -1889,28 +1878,28 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { server::MetricCollector metric; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - for(auto& table_id : table_ids) { - Query cleanUpFilesWithTTLQuery = connectionPtr->query(); + for (auto &table_id : table_ids) { + mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); cleanUpFilesWithTTLQuery << "SELECT file_id FROM " << META_TABLEFILES << " " << - "WHERE table_id = " << quote << table_id << ";"; + "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + mysqlpp::StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); if (res.empty()) { utils::DeleteTablePath(options_, table_id); } } - if(table_ids.size() > 0) { + if (table_ids.size() > 0) { ENGINE_LOG_DEBUG << "Remove " << table_ids.size() << " tables folder"; } } @@ -1921,30 +1910,31 @@ Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return Status::OK(); } -Status MySQLMetaImpl::CleanUp() { +Status +MySQLMetaImpl::CleanUp() { try { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query cleanUpQuery = connectionPtr->query(); + mysqlpp::Query cleanUpQuery = connectionPtr->query(); cleanUpQuery << "SELECT table_name " << "FROM information_schema.tables " << - "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << " " << - "AND table_name = " << quote << META_TABLEFILES << ";"; + "WHERE table_schema = " << mysqlpp::quote << mysql_connection_pool_->getDB() << " " << + "AND table_name = " << mysqlpp::quote << META_TABLEFILES << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); - StoreQueryResult res = cleanUpQuery.store(); + mysqlpp::StoreQueryResult res = cleanUpQuery.store(); if (!res.empty()) { ENGINE_LOG_DEBUG << "Remove table file type as NEW"; cleanUpQuery << "DELETE FROM " << META_TABLEFILES << " WHERE file_type IN (" - << std::to_string(TableFileSchema::NEW) << "," - << std::to_string(TableFileSchema::NEW_MERGE) << "," - << std::to_string(TableFileSchema::NEW_INDEX) << ");"; + << std::to_string(TableFileSchema::NEW) << "," + << std::to_string(TableFileSchema::NEW_MERGE) << "," + << std::to_string(TableFileSchema::NEW_INDEX) << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); @@ -1953,7 +1943,7 @@ Status MySQLMetaImpl::CleanUp() { } } - if(res.size() > 0) { + if (res.size() > 0) { ENGINE_LOG_DEBUG << "Clean " << res.size() << " files"; } } catch (std::exception &e) { @@ -1963,7 +1953,8 @@ Status MySQLMetaImpl::CleanUp() { return Status::OK(); } -Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { +Status +MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { try { server::MetricCollector metric; @@ -1975,19 +1966,18 @@ Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { return status; } - StoreQueryResult res; + mysqlpp::StoreQueryResult res; { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - - Query countQuery = connectionPtr->query(); + mysqlpp::Query countQuery = connectionPtr->query(); countQuery << "SELECT row_count FROM " << META_TABLEFILES << " " << - "WHERE table_id = " << quote << table_id << " AND " << + "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; @@ -2002,7 +1992,6 @@ Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { size_t size = resRow["row_count"]; result += size; } - } catch (std::exception &e) { return HandleException("GENERAL ERROR WHEN RETRIEVING COUNT", e.what()); } @@ -2010,16 +1999,17 @@ Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { return Status::OK(); } -Status MySQLMetaImpl::DropAll() { +Status +MySQLMetaImpl::DropAll() { try { ENGINE_LOG_DEBUG << "Drop all mysql meta"; - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { return Status(DB_ERROR, "Failed to connect to database server"); } - Query dropTableQuery = connectionPtr->query(); + mysqlpp::Query dropTableQuery = connectionPtr->query(); dropTableQuery << "DROP TABLE IF EXISTS " << TABLES_SCHEMA.name() << ", " << TABLEFILES_SCHEMA.name() << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str(); diff --git a/cpp/src/db/meta/MySQLMetaImpl.h b/cpp/src/db/meta/MySQLMetaImpl.h index b845b45a8c..d9d67fd748 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.h +++ b/cpp/src/db/meta/MySQLMetaImpl.h @@ -21,26 +21,25 @@ #include "db/Options.h" #include "MySQLConnectionPool.h" -#include "mysql++/mysql++.h" +#include #include - +#include +#include +#include namespace zilliz { namespace milvus { namespace engine { namespace meta { -// auto StoragePrototype(const std::string& path); -using namespace mysqlpp; - class MySQLMetaImpl : public Meta { public: - MySQLMetaImpl(const DBMetaOptions &options_, const int &mode); + MySQLMetaImpl(const DBMetaOptions &options, const int &mode); ~MySQLMetaImpl(); Status CreateTable(TableSchema &table_schema) override; - Status DescribeTable(TableSchema &group_info_) override; + Status DescribeTable(TableSchema &table_schema) override; Status HasTable(const std::string &table_id, bool &has_or_not) override; @@ -63,11 +62,11 @@ class MySQLMetaImpl : public Meta { const std::vector &file_types, std::vector &file_ids) override; - Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) override; + Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) override; Status UpdateTableFlag(const std::string &table_id, int64_t flag) override; - Status DescribeTableIndex(const std::string &table_id, TableIndex& index) override; + Status DescribeTableIndex(const std::string &table_id, TableIndex &index) override; Status DropTableIndex(const std::string &table_id) override; @@ -102,12 +101,12 @@ class MySQLMetaImpl : public Meta { private: Status NextFileId(std::string &file_id); Status NextTableId(std::string &table_id); - Status DiscardFiles(long long to_discard_size); + Status DiscardFiles(int64_t to_discard_size); void ValidateMetaSchema(); Status Initialize(); -private: + private: const DBMetaOptions options_; const int mode_; diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 1fe7e3d76c..471b182981 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "SqliteMetaImpl.h" +#include "db/meta/SqliteMetaImpl.h" #include "db/IDGenerator.h" #include "db/Utils.h" #include "utils/Log.h" @@ -29,9 +29,11 @@ #include #include #include +#include +#include +#include #include - namespace zilliz { namespace milvus { namespace engine { @@ -41,8 +43,9 @@ using namespace sqlite_orm; namespace { -Status HandleException(const std::string &desc, const char* what = nullptr) { - if(what == nullptr) { +Status +HandleException(const std::string &desc, const char *what = nullptr) { + if (what == nullptr) { ENGINE_LOG_ERROR << desc; return Status(DB_META_TRANSACTION_FAILED, desc); } else { @@ -52,9 +55,10 @@ Status HandleException(const std::string &desc, const char* what = nullptr) { } } -} +} // namespace -inline auto StoragePrototype(const std::string &path) { +inline auto +StoragePrototype(const std::string &path) { return make_storage(path, make_table(META_TABLES, make_column("id", &TableSchema::id_, primary_key()), @@ -77,24 +81,22 @@ inline auto StoragePrototype(const std::string &path) { make_column("row_count", &TableFileSchema::row_count_, default_value(0)), make_column("updated_time", &TableFileSchema::updated_time_), make_column("created_on", &TableFileSchema::created_on_), - make_column("date", &TableFileSchema::date_)) - ); - + make_column("date", &TableFileSchema::date_))); } using ConnectorT = decltype(StoragePrototype("")); static std::unique_ptr ConnectorPtr; -SqliteMetaImpl::SqliteMetaImpl(const DBMetaOptions &options_) - : options_(options_) { +SqliteMetaImpl::SqliteMetaImpl(const DBMetaOptions &options) + : options_(options) { Initialize(); } SqliteMetaImpl::~SqliteMetaImpl() { - } -Status SqliteMetaImpl::NextTableId(std::string &table_id) { +Status +SqliteMetaImpl::NextTableId(std::string &table_id) { std::stringstream ss; SimpleIDGenerator g; ss << g.GetNextIDNumber(); @@ -102,7 +104,8 @@ Status SqliteMetaImpl::NextTableId(std::string &table_id) { return Status::OK(); } -Status SqliteMetaImpl::NextFileId(std::string &file_id) { +Status +SqliteMetaImpl::NextFileId(std::string &file_id) { std::stringstream ss; SimpleIDGenerator g; ss << g.GetNextIDNumber(); @@ -110,24 +113,26 @@ Status SqliteMetaImpl::NextFileId(std::string &file_id) { return Status::OK(); } -void SqliteMetaImpl::ValidateMetaSchema() { - if(ConnectorPtr == nullptr) { +void +SqliteMetaImpl::ValidateMetaSchema() { + if (ConnectorPtr == nullptr) { return; } //old meta could be recreated since schema changed, throw exception if meta schema is not compatible auto ret = ConnectorPtr->sync_schema_simulate(); - if(ret.find(META_TABLES) != ret.end() - && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLES]) { + if (ret.find(META_TABLES) != ret.end() + && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLES]) { throw Exception(DB_INCOMPATIB_META, "Meta Tables schema is created by Milvus old version"); } - if(ret.find(META_TABLEFILES) != ret.end() - && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLEFILES]) { + if (ret.find(META_TABLEFILES) != ret.end() + && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLEFILES]) { throw Exception(DB_INCOMPATIB_META, "Meta TableFiles schema is created by Milvus old version"); } } -Status SqliteMetaImpl::Initialize() { +Status +SqliteMetaImpl::Initialize() { if (!boost::filesystem::is_directory(options_.path_)) { auto ret = boost::filesystem::create_directory(options_.path_); if (!ret) { @@ -151,8 +156,9 @@ Status SqliteMetaImpl::Initialize() { } // PXU TODO: Temp solution. Will fix later -Status SqliteMetaImpl::DropPartitionsByDates(const std::string &table_id, - const DatesT &dates) { +Status +SqliteMetaImpl::DropPartitionsByDates(const std::string &table_id, + const DatesT &dates) { if (dates.size() == 0) { return Status::OK(); } @@ -171,15 +177,12 @@ Status SqliteMetaImpl::DropPartitionsByDates(const std::string &table_id, ConnectorPtr->update_all( set( c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, - c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp() - ), + c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), where( c(&TableFileSchema::table_id_) == table_id and - in(&TableFileSchema::date_, dates) - )); + in(&TableFileSchema::date_, dates))); ENGINE_LOG_DEBUG << "Successfully drop partitions, table id = " << table_schema.table_id_; - } catch (std::exception &e) { return HandleException("Encounter exception when drop partition", e.what()); } @@ -187,8 +190,8 @@ Status SqliteMetaImpl::DropPartitionsByDates(const std::string &table_id, return Status::OK(); } -Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) { - +Status +SqliteMetaImpl::CreateTable(TableSchema &table_schema) { try { server::MetricCollector metric; @@ -199,9 +202,9 @@ Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) { NextTableId(table_schema.table_id_); } else { auto table = ConnectorPtr->select(columns(&TableSchema::state_), - where(c(&TableSchema::table_id_) == table_schema.table_id_)); + where(c(&TableSchema::table_id_) == table_schema.table_id_)); if (table.size() == 1) { - if(TableSchema::TO_DELETE == std::get<0>(table[0])) { + if (TableSchema::TO_DELETE == std::get<0>(table[0])) { return Status(DB_ERROR, "Table already exists and it is in delete state, please wait a second"); } else { // Change from no error to already exist. @@ -223,13 +226,13 @@ Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) { ENGINE_LOG_DEBUG << "Successfully create table: " << table_schema.table_id_; return utils::CreateTablePath(options_, table_schema.table_id_); - } catch (std::exception &e) { return HandleException("Encounter exception when create table", e.what()); } } -Status SqliteMetaImpl::DeleteTable(const std::string& table_id) { +Status +SqliteMetaImpl::DeleteTable(const std::string &table_id) { try { server::MetricCollector metric; @@ -238,16 +241,13 @@ Status SqliteMetaImpl::DeleteTable(const std::string& table_id) { //soft delete table ConnectorPtr->update_all( - set( - c(&TableSchema::state_) = (int) TableSchema::TO_DELETE - ), - where( - c(&TableSchema::table_id_) == table_id and - c(&TableSchema::state_) != (int) TableSchema::TO_DELETE - )); + set( + c(&TableSchema::state_) = (int) TableSchema::TO_DELETE), + where( + c(&TableSchema::table_id_) == table_id and + c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); ENGINE_LOG_DEBUG << "Successfully delete table, table id = " << table_id; - } catch (std::exception &e) { return HandleException("Encounter exception when delete table", e.what()); } @@ -255,7 +255,8 @@ Status SqliteMetaImpl::DeleteTable(const std::string& table_id) { return Status::OK(); } -Status SqliteMetaImpl::DeleteTableFiles(const std::string& table_id) { +Status +SqliteMetaImpl::DeleteTableFiles(const std::string &table_id) { try { server::MetricCollector metric; @@ -264,17 +265,14 @@ Status SqliteMetaImpl::DeleteTableFiles(const std::string& table_id) { //soft delete table files ConnectorPtr->update_all( - set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, - c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp() - ), - where( - c(&TableFileSchema::table_id_) == table_id and - c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE - )); + set( + c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, + c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), + where( + c(&TableFileSchema::table_id_) == table_id and + c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE)); ENGINE_LOG_DEBUG << "Successfully delete table files, table id = " << table_id; - } catch (std::exception &e) { return HandleException("Encounter exception when delete table files", e.what()); } @@ -282,7 +280,8 @@ Status SqliteMetaImpl::DeleteTableFiles(const std::string& table_id) { return Status::OK(); } -Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { +Status +SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { try { server::MetricCollector metric; @@ -296,7 +295,7 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { &TableSchema::nlist_, &TableSchema::metric_type_), where(c(&TableSchema::table_id_) == table_schema.table_id_ - and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); + and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); if (groups.size() == 1) { table_schema.id_ = std::get<0>(groups[0]); @@ -311,7 +310,6 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { } else { return Status(DB_NOT_FOUND, "Table " + table_schema.table_id_ + " not found"); } - } catch (std::exception &e) { return HandleException("Encounter exception when describe table", e.what()); } @@ -319,10 +317,11 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { return Status::OK(); } -Status SqliteMetaImpl::FilesByType(const std::string& table_id, - const std::vector& file_types, - std::vector& file_ids) { - if(file_types.empty()) { +Status +SqliteMetaImpl::FilesByType(const std::string &table_id, + const std::vector &file_types, + std::vector &file_ids) { + if (file_types.empty()) { return Status(DB_ERROR, "file types array is empty"); } @@ -331,8 +330,7 @@ Status SqliteMetaImpl::FilesByType(const std::string& table_id, auto selected = ConnectorPtr->select(columns(&TableFileSchema::file_id_, &TableFileSchema::file_type_), where(in(&TableFileSchema::file_type_, file_types) - and c(&TableFileSchema::table_id_) == table_id - )); + and c(&TableFileSchema::table_id_) == table_id)); if (selected.size() >= 1) { int raw_count = 0, new_count = 0, new_merge_count = 0, new_index_count = 0; @@ -340,29 +338,21 @@ Status SqliteMetaImpl::FilesByType(const std::string& table_id, for (auto &file : selected) { file_ids.push_back(std::get<0>(file)); switch (std::get<1>(file)) { - case (int) TableFileSchema::RAW: - raw_count++; + case (int) TableFileSchema::RAW:raw_count++; break; - case (int) TableFileSchema::NEW: - new_count++; + case (int) TableFileSchema::NEW:new_count++; break; - case (int) TableFileSchema::NEW_MERGE: - new_merge_count++; + case (int) TableFileSchema::NEW_MERGE:new_merge_count++; break; - case (int) TableFileSchema::NEW_INDEX: - new_index_count++; + case (int) TableFileSchema::NEW_INDEX:new_index_count++; break; - case (int) TableFileSchema::TO_INDEX: - to_index_count++; + case (int) TableFileSchema::TO_INDEX:to_index_count++; break; - case (int) TableFileSchema::INDEX: - index_count++; + case (int) TableFileSchema::INDEX:index_count++; break; - case (int) TableFileSchema::BACKUP: - backup_count++; - break; - default: + case (int) TableFileSchema::BACKUP:backup_count++; break; + default:break; } } @@ -371,14 +361,14 @@ Status SqliteMetaImpl::FilesByType(const std::string& table_id, << " new_index files:" << new_index_count << " to_index files:" << to_index_count << " index files:" << index_count << " backup files:" << backup_count; } - } catch (std::exception &e) { return HandleException("Encounter exception when check non index files", e.what()); } return Status::OK(); } -Status SqliteMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex& index) { +Status +SqliteMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex &index) { try { server::MetricCollector metric; @@ -392,9 +382,9 @@ Status SqliteMetaImpl::UpdateTableIndex(const std::string &table_id, const Table &TableSchema::flag_, &TableSchema::index_file_size_), where(c(&TableSchema::table_id_) == table_id - and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); + and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); - if(tables.size() > 0) { + if (tables.size() > 0) { meta::TableSchema table_schema; table_schema.id_ = std::get<0>(tables[0]); table_schema.table_id_ = table_id; @@ -414,17 +404,14 @@ Status SqliteMetaImpl::UpdateTableIndex(const std::string &table_id, const Table //set all backup file to raw ConnectorPtr->update_all( - set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::RAW, - c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp() - ), - where( - c(&TableFileSchema::table_id_) == table_id and - c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP - )); + set( + c(&TableFileSchema::file_type_) = (int) TableFileSchema::RAW, + c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), + where( + c(&TableFileSchema::table_id_) == table_id and + c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP)); ENGINE_LOG_DEBUG << "Successfully update table index, table id = " << table_id; - } catch (std::exception &e) { std::string msg = "Encounter exception when update table index: table_id = " + table_id; return HandleException(msg, e.what()); @@ -433,20 +420,18 @@ Status SqliteMetaImpl::UpdateTableIndex(const std::string &table_id, const Table return Status::OK(); } -Status SqliteMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { +Status +SqliteMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { try { server::MetricCollector metric; //set all backup file to raw ConnectorPtr->update_all( - set( - c(&TableSchema::flag_) = flag - ), - where( - c(&TableSchema::table_id_) == table_id - )); + set( + c(&TableSchema::flag_) = flag), + where( + c(&TableSchema::table_id_) == table_id)); ENGINE_LOG_DEBUG << "Successfully update table flag, table id = " << table_id; - } catch (std::exception &e) { std::string msg = "Encounter exception when update table flag: table_id = " + table_id; return HandleException(msg, e.what()); @@ -455,7 +440,8 @@ Status SqliteMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag return Status::OK(); } -Status SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex& index) { +Status +SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex &index) { try { server::MetricCollector metric; @@ -463,7 +449,7 @@ Status SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableInde &TableSchema::nlist_, &TableSchema::metric_type_), where(c(&TableSchema::table_id_) == table_id - and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); + and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); if (groups.size() == 1) { index.engine_type_ = std::get<0>(groups[0]); @@ -472,7 +458,6 @@ Status SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableInde } else { return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); } - } catch (std::exception &e) { return HandleException("Encounter exception when describe index", e.what()); } @@ -480,7 +465,8 @@ Status SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableInde return Status::OK(); } -Status SqliteMetaImpl::DropTableIndex(const std::string &table_id) { +Status +SqliteMetaImpl::DropTableIndex(const std::string &table_id) { try { server::MetricCollector metric; @@ -489,39 +475,32 @@ Status SqliteMetaImpl::DropTableIndex(const std::string &table_id) { //soft delete index files ConnectorPtr->update_all( - set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, - c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp() - ), - where( - c(&TableFileSchema::table_id_) == table_id and - c(&TableFileSchema::file_type_) == (int) TableFileSchema::INDEX - )); + set( + c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, + c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), + where( + c(&TableFileSchema::table_id_) == table_id and + c(&TableFileSchema::file_type_) == (int) TableFileSchema::INDEX)); //set all backup file to raw ConnectorPtr->update_all( - set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::RAW, - c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp() - ), - where( - c(&TableFileSchema::table_id_) == table_id and - c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP - )); + set( + c(&TableFileSchema::file_type_) = (int) TableFileSchema::RAW, + c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), + where( + c(&TableFileSchema::table_id_) == table_id and + c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP)); //set table index type to raw ConnectorPtr->update_all( - set( - c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE, - c(&TableSchema::nlist_) = DEFAULT_NLIST, - c(&TableSchema::metric_type_) = DEFAULT_METRIC_TYPE - ), - where( - c(&TableSchema::table_id_) == table_id - )); + set( + c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE, + c(&TableSchema::nlist_) = DEFAULT_NLIST, + c(&TableSchema::metric_type_) = DEFAULT_METRIC_TYPE), + where( + c(&TableSchema::table_id_) == table_id)); ENGINE_LOG_DEBUG << "Successfully drop table index, table id = " << table_id; - } catch (std::exception &e) { return HandleException("Encounter exception when delete table index files", e.what()); } @@ -529,20 +508,20 @@ Status SqliteMetaImpl::DropTableIndex(const std::string &table_id) { return Status::OK(); } -Status SqliteMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { +Status +SqliteMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { has_or_not = false; try { server::MetricCollector metric; auto tables = ConnectorPtr->select(columns(&TableSchema::id_), where(c(&TableSchema::table_id_) == table_id - and c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); + and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); if (tables.size() == 1) { has_or_not = true; } else { has_or_not = false; } - } catch (std::exception &e) { return HandleException("Encounter exception when lookup table", e.what()); } @@ -550,7 +529,8 @@ Status SqliteMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { return Status::OK(); } -Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { +Status +SqliteMetaImpl::AllTables(std::vector &table_schema_array) { try { server::MetricCollector metric; @@ -563,7 +543,7 @@ Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { &TableSchema::engine_type_, &TableSchema::nlist_, &TableSchema::metric_type_), - where(c(&TableSchema::state_) != (int)TableSchema::TO_DELETE)); + where(c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); for (auto &table : selected) { TableSchema schema; schema.id_ = std::get<0>(table); @@ -578,7 +558,6 @@ Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { table_schema_array.emplace_back(schema); } - } catch (std::exception &e) { return HandleException("Encounter exception when lookup all tables", e.what()); } @@ -586,7 +565,8 @@ Status SqliteMetaImpl::AllTables(std::vector& table_schema_array) { return Status::OK(); } -Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { +Status +SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { if (file_schema.date_ == EmptyDate) { file_schema.date_ = utils::GetDate(); } @@ -619,15 +599,15 @@ Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { ENGINE_LOG_DEBUG << "Successfully create table file, file id = " << file_schema.file_id_; return utils::CreateTableFilePath(options_, file_schema); - - } catch (std::exception& e) { + } catch (std::exception &e) { return HandleException("Encounter exception when create table file", e.what()); } return Status::OK(); } -Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { +Status +SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { files.clear(); try { @@ -661,7 +641,7 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { table_file.created_on_ = std::get<8>(file); auto status = utils::GetTableFilePath(options_, table_file); - if(!status.ok()) { + if (!status.ok()) { ret = status; } auto groupItr = groups.find(table_file.table_id_); @@ -681,20 +661,20 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { files.push_back(table_file); } - if(selected.size() > 0) { + if (selected.size() > 0) { ENGINE_LOG_DEBUG << "Collect " << selected.size() << " to-index files"; } return ret; - } catch (std::exception &e) { return HandleException("Encounter exception when iterate raw files", e.what()); } } -Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, - const std::vector &ids, - const DatesT &partition, - DatePartionedTableFilesSchema &files) { +Status +SqliteMetaImpl::FilesToSearch(const std::string &table_id, + const std::vector &ids, + const DatesT &partition, + DatePartionedTableFilesSchema &files) { files.clear(); server::MetricCollector metric; @@ -711,9 +691,9 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, auto match_tableid = c(&TableFileSchema::table_id_) == table_id; std::vector file_types = { - (int) TableFileSchema::RAW, - (int) TableFileSchema::TO_INDEX, - (int) TableFileSchema::INDEX + (int) TableFileSchema::RAW, + (int) TableFileSchema::TO_INDEX, + (int) TableFileSchema::INDEX }; auto match_type = in(&TableFileSchema::file_type_, file_types); @@ -726,18 +706,15 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, if (partition.empty() && ids.empty()) { auto filter = where(match_tableid and match_type); selected = ConnectorPtr->select(select_columns, filter); - } - else if (partition.empty() && !ids.empty()) { + } else if (partition.empty() && !ids.empty()) { auto match_fileid = in(&TableFileSchema::id_, ids); auto filter = where(match_tableid and match_fileid and match_type); selected = ConnectorPtr->select(select_columns, filter); - } - else if (!partition.empty() && ids.empty()) { + } else if (!partition.empty() && ids.empty()) { auto match_date = in(&TableFileSchema::date_, partition); auto filter = where(match_tableid and match_date and match_type); selected = ConnectorPtr->select(select_columns, filter); - } - else if (!partition.empty() && !ids.empty()) { + } else if (!partition.empty() && !ids.empty()) { auto match_fileid = in(&TableFileSchema::id_, ids); auto match_date = in(&TableFileSchema::date_, partition); auto filter = where(match_tableid and match_fileid and match_date and match_type); @@ -761,7 +738,7 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, table_file.metric_type_ = table_schema.metric_type_; auto status = utils::GetTableFilePath(options_, table_file); - if(!status.ok()) { + if (!status.ok()) { ret = status; } @@ -771,22 +748,22 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, } files[table_file.date_].push_back(table_file); } - if(files.empty()) { + if (files.empty()) { ENGINE_LOG_ERROR << "No file to search for table: " << table_id; } - if(selected.size() > 0) { + if (selected.size() > 0) { ENGINE_LOG_DEBUG << "Collect " << selected.size() << " to-search files"; } return ret; - } catch (std::exception &e) { return HandleException("Encounter exception when iterate index files", e.what()); } } -Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, - DatePartionedTableFilesSchema &files) { +Status +SqliteMetaImpl::FilesToMerge(const std::string &table_id, + DatePartionedTableFilesSchema &files) { files.clear(); try { @@ -817,7 +794,7 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, for (auto &file : selected) { TableFileSchema table_file; table_file.file_size_ = std::get<4>(file); - if(table_file.file_size_ >= table_schema.index_file_size_) { + if (table_file.file_size_ >= table_schema.index_file_size_) { continue;//skip large file } @@ -834,7 +811,7 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, table_file.metric_type_ = table_schema.metric_type_; auto status = utils::GetTableFilePath(options_, table_file); - if(!status.ok()) { + if (!status.ok()) { result = status; } @@ -845,19 +822,19 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, files[table_file.date_].push_back(table_file); } - if(selected.size() > 0) { + if (selected.size() > 0) { ENGINE_LOG_DEBUG << "Collect " << selected.size() << " to-merge files"; } return result; - } catch (std::exception &e) { return HandleException("Encounter exception when iterate merge files", e.what()); } } -Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, - const std::vector& ids, - TableFilesSchema& table_files) { +Status +SqliteMetaImpl::GetTableFiles(const std::string &table_id, + const std::vector &ids, + TableFilesSchema &table_files) { try { table_files.clear(); auto files = ConnectorPtr->select(columns(&TableFileSchema::id_, @@ -869,9 +846,8 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, &TableFileSchema::engine_type_, &TableFileSchema::created_on_), where(c(&TableFileSchema::table_id_) == table_id and - in(&TableFileSchema::id_, ids) and - c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE - )); + in(&TableFileSchema::id_, ids) and + c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE)); TableSchema table_schema; table_schema.table_id_ = table_id; @@ -910,7 +886,8 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, } // PXU TODO: Support Swap -Status SqliteMetaImpl::Archive() { +Status +SqliteMetaImpl::Archive() { auto &criterias = options_.archive_conf_.GetCriterias(); if (criterias.size() == 0) { return Status::OK(); @@ -920,20 +897,18 @@ Status SqliteMetaImpl::Archive() { auto &criteria = kv.first; auto &limit = kv.second; if (criteria == engine::ARCHIVE_CONF_DAYS) { - long usecs = limit * D_SEC * US_PS; - long now = utils::GetMicroSecTimeStamp(); + int64_t usecs = limit * D_SEC * US_PS; + int64_t now = utils::GetMicroSecTimeStamp(); try { //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here std::lock_guard meta_lock(meta_mutex_); ConnectorPtr->update_all( set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE - ), + c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE), where( - c(&TableFileSchema::created_on_) < (long) (now - usecs) and - c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE - )); + c(&TableFileSchema::created_on_) < (int64_t) (now - usecs) and + c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE)); } catch (std::exception &e) { return HandleException("Encounter exception when update table files", e.what()); } @@ -944,7 +919,7 @@ Status SqliteMetaImpl::Archive() { uint64_t sum = 0; Size(sum); - int64_t to_delete = (int64_t)sum - limit * G; + int64_t to_delete = (int64_t) sum - limit * G; DiscardFiles(to_delete); ENGINE_LOG_DEBUG << "Archive files to free disk"; @@ -954,20 +929,19 @@ Status SqliteMetaImpl::Archive() { return Status::OK(); } -Status SqliteMetaImpl::Size(uint64_t &result) { +Status +SqliteMetaImpl::Size(uint64_t &result) { result = 0; try { auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::file_size_)), - where( - c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE - )); + where( + c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE)); for (auto &total_size : selected) { if (!std::get<0>(total_size)) { continue; } result += (uint64_t) (*std::get<0>(total_size)); } - } catch (std::exception &e) { return HandleException("Encounter exception when calculte db size", e.what()); } @@ -975,7 +949,8 @@ Status SqliteMetaImpl::Size(uint64_t &result) { return Status::OK(); } -Status SqliteMetaImpl::DiscardFiles(long to_discard_size) { +Status +SqliteMetaImpl::DiscardFiles(int64_t to_discard_size) { if (to_discard_size <= 0) { return Status::OK(); } @@ -992,7 +967,7 @@ Status SqliteMetaImpl::DiscardFiles(long to_discard_size) { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, &TableFileSchema::file_size_), where(c(&TableFileSchema::file_type_) - != (int) TableFileSchema::TO_DELETE), + != (int) TableFileSchema::TO_DELETE), order_by(&TableFileSchema::id_), limit(10)); @@ -1014,13 +989,11 @@ Status SqliteMetaImpl::DiscardFiles(long to_discard_size) { } ConnectorPtr->update_all( - set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, - c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp() - ), - where( - in(&TableFileSchema::id_, ids) - )); + set( + c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, + c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), + where( + in(&TableFileSchema::id_, ids))); return true; }); @@ -1028,7 +1001,6 @@ Status SqliteMetaImpl::DiscardFiles(long to_discard_size) { if (!commited) { return HandleException("DiscardFiles error: sqlite transaction failed"); } - } catch (std::exception &e) { return HandleException("Encounter exception when discard table file", e.what()); } @@ -1036,7 +1008,8 @@ Status SqliteMetaImpl::DiscardFiles(long to_discard_size) { return DiscardFiles(to_discard_size); } -Status SqliteMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { +Status +SqliteMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); try { server::MetricCollector metric; @@ -1049,14 +1022,13 @@ Status SqliteMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { //if the table has been deleted, just mark the table file as TO_DELETE //clean thread will delete the file later - if(tables.size() < 1 || std::get<0>(tables[0]) == (int)TableSchema::TO_DELETE) { + if (tables.size() < 1 || std::get<0>(tables[0]) == (int) TableSchema::TO_DELETE) { file_schema.file_type_ = TableFileSchema::TO_DELETE; } ConnectorPtr->update(file_schema); ENGINE_LOG_DEBUG << "Update single table file, file id = " << file_schema.file_id_; - } catch (std::exception &e) { std::string msg = "Exception update table file: table_id = " + file_schema.table_id_ + " file_id = " + file_schema.file_id_; @@ -1065,7 +1037,8 @@ Status SqliteMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { return Status::OK(); } -Status SqliteMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { +Status +SqliteMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { try { server::MetricCollector metric; @@ -1074,15 +1047,12 @@ Status SqliteMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { ConnectorPtr->update_all( set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_INDEX - ), + c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_INDEX), where( c(&TableFileSchema::table_id_) == table_id and - c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW - )); + c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW)); ENGINE_LOG_DEBUG << "Update files to to_index, table id = " << table_id; - } catch (std::exception &e) { return HandleException("Encounter exception when update table files to to_index", e.what()); } @@ -1090,7 +1060,8 @@ Status SqliteMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { return Status::OK(); } -Status SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { +Status +SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { try { server::MetricCollector metric; @@ -1099,13 +1070,13 @@ Status SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { std::map has_tables; for (auto &file : files) { - if(has_tables.find(file.table_id_) != has_tables.end()) { + if (has_tables.find(file.table_id_) != has_tables.end()) { continue; } auto tables = ConnectorPtr->select(columns(&TableSchema::id_), where(c(&TableSchema::table_id_) == file.table_id_ - and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); - if(tables.size() >= 1) { + and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); + if (tables.size() >= 1) { has_tables[file.table_id_] = true; } else { has_tables[file.table_id_] = false; @@ -1114,7 +1085,7 @@ Status SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { auto commited = ConnectorPtr->transaction([&]() mutable { for (auto &file : files) { - if(!has_tables[file.table_id_]) { + if (!has_tables[file.table_id_]) { file.file_type_ = TableFileSchema::TO_DELETE; } @@ -1135,7 +1106,8 @@ Status SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { return Status::OK(); } -Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { +Status +SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { auto now = utils::GetMicroSecTimeStamp(); std::set table_ids; @@ -1151,11 +1123,11 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { &TableFileSchema::file_id_, &TableFileSchema::date_), where( - c(&TableFileSchema::file_type_) == + c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_DELETE and - c(&TableFileSchema::updated_time_) - < now - seconds * US_PS)); + c(&TableFileSchema::updated_time_) + < now - seconds * US_PS)); auto commited = ConnectorPtr->transaction([&]() mutable { TableFileSchema table_file; @@ -1178,10 +1150,9 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed"); } - if(files.size() > 0) { + if (files.size() > 0) { ENGINE_LOG_DEBUG << "Clean " << files.size() << " files deleted in " << seconds << " seconds"; } - } catch (std::exception &e) { return HandleException("Encounter exception when clean table files", e.what()); } @@ -1210,10 +1181,9 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed"); } - if(tables.size() > 0) { + if (tables.size() > 0) { ENGINE_LOG_DEBUG << "Remove " << tables.size() << " tables from meta"; } - } catch (std::exception &e) { return HandleException("Encounter exception when clean table files", e.what()); } @@ -1223,18 +1193,17 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { try { server::MetricCollector metric; - for(auto& table_id : table_ids) { + for (auto &table_id : table_ids) { auto selected = ConnectorPtr->select(columns(&TableFileSchema::file_id_), where(c(&TableFileSchema::table_id_) == table_id)); - if(selected.size() == 0) { + if (selected.size() == 0) { utils::DeleteTablePath(options_, table_id); } } - if(table_ids.size() > 0) { + if (table_ids.size() > 0) { ENGINE_LOG_DEBUG << "Remove " << table_ids.size() << " tables folder"; } - } catch (std::exception &e) { return HandleException("Encounter exception when delete table folder", e.what()); } @@ -1242,7 +1211,8 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return Status::OK(); } -Status SqliteMetaImpl::CleanUp() { +Status +SqliteMetaImpl::CleanUp() { try { server::MetricCollector metric; @@ -1250,11 +1220,12 @@ Status SqliteMetaImpl::CleanUp() { std::lock_guard meta_lock(meta_mutex_); std::vector file_types = { - (int) TableFileSchema::NEW, - (int) TableFileSchema::NEW_INDEX, - (int) TableFileSchema::NEW_MERGE + (int) TableFileSchema::NEW, + (int) TableFileSchema::NEW_INDEX, + (int) TableFileSchema::NEW_MERGE }; - auto files = ConnectorPtr->select(columns(&TableFileSchema::id_), where(in(&TableFileSchema::file_type_, file_types))); + auto files = + ConnectorPtr->select(columns(&TableFileSchema::id_), where(in(&TableFileSchema::file_type_, file_types))); auto commited = ConnectorPtr->transaction([&]() mutable { for (auto &file : files) { @@ -1268,10 +1239,9 @@ Status SqliteMetaImpl::CleanUp() { return HandleException("CleanUp error: sqlite transaction failed"); } - if(files.size() > 0) { + if (files.size() > 0) { ENGINE_LOG_DEBUG << "Clean " << files.size() << " files"; } - } catch (std::exception &e) { return HandleException("Encounter exception when clean table file", e.what()); } @@ -1279,19 +1249,19 @@ Status SqliteMetaImpl::CleanUp() { return Status::OK(); } -Status SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) { - +Status +SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) { try { server::MetricCollector metric; std::vector file_types = { - (int) TableFileSchema::RAW, - (int) TableFileSchema::TO_INDEX, - (int) TableFileSchema::INDEX + (int) TableFileSchema::RAW, + (int) TableFileSchema::TO_INDEX, + (int) TableFileSchema::INDEX }; auto selected = ConnectorPtr->select(columns(&TableFileSchema::row_count_), where(in(&TableFileSchema::file_type_, file_types) - and c(&TableFileSchema::table_id_) == table_id)); + and c(&TableFileSchema::table_id_) == table_id)); TableSchema table_schema; table_schema.table_id_ = table_id; @@ -1305,14 +1275,14 @@ Status SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) { for (auto &file : selected) { result += std::get<0>(file); } - } catch (std::exception &e) { return HandleException("Encounter exception when calculate table file size", e.what()); } return Status::OK(); } -Status SqliteMetaImpl::DropAll() { +Status +SqliteMetaImpl::DropAll() { ENGINE_LOG_DEBUG << "Drop all sqlite meta"; try { diff --git a/cpp/src/db/meta/SqliteMetaImpl.h b/cpp/src/db/meta/SqliteMetaImpl.h index db7a70fe53..978e5e5a4b 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.h +++ b/cpp/src/db/meta/SqliteMetaImpl.h @@ -21,22 +21,25 @@ #include "db/Options.h" #include +#include +#include namespace zilliz { namespace milvus { namespace engine { namespace meta { -auto StoragePrototype(const std::string &path); +auto +StoragePrototype(const std::string &path); class SqliteMetaImpl : public Meta { public: - explicit SqliteMetaImpl(const DBMetaOptions &options_); + explicit SqliteMetaImpl(const DBMetaOptions &options); ~SqliteMetaImpl(); Status CreateTable(TableSchema &table_schema) override; - Status DescribeTable(TableSchema &group_info_) override; + Status DescribeTable(TableSchema &table_schema) override; Status HasTable(const std::string &table_id, bool &has_or_not) override; @@ -58,11 +61,11 @@ class SqliteMetaImpl : public Meta { const std::vector &file_types, std::vector &file_ids) override; - Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) override; + Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) override; Status UpdateTableFlag(const std::string &table_id, int64_t flag) override; - Status DescribeTableIndex(const std::string &table_id, TableIndex& index) override; + Status DescribeTableIndex(const std::string &table_id, TableIndex &index) override; Status DropTableIndex(const std::string &table_id) override; @@ -96,12 +99,12 @@ class SqliteMetaImpl : public Meta { private: Status NextFileId(std::string &file_id); Status NextTableId(std::string &table_id); - Status DiscardFiles(long to_discard_size); + Status DiscardFiles(int64_t to_discard_size); void ValidateMetaSchema(); Status Initialize(); -private: + private: const DBMetaOptions options_; std::mutex meta_mutex_; }; // DBMetaImpl From 45a5d644d5f7421524014789520cf3ac2234ff01 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Fri, 27 Sep 2019 14:46:15 +0800 Subject: [PATCH 131/217] remove MetricBase.h from code coverage check Former-commit-id: a13611379840983ac4b2ea9573a353485d4a9ac6 --- cpp/coverage.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/coverage.sh b/cpp/coverage.sh index f1c42ede61..84459fcc84 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -101,6 +101,7 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \ "src/core/cmake_build*" \ "src/core/thirdparty*" \ "src/grpc*"\ + "src/metrics/MetricBase.h"\ "src/server/Server.cpp"\ "src/server/DBWrapper.cpp"\ "src/server/grpc_impl/GrpcMilvusServer.cpp"\ From d89aa2020c947fd7ea336f871532d1b3fb06f00f Mon Sep 17 00:00:00 2001 From: Yu Kun Date: Fri, 27 Sep 2019 14:53:27 +0800 Subject: [PATCH 132/217] add showtables unittest Former-commit-id: a32787a331c94245e1cf87ad43e5e365120cb38e --- cpp/unittest/server/rpc_test.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/unittest/server/rpc_test.cpp b/cpp/unittest/server/rpc_test.cpp index 7895d081a1..6b27db2d48 100644 --- a/cpp/unittest/server/rpc_test.cpp +++ b/cpp/unittest/server/rpc_test.cpp @@ -349,11 +349,11 @@ TEST_F(RpcHandlerTest, TABLES_TEST) { handler->Insert(&context, &request, &vector_ids); -//Show table -// ::milvus::grpc::Command cmd; -// ::grpc::ServerWriter<::milvus::grpc::TableName> *writer; -// status = handler->ShowTables(&context, &cmd, writer); -// ASSERT_EQ(status.error_code(), ::grpc::Status::OK.error_code()); + //show tables + ::milvus::grpc::Command cmd; + ::milvus::grpc::TableNameList table_name_list; + status = handler->ShowTables(&context, &cmd, &table_name_list); + ASSERT_EQ(status.error_code(), ::grpc::Status::OK.error_code()); //Count Table ::milvus::grpc::TableRowCount count; From ca1ab56cb4e06cc8005b96b3b9dc32c56993adcb Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 27 Sep 2019 15:12:08 +0800 Subject: [PATCH 133/217] format metric code Former-commit-id: ea430e044ae3c93b051c4ac6261cfe3c241ae80b --- cpp/build-support/lint_exclusions.txt | 3 +- cpp/src/metrics/MetricBase.h | 180 ++++++++++---- cpp/src/metrics/Metrics.cpp | 9 +- cpp/src/metrics/Metrics.h | 73 +++--- cpp/src/metrics/PrometheusMetrics.cpp | 105 ++++---- cpp/src/metrics/PrometheusMetrics.h | 337 +++++++++++++++++++------- cpp/src/metrics/SystemInfo.cpp | 155 ++++++------ cpp/src/metrics/SystemInfo.h | 97 +++++--- 8 files changed, 611 insertions(+), 348 deletions(-) diff --git a/cpp/build-support/lint_exclusions.txt b/cpp/build-support/lint_exclusions.txt index f08664a1a9..27bd780f42 100644 --- a/cpp/build-support/lint_exclusions.txt +++ b/cpp/build-support/lint_exclusions.txt @@ -4,4 +4,5 @@ *src/thirdparty* *src/core/thirdparty* *src/grpc* -*easylogging++* \ No newline at end of file +*easylogging++* +*SqliteMetaImpl.cpp \ No newline at end of file diff --git a/cpp/src/metrics/MetricBase.h b/cpp/src/metrics/MetricBase.h index 8a3f11d7df..32a0e715e6 100644 --- a/cpp/src/metrics/MetricBase.h +++ b/cpp/src/metrics/MetricBase.h @@ -21,67 +21,149 @@ #include "utils/Error.h" #include "SystemInfo.h" +#include + namespace zilliz { namespace milvus { namespace server { -class MetricsBase{ +class MetricsBase { public: - static MetricsBase& - GetInstance(){ + static MetricsBase & + GetInstance() { static MetricsBase instance; return instance; } - virtual ErrorCode Init() {}; + virtual ErrorCode Init() { + } - virtual void AddVectorsSuccessTotalIncrement(double value = 1) {}; - virtual void AddVectorsFailTotalIncrement(double value = 1) {}; - virtual void AddVectorsDurationHistogramOberve(double value) {}; + virtual void AddVectorsSuccessTotalIncrement(double value = 1) { + } - virtual void RawFileSizeHistogramObserve(double value) {}; - virtual void IndexFileSizeHistogramObserve(double value) {}; - virtual void BuildIndexDurationSecondsHistogramObserve(double value) {}; + virtual void AddVectorsFailTotalIncrement(double value = 1) { + } - virtual void CpuCacheUsageGaugeSet(double value) {}; - virtual void GpuCacheUsageGaugeSet() {}; + virtual void AddVectorsDurationHistogramOberve(double value) { + } - virtual void MetaAccessTotalIncrement(double value = 1) {}; - virtual void MetaAccessDurationSecondsHistogramObserve(double value) {}; - virtual void FaissDiskLoadDurationSecondsHistogramObserve(double value) {}; - virtual void FaissDiskLoadSizeBytesHistogramObserve(double value) {}; - virtual void CacheAccessTotalIncrement(double value = 1) {}; - virtual void MemTableMergeDurationSecondsHistogramObserve(double value) {}; - virtual void SearchIndexDataDurationSecondsHistogramObserve(double value) {}; - virtual void SearchRawDataDurationSecondsHistogramObserve(double value) {}; - virtual void IndexFileSizeTotalIncrement(double value = 1) {}; - virtual void RawFileSizeTotalIncrement(double value = 1) {}; - virtual void IndexFileSizeGaugeSet(double value) {}; - virtual void RawFileSizeGaugeSet(double value) {}; - virtual void FaissDiskLoadIOSpeedGaugeSet(double value) {}; - virtual void QueryResponseSummaryObserve(double value) {}; - virtual void DiskStoreIOSpeedGaugeSet(double value) {}; - virtual void DataFileSizeGaugeSet(double value) {}; - virtual void AddVectorsSuccessGaugeSet(double value) {}; - virtual void AddVectorsFailGaugeSet(double value) {}; - virtual void QueryVectorResponseSummaryObserve(double value, int count = 1) {}; - virtual void QueryVectorResponsePerSecondGaugeSet(double value) {}; - virtual void CPUUsagePercentSet() {}; - virtual void RAMUsagePercentSet() {}; - virtual void QueryResponsePerSecondGaugeSet(double value) {}; - virtual void GPUPercentGaugeSet() {}; - virtual void GPUMemoryUsageGaugeSet() {}; - virtual void AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) {}; - virtual void QueryIndexTypePerSecondSet(std::string type, double value) {}; - virtual void ConnectionGaugeIncrement() {}; - virtual void ConnectionGaugeDecrement() {}; - virtual void KeepingAliveCounterIncrement(double value = 1) {}; - virtual void OctetsSet() {}; + virtual void RawFileSizeHistogramObserve(double value) { + } - virtual void CPUCoreUsagePercentSet() {}; - virtual void GPUTemperature() {}; - virtual void CPUTemperature() {}; + virtual void IndexFileSizeHistogramObserve(double value) { + } + + virtual void BuildIndexDurationSecondsHistogramObserve(double value) { + } + + virtual void CpuCacheUsageGaugeSet(double value) { + } + + virtual void GpuCacheUsageGaugeSet() { + } + + virtual void MetaAccessTotalIncrement(double value = 1) { + } + + virtual void MetaAccessDurationSecondsHistogramObserve(double value) { + } + + virtual void FaissDiskLoadDurationSecondsHistogramObserve(double value) { + } + + virtual void FaissDiskLoadSizeBytesHistogramObserve(double value) { + } + + virtual void CacheAccessTotalIncrement(double value = 1) { + } + + virtual void MemTableMergeDurationSecondsHistogramObserve(double value) { + } + + virtual void SearchIndexDataDurationSecondsHistogramObserve(double value) { + } + + virtual void SearchRawDataDurationSecondsHistogramObserve(double value) { + } + + virtual void IndexFileSizeTotalIncrement(double value = 1) { + } + + virtual void RawFileSizeTotalIncrement(double value = 1) { + } + + virtual void IndexFileSizeGaugeSet(double value) { + } + + virtual void RawFileSizeGaugeSet(double value) { + } + + virtual void FaissDiskLoadIOSpeedGaugeSet(double value) { + } + + virtual void QueryResponseSummaryObserve(double value) { + } + + virtual void DiskStoreIOSpeedGaugeSet(double value) { + } + + virtual void DataFileSizeGaugeSet(double value) { + } + + virtual void AddVectorsSuccessGaugeSet(double value) { + } + + virtual void AddVectorsFailGaugeSet(double value) { + } + + virtual void QueryVectorResponseSummaryObserve(double value, int count = 1) { + } + + virtual void QueryVectorResponsePerSecondGaugeSet(double value) { + } + + virtual void CPUUsagePercentSet() { + } + + virtual void RAMUsagePercentSet() { + } + + virtual void QueryResponsePerSecondGaugeSet(double value) { + } + + virtual void GPUPercentGaugeSet() { + } + + virtual void GPUMemoryUsageGaugeSet() { + } + + virtual void AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) { + } + + virtual void QueryIndexTypePerSecondSet(std::string type, double value) { + } + + virtual void ConnectionGaugeIncrement() { + } + + virtual void ConnectionGaugeDecrement() { + } + + virtual void KeepingAliveCounterIncrement(double value = 1) { + } + + virtual void OctetsSet() { + } + + virtual void CPUCoreUsagePercentSet() { + } + + virtual void GPUTemperature() { + } + + virtual void CPUTemperature() { + } }; -} -} -} \ No newline at end of file +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/metrics/Metrics.cpp b/cpp/src/metrics/Metrics.cpp index 612f5e7fce..0a1b333b42 100644 --- a/cpp/src/metrics/Metrics.cpp +++ b/cpp/src/metrics/Metrics.cpp @@ -15,10 +15,11 @@ // specific language governing permissions and limitations // under the License. -#include "Metrics.h" +#include "metrics/Metrics.h" #include "server/Config.h" #include "PrometheusMetrics.h" +#include namespace zilliz { namespace milvus { @@ -44,6 +45,6 @@ Metrics::CreateMetricsCollector() { } } -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/metrics/Metrics.h b/cpp/src/metrics/Metrics.h index 30d6e68f52..66e06b8ba1 100644 --- a/cpp/src/metrics/Metrics.h +++ b/cpp/src/metrics/Metrics.h @@ -21,7 +21,6 @@ #include "MetricBase.h" #include "db/meta/MetaTypes.h" - namespace zilliz { namespace milvus { namespace server { @@ -44,7 +43,7 @@ class Metrics { }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CollectMetricsBase { -protected: + protected: CollectMetricsBase() { start_time_ = METRICS_NOW_TIME; } @@ -56,19 +55,19 @@ protected: return METRICS_MICROSECONDS(start_time_, end_time); } -protected: + protected: using TIME_POINT = std::chrono::system_clock::time_point; TIME_POINT start_time_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CollectInsertMetrics : CollectMetricsBase { -public: - CollectInsertMetrics(size_t n, Status& status) : n_(n), status_(status) { + public: + CollectInsertMetrics(size_t n, Status &status) : n_(n), status_(status) { } ~CollectInsertMetrics() { - if(n_ > 0) { + if (n_ > 0) { auto total_time = TimeFromBegine(); double avg_time = total_time / n_; for (int i = 0; i < n_; ++i) { @@ -86,19 +85,19 @@ public: } } -private: + private: size_t n_; - Status& status_; + Status &status_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CollectQueryMetrics : CollectMetricsBase { -public: - CollectQueryMetrics(size_t nq) : nq_(nq) { + public: + explicit CollectQueryMetrics(size_t nq) : nq_(nq) { } ~CollectQueryMetrics() { - if(nq_ > 0) { + if (nq_ > 0) { auto total_time = TimeFromBegine(); for (int i = 0; i < nq_; ++i) { server::Metrics::GetInstance().QueryResponseSummaryObserve(total_time); @@ -109,13 +108,13 @@ public: } } -private: + private: size_t nq_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CollectMergeFilesMetrics : CollectMetricsBase { -public: + public: CollectMergeFilesMetrics() { } @@ -127,7 +126,7 @@ public: //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CollectBuildIndexMetrics : CollectMetricsBase { -public: + public: CollectBuildIndexMetrics() { } @@ -139,8 +138,8 @@ public: //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CollectExecutionEngineMetrics : CollectMetricsBase { -public: - CollectExecutionEngineMetrics(double physical_size) : physical_size_(physical_size) { + public: + explicit CollectExecutionEngineMetrics(double physical_size) : physical_size_(physical_size) { } ~CollectExecutionEngineMetrics() { @@ -151,27 +150,28 @@ public: server::Metrics::GetInstance().FaissDiskLoadIOSpeedGaugeSet(physical_size_ / double(total_time)); } -private: + private: double physical_size_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CollectSerializeMetrics : CollectMetricsBase { -public: - CollectSerializeMetrics(size_t size) : size_(size) { + public: + explicit CollectSerializeMetrics(size_t size) : size_(size) { } ~CollectSerializeMetrics() { auto total_time = TimeFromBegine(); server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet((double) size_ / total_time); } -private: + + private: size_t size_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CollectAddMetrics : CollectMetricsBase { -public: + public: CollectAddMetrics(size_t n, uint16_t dimension) : n_(n), dimension_(dimension) { } @@ -181,15 +181,16 @@ public: static_cast(dimension_), total_time); } -private: + + private: size_t n_; uint16_t dimension_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CollectDurationMetrics : CollectMetricsBase { -public: - CollectDurationMetrics(int index_type) : index_type_(index_type) { + public: + explicit CollectDurationMetrics(int index_type) : index_type_(index_type) { } ~CollectDurationMetrics() { @@ -209,19 +210,20 @@ public: } } } -private: + + private: int index_type_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CollectSearchTaskMetrics : CollectMetricsBase { -public: - CollectSearchTaskMetrics(int index_type) : index_type_(index_type) { + public: + explicit CollectSearchTaskMetrics(int index_type) : index_type_(index_type) { } ~CollectSearchTaskMetrics() { auto total_time = TimeFromBegine(); - switch(index_type_) { + switch (index_type_) { case engine::meta::TableFileSchema::RAW: { server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time); break; @@ -237,13 +239,13 @@ public: } } -private: + private: int index_type_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class MetricCollector : CollectMetricsBase { -public: + public: MetricCollector() { server::Metrics::GetInstance().MetaAccessTotalIncrement(); } @@ -254,11 +256,6 @@ public: } }; - - -} -} -} - - - +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index 6f3513072a..8dcc97aa8b 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -16,12 +16,14 @@ // under the License. +#include "metrics/PrometheusMetrics.h" #include "cache/GpuCacheMgr.h" -#include "PrometheusMetrics.h" #include "server/Config.h" #include "utils/Log.h" #include "SystemInfo.h" +#include +#include namespace zilliz { namespace milvus { @@ -47,93 +49,96 @@ PrometheusMetrics::Init() { // Exposer Registry exposer_ptr_->RegisterCollectable(registry_); - } catch (std::exception& ex) { + } catch (std::exception &ex) { SERVER_LOG_ERROR << "Failed to connect prometheus server: " << std::string(ex.what()); return SERVER_UNEXPECTED_ERROR; } return SERVER_SUCCESS; - } - void -PrometheusMetrics::CPUUsagePercentSet() { - if(!startup_) return ; +PrometheusMetrics::CPUUsagePercentSet() { + if (!startup_) return; double usage_percent = server::SystemInfo::GetInstance().CPUPercent(); CPU_usage_percent_.Set(usage_percent); } void PrometheusMetrics::RAMUsagePercentSet() { - if(!startup_) return ; + if (!startup_) return; double usage_percent = server::SystemInfo::GetInstance().MemoryPercent(); RAM_usage_percent_.Set(usage_percent); } void PrometheusMetrics::GPUPercentGaugeSet() { - if(!startup_) return; + if (!startup_) return; int numDevice = server::SystemInfo::GetInstance().num_device(); - std::vector used_total = server::SystemInfo::GetInstance().GPUMemoryTotal(); - std::vector used_memory = server::SystemInfo::GetInstance().GPUMemoryUsed(); + std::vector used_total = server::SystemInfo::GetInstance().GPUMemoryTotal(); + std::vector used_memory = server::SystemInfo::GetInstance().GPUMemoryUsed(); for (int i = 0; i < numDevice; ++i) { prometheus::Gauge &GPU_percent = GPU_percent_.Add({{"DeviceNum", std::to_string(i)}}); - double percent = (double)used_memory[i] / (double)used_total[i]; + double percent = (double) used_memory[i] / (double) used_total[i]; GPU_percent.Set(percent * 100); } } -void PrometheusMetrics::GPUMemoryUsageGaugeSet() { - if(!startup_) return; - std::vector values = server::SystemInfo::GetInstance().GPUMemoryUsed(); - constexpr unsigned long long MtoB = 1024*1024; +void +PrometheusMetrics::GPUMemoryUsageGaugeSet() { + if (!startup_) return; + std::vector values = server::SystemInfo::GetInstance().GPUMemoryUsed(); + constexpr uint64_t MtoB = 1024 * 1024; int numDevice = server::SystemInfo::GetInstance().num_device(); for (int i = 0; i < numDevice; ++i) { prometheus::Gauge &GPU_memory = GPU_memory_usage_.Add({{"DeviceNum", std::to_string(i)}}); GPU_memory.Set(values[i] / MtoB); } - } -void PrometheusMetrics::AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) { + +void +PrometheusMetrics::AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) { // MB/s - if(!startup_) return; - - long long MtoB = 1024*1024; - long long size = num_vector * dim * 4; - add_vectors_per_second_gauge_.Set(size/time/MtoB); + if (!startup_) return; + int64_t MtoB = 1024 * 1024; + int64_t size = num_vector * dim * 4; + add_vectors_per_second_gauge_.Set(size / time / MtoB); } -void PrometheusMetrics::QueryIndexTypePerSecondSet(std::string type, double value) { - if(!startup_) return; - if(type == "IVF"){ + +void +PrometheusMetrics::QueryIndexTypePerSecondSet(std::string type, double value) { + if (!startup_) return; + if (type == "IVF") { query_index_IVF_type_per_second_gauge_.Set(value); - } else if(type == "IDMap"){ + } else if (type == "IDMap") { query_index_IDMAP_type_per_second_gauge_.Set(value); } - } -void PrometheusMetrics::ConnectionGaugeIncrement() { - if(!startup_) return; +void +PrometheusMetrics::ConnectionGaugeIncrement() { + if (!startup_) return; connection_gauge_.Increment(); } -void PrometheusMetrics::ConnectionGaugeDecrement() { - if(!startup_) return; +void +PrometheusMetrics::ConnectionGaugeDecrement() { + if (!startup_) return; connection_gauge_.Decrement(); } -void PrometheusMetrics::OctetsSet() { - if(!startup_) return; +void +PrometheusMetrics::OctetsSet() { + if (!startup_) return; // get old stats and reset them - unsigned long long old_inoctets = SystemInfo::GetInstance().get_inoctets(); - unsigned long long old_outoctets = SystemInfo::GetInstance().get_octets(); + uint64_t old_inoctets = SystemInfo::GetInstance().get_inoctets(); + uint64_t old_outoctets = SystemInfo::GetInstance().get_octets(); auto old_time = SystemInfo::GetInstance().get_nettime(); - std::pair in_and_out_octets = SystemInfo::GetInstance().Octets(); + std::pair in_and_out_octets = SystemInfo::GetInstance().Octets(); SystemInfo::GetInstance().set_inoctets(in_and_out_octets.first); SystemInfo::GetInstance().set_outoctets(in_and_out_octets.second); SystemInfo::GetInstance().set_nettime(); @@ -142,13 +147,14 @@ void PrometheusMetrics::OctetsSet() { constexpr double micro_to_second = 1e-6; auto now_time = std::chrono::system_clock::now(); auto total_microsecond = METRICS_MICROSECONDS(old_time, now_time); - auto total_second = total_microsecond*micro_to_second; - if(total_second == 0) return; - inoctets_gauge_.Set((in_and_out_octets.first-old_inoctets)/total_second); - outoctets_gauge_.Set((in_and_out_octets.second-old_outoctets)/total_second); + auto total_second = total_microsecond * micro_to_second; + if (total_second == 0) return; + inoctets_gauge_.Set((in_and_out_octets.first - old_inoctets) / total_second); + outoctets_gauge_.Set((in_and_out_octets.second - old_outoctets) / total_second); } -void PrometheusMetrics::CPUCoreUsagePercentSet() { +void +PrometheusMetrics::CPUCoreUsagePercentSet() { if (!startup_) return; @@ -160,11 +166,12 @@ void PrometheusMetrics::CPUCoreUsagePercentSet() { } } -void PrometheusMetrics::GPUTemperature() { +void +PrometheusMetrics::GPUTemperature() { if (!startup_) return; - std::vector GPU_temperatures = server::SystemInfo::GetInstance().GPUTemperature(); + std::vector GPU_temperatures = server::SystemInfo::GetInstance().GPUTemperature(); for (int i = 0; i < GPU_temperatures.size(); ++i) { prometheus::Gauge &gpu_temp = GPU_temperature_.Add({{"GPU", std::to_string(i)}}); @@ -172,7 +179,8 @@ void PrometheusMetrics::GPUTemperature() { } } -void PrometheusMetrics::CPUTemperature() { +void +PrometheusMetrics::CPUTemperature() { if (!startup_) return; @@ -184,7 +192,8 @@ void PrometheusMetrics::CPUTemperature() { } } -void PrometheusMetrics::GpuCacheUsageGaugeSet() { +void +PrometheusMetrics::GpuCacheUsageGaugeSet() { // std::vector gpu_ids = {0}; // for(auto i = 0; i < gpu_ids.size(); ++i) { // uint64_t cache_usage = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheUsage(); @@ -194,6 +203,6 @@ void PrometheusMetrics::GpuCacheUsageGaugeSet() { // } } -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/metrics/PrometheusMetrics.h b/cpp/src/metrics/PrometheusMetrics.h index 4ee6699b77..ce45c5e711 100644 --- a/cpp/src/metrics/PrometheusMetrics.h +++ b/cpp/src/metrics/PrometheusMetrics.h @@ -22,22 +22,20 @@ #include #include #include +#include #include "utils/Error.h" #include "MetricBase.h" - #define METRICS_NOW_TIME std::chrono::system_clock::now() //#define server::Metrics::GetInstance() server::GetInstance() -#define METRICS_MICROSECONDS(a,b) (std::chrono::duration_cast (b-a)).count(); - +#define METRICS_MICROSECONDS(a, b) (std::chrono::duration_cast (b-a)).count(); namespace zilliz { namespace milvus { namespace server { -class PrometheusMetrics: public MetricsBase { - +class PrometheusMetrics : public MetricsBase { public: static PrometheusMetrics & GetInstance() { @@ -51,59 +49,215 @@ class PrometheusMetrics: public MetricsBase { std::shared_ptr exposer_ptr_; std::shared_ptr registry_ = std::make_shared(); bool startup_ = false; + public: - void SetStartup(bool startup) {startup_ = startup;}; - void AddVectorsSuccessTotalIncrement(double value = 1.0) override { if(startup_) add_vectors_success_total_.Increment(value);}; - void AddVectorsFailTotalIncrement(double value = 1.0) override { if(startup_) add_vectors_fail_total_.Increment(value);}; - void AddVectorsDurationHistogramOberve(double value) override { if(startup_) add_vectors_duration_histogram_.Observe(value);}; - void RawFileSizeHistogramObserve(double value) override { if(startup_) raw_files_size_histogram_.Observe(value);}; - void IndexFileSizeHistogramObserve(double value) override { if(startup_) index_files_size_histogram_.Observe(value);}; - void BuildIndexDurationSecondsHistogramObserve(double value) override { if(startup_) build_index_duration_seconds_histogram_.Observe(value);}; - void CpuCacheUsageGaugeSet(double value) override { if(startup_) cpu_cache_usage_gauge_.Set(value);}; + void SetStartup(bool startup) { + startup_ = startup; + } + + void AddVectorsSuccessTotalIncrement(double value = 1.0) override { + if (startup_) { + add_vectors_success_total_.Increment(value); + } + } + + void AddVectorsFailTotalIncrement(double value = 1.0) override { + if (startup_) { + add_vectors_fail_total_.Increment(value); + } + } + + void AddVectorsDurationHistogramOberve(double value) override { + if (startup_) { + add_vectors_duration_histogram_.Observe(value); + } + } + + void RawFileSizeHistogramObserve(double value) override { + if (startup_) { + raw_files_size_histogram_.Observe(value); + } + } + + void IndexFileSizeHistogramObserve(double value) override { + if (startup_) { + index_files_size_histogram_.Observe(value); + } + } + + void BuildIndexDurationSecondsHistogramObserve(double value) override { + if (startup_) { + build_index_duration_seconds_histogram_.Observe(value); + } + } + + void CpuCacheUsageGaugeSet(double value) override { + if (startup_) { + cpu_cache_usage_gauge_.Set(value); + } + } + void GpuCacheUsageGaugeSet() override; - void MetaAccessTotalIncrement(double value = 1) override { if(startup_) meta_access_total_.Increment(value);}; - void MetaAccessDurationSecondsHistogramObserve(double value) override { if(startup_) meta_access_duration_seconds_histogram_.Observe(value);}; + void MetaAccessTotalIncrement(double value = 1) override { + if (startup_) { + meta_access_total_.Increment(value); + } + } - void FaissDiskLoadDurationSecondsHistogramObserve(double value) override { if(startup_) faiss_disk_load_duration_seconds_histogram_.Observe(value);}; - void FaissDiskLoadSizeBytesHistogramObserve(double value) override { if(startup_) faiss_disk_load_size_bytes_histogram_.Observe(value);}; - void FaissDiskLoadIOSpeedGaugeSet(double value) override { if(startup_) faiss_disk_load_IO_speed_gauge_.Set(value);}; + void MetaAccessDurationSecondsHistogramObserve(double value) override { + if (startup_) { + meta_access_duration_seconds_histogram_.Observe(value); + } + } - void CacheAccessTotalIncrement(double value = 1) override { if(startup_) cache_access_total_.Increment(value);}; - void MemTableMergeDurationSecondsHistogramObserve(double value) override { if(startup_) mem_table_merge_duration_seconds_histogram_.Observe(value);}; - void SearchIndexDataDurationSecondsHistogramObserve(double value) override { if(startup_) search_index_data_duration_seconds_histogram_.Observe(value);}; - void SearchRawDataDurationSecondsHistogramObserve(double value) override { if(startup_) search_raw_data_duration_seconds_histogram_.Observe(value);}; - void IndexFileSizeTotalIncrement(double value = 1) override { if(startup_) index_file_size_total_.Increment(value);}; - void RawFileSizeTotalIncrement(double value = 1) override { if(startup_) raw_file_size_total_.Increment(value);}; - void IndexFileSizeGaugeSet(double value) override { if(startup_) index_file_size_gauge_.Set(value);}; - void RawFileSizeGaugeSet(double value) override { if(startup_) raw_file_size_gauge_.Set(value);}; - void QueryResponseSummaryObserve(double value) override {if(startup_) query_response_summary_.Observe(value);}; - void DiskStoreIOSpeedGaugeSet(double value) override { if(startup_) disk_store_IO_speed_gauge_.Set(value);}; - void DataFileSizeGaugeSet(double value) override { if(startup_) data_file_size_gauge_.Set(value);}; - void AddVectorsSuccessGaugeSet(double value) override { if(startup_) add_vectors_success_gauge_.Set(value);}; - void AddVectorsFailGaugeSet(double value) override { if(startup_) add_vectors_fail_gauge_.Set(value);}; - void QueryVectorResponseSummaryObserve(double value, int count = 1) override { if (startup_) for(int i = 0 ; i < count ; ++i) query_vector_response_summary_.Observe(value);}; - void QueryVectorResponsePerSecondGaugeSet(double value) override {if (startup_) query_vector_response_per_second_gauge_.Set(value);}; - void CPUUsagePercentSet() override ; + void FaissDiskLoadDurationSecondsHistogramObserve(double value) override { + if (startup_) { + faiss_disk_load_duration_seconds_histogram_.Observe(value); + } + } + + void FaissDiskLoadSizeBytesHistogramObserve(double value) override { + if (startup_) { + faiss_disk_load_size_bytes_histogram_.Observe(value); + } + } + + void FaissDiskLoadIOSpeedGaugeSet(double value) override { + if (startup_) { + faiss_disk_load_IO_speed_gauge_.Set(value); + } + } + + void CacheAccessTotalIncrement(double value = 1) override { + if (startup_) { + cache_access_total_.Increment(value); + } + } + + void MemTableMergeDurationSecondsHistogramObserve(double value) override { + if (startup_) { + mem_table_merge_duration_seconds_histogram_.Observe(value); + } + } + + void SearchIndexDataDurationSecondsHistogramObserve(double value) override { + if (startup_) { + search_index_data_duration_seconds_histogram_.Observe(value); + } + } + + void SearchRawDataDurationSecondsHistogramObserve(double value) override { + if (startup_) { + search_raw_data_duration_seconds_histogram_.Observe(value); + } + } + + void IndexFileSizeTotalIncrement(double value = 1) override { + if (startup_) { + index_file_size_total_.Increment(value); + } + } + + void RawFileSizeTotalIncrement(double value = 1) override { + if (startup_) { + raw_file_size_total_.Increment(value); + } + } + + void IndexFileSizeGaugeSet(double value) override { + if (startup_) { + index_file_size_gauge_.Set(value); + } + } + + void RawFileSizeGaugeSet(double value) override { + if (startup_) { + raw_file_size_gauge_.Set(value); + } + } + + void QueryResponseSummaryObserve(double value) override { + if (startup_) { + query_response_summary_.Observe(value); + } + } + + void DiskStoreIOSpeedGaugeSet(double value) override { + if (startup_) { + disk_store_IO_speed_gauge_.Set(value); + } + } + + void DataFileSizeGaugeSet(double value) override { + if (startup_) { + data_file_size_gauge_.Set(value); + } + } + + void AddVectorsSuccessGaugeSet(double value) override { + if (startup_) { + add_vectors_success_gauge_.Set(value); + } + } + + void AddVectorsFailGaugeSet(double value) override { + if (startup_) { + add_vectors_fail_gauge_.Set(value); + } + } + + void QueryVectorResponseSummaryObserve(double value, int count = 1) override { + if (startup_) { + for (int i = 0; i < count; ++i) { + query_vector_response_summary_.Observe(value); + } + } + } + + void QueryVectorResponsePerSecondGaugeSet(double value) override { + if (startup_) { + query_vector_response_per_second_gauge_.Set(value); + } + } + + void CPUUsagePercentSet() override; void CPUCoreUsagePercentSet() override; - void RAMUsagePercentSet() override ; - void QueryResponsePerSecondGaugeSet(double value) override {if(startup_) query_response_per_second_gauge.Set(value);}; - void GPUPercentGaugeSet() override ; - void GPUMemoryUsageGaugeSet() override ; - void AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) override ; - void QueryIndexTypePerSecondSet(std::string type, double value) override ; - void ConnectionGaugeIncrement() override ; - void ConnectionGaugeDecrement() override ; - void KeepingAliveCounterIncrement(double value = 1) override {if(startup_) keeping_alive_counter_.Increment(value);}; - void OctetsSet() override ; + void RAMUsagePercentSet() override; + + void QueryResponsePerSecondGaugeSet(double value) override { + if (startup_) { + query_response_per_second_gauge.Set(value); + } + } + + void GPUPercentGaugeSet() override; + void GPUMemoryUsageGaugeSet() override; + void AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) override; + void QueryIndexTypePerSecondSet(std::string type, double value) override; + void ConnectionGaugeIncrement() override; + void ConnectionGaugeDecrement() override; + + void KeepingAliveCounterIncrement(double value = 1) override { + if (startup_) { + keeping_alive_counter_.Increment(value); + } + } + + void OctetsSet() override; void GPUTemperature() override; void CPUTemperature() override; - std::shared_ptr &exposer_ptr() {return exposer_ptr_; } + std::shared_ptr &exposer_ptr() { + return exposer_ptr_; + } + // prometheus::Exposer& exposer() { return exposer_;} - std::shared_ptr ®istry_ptr() {return registry_; } + std::shared_ptr ®istry_ptr() { + return registry_; + } // ..... private: @@ -125,7 +279,6 @@ class PrometheusMetrics: public MetricsBase { prometheus::Counter &add_group_success_total_ = add_group_request_.Add({{"outcome", "success"}}); prometheus::Counter &add_group_fail_total_ = add_group_request_.Add({{"outcome", "fail"}}); - //record get_group request prometheus::Family &get_group_request_ = prometheus::BuildCounter() .Name("get_group_request_total") @@ -135,7 +288,6 @@ class PrometheusMetrics: public MetricsBase { prometheus::Counter &get_group_success_total_ = get_group_request_.Add({{"outcome", "success"}}); prometheus::Counter &get_group_fail_total_ = get_group_request_.Add({{"outcome", "fail"}}); - //record has_group request prometheus::Family &has_group_request_ = prometheus::BuildCounter() .Name("has_group_request_total") @@ -145,7 +297,6 @@ class PrometheusMetrics: public MetricsBase { prometheus::Counter &has_group_success_total_ = has_group_request_.Add({{"outcome", "success"}}); prometheus::Counter &has_group_fail_total_ = has_group_request_.Add({{"outcome", "fail"}}); - //record get_group_files prometheus::Family &get_group_files_request_ = prometheus::BuildCounter() .Name("get_group_files_request_total") @@ -155,7 +306,6 @@ class PrometheusMetrics: public MetricsBase { prometheus::Counter &get_group_files_success_total_ = get_group_files_request_.Add({{"outcome", "success"}}); prometheus::Counter &get_group_files_fail_total_ = get_group_files_request_.Add({{"outcome", "fail"}}); - //record add_vectors count and average time //need to be considered prometheus::Family &add_vectors_request_ = prometheus::BuildCounter() @@ -169,36 +319,39 @@ class PrometheusMetrics: public MetricsBase { .Name("add_vector_duration_microseconds") .Help("average time of adding every vector") .Register(*registry_); - prometheus::Histogram &add_vectors_duration_histogram_ = add_vectors_duration_seconds_.Add({}, BucketBoundaries{0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.08, 0.1, 0.5, 1}); - + prometheus::Histogram &add_vectors_duration_histogram_ = + add_vectors_duration_seconds_.Add({}, BucketBoundaries{0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.08, 0.1, 0.5, 1}); //record search count and average time prometheus::Family &search_request_ = prometheus::BuildCounter() .Name("search_request_total") .Help("the number of search request") .Register(*registry_); - prometheus::Counter &search_success_total_ = search_request_.Add({{"outcome","success"}}); - prometheus::Counter &search_fail_total_ = search_request_.Add({{"outcome","fail"}}); + prometheus::Counter &search_success_total_ = search_request_.Add({{"outcome", "success"}}); + prometheus::Counter &search_fail_total_ = search_request_.Add({{"outcome", "fail"}}); prometheus::Family &search_request_duration_seconds_ = prometheus::BuildHistogram() .Name("search_request_duration_microsecond") .Help("histogram of processing time for each search") .Register(*registry_); - prometheus::Histogram &search_duration_histogram_ = search_request_duration_seconds_.Add({}, BucketBoundaries{0.1, 1.0, 10.0}); + prometheus::Histogram + &search_duration_histogram_ = search_request_duration_seconds_.Add({}, BucketBoundaries{0.1, 1.0, 10.0}); //record raw_files size histogram prometheus::Family &raw_files_size_ = prometheus::BuildHistogram() .Name("search_raw_files_bytes") .Help("histogram of raw files size by bytes") .Register(*registry_); - prometheus::Histogram &raw_files_size_histogram_ = raw_files_size_.Add({}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9, 1e10}); + prometheus::Histogram + &raw_files_size_histogram_ = raw_files_size_.Add({}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9, 1e10}); //record index_files size histogram prometheus::Family &index_files_size_ = prometheus::BuildHistogram() .Name("search_index_files_bytes") .Help("histogram of index files size by bytes") .Register(*registry_); - prometheus::Histogram &index_files_size_histogram_ = index_files_size_.Add({}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9, 1e10}); + prometheus::Histogram + &index_files_size_histogram_ = index_files_size_.Add({}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9, 1e10}); //record index and raw files size counter prometheus::Family &file_size_total_ = prometheus::BuildCounter() @@ -221,30 +374,34 @@ class PrometheusMetrics: public MetricsBase { .Name("build_index_duration_microseconds") .Help("histogram of processing time for building index") .Register(*registry_); - prometheus::Histogram &build_index_duration_seconds_histogram_ = build_index_duration_seconds_.Add({}, BucketBoundaries{5e5, 2e6, 4e6, 6e6, 8e6, 1e7}); - + prometheus::Histogram &build_index_duration_seconds_histogram_ = + build_index_duration_seconds_.Add({}, BucketBoundaries{5e5, 2e6, 4e6, 6e6, 8e6, 1e7}); //record processing time for all building index prometheus::Family &all_build_index_duration_seconds_ = prometheus::BuildHistogram() .Name("all_build_index_duration_microseconds") .Help("histogram of processing time for building index") .Register(*registry_); - prometheus::Histogram &all_build_index_duration_seconds_histogram_ = all_build_index_duration_seconds_.Add({}, BucketBoundaries{2e6, 4e6, 6e6, 8e6, 1e7}); + prometheus::Histogram &all_build_index_duration_seconds_histogram_ = + all_build_index_duration_seconds_.Add({}, BucketBoundaries{2e6, 4e6, 6e6, 8e6, 1e7}); //record duration of merging mem table prometheus::Family &mem_table_merge_duration_seconds_ = prometheus::BuildHistogram() .Name("mem_table_merge_duration_microseconds") .Help("histogram of processing time for merging mem tables") .Register(*registry_); - prometheus::Histogram &mem_table_merge_duration_seconds_histogram_ = mem_table_merge_duration_seconds_.Add({}, BucketBoundaries{5e4, 1e5, 2e5, 4e5, 6e5, 8e5, 1e6}); + prometheus::Histogram &mem_table_merge_duration_seconds_histogram_ = + mem_table_merge_duration_seconds_.Add({}, BucketBoundaries{5e4, 1e5, 2e5, 4e5, 6e5, 8e5, 1e6}); //record search index and raw data duration prometheus::Family &search_data_duration_seconds_ = prometheus::BuildHistogram() .Name("search_data_duration_microseconds") .Help("histograms of processing time for search index and raw data") .Register(*registry_); - prometheus::Histogram &search_index_data_duration_seconds_histogram_ = search_data_duration_seconds_.Add({{"type", "index"}}, BucketBoundaries{1e5, 2e5, 4e5, 6e5, 8e5}); - prometheus::Histogram &search_raw_data_duration_seconds_histogram_ = search_data_duration_seconds_.Add({{"type", "raw"}}, BucketBoundaries{1e5, 2e5, 4e5, 6e5, 8e5}); + prometheus::Histogram &search_index_data_duration_seconds_histogram_ = + search_data_duration_seconds_.Add({{"type", "index"}}, BucketBoundaries{1e5, 2e5, 4e5, 6e5, 8e5}); + prometheus::Histogram &search_raw_data_duration_seconds_histogram_ = + search_data_duration_seconds_.Add({{"type", "raw"}}, BucketBoundaries{1e5, 2e5, 4e5, 6e5, 8e5}); ////all form Cache.cpp @@ -263,7 +420,8 @@ class PrometheusMetrics: public MetricsBase { // .Name("meta_visit_duration_seconds") // .Help("histogram of processing time to get data from mata") // .Register(*registry_); -// prometheus::Histogram &meta_visit_duration_seconds_histogram_ = meta_visit_duration_seconds_.Add({{}}, BucketBoundaries{0.1, 1.0, 10.0}); +// prometheus::Histogram &meta_visit_duration_seconds_histogram_ = +// meta_visit_duration_seconds_.Add({{}}, BucketBoundaries{0.1, 1.0, 10.0}); ////all from MemManager.cpp @@ -281,8 +439,6 @@ class PrometheusMetrics: public MetricsBase { .Register(*registry_); prometheus::Gauge &mem_usage_total_gauge_ = mem_usage_total_.Add({}); - - ////all from DBMetaImpl.cpp //record meta access count prometheus::Family &meta_access_ = prometheus::BuildCounter() @@ -296,9 +452,8 @@ class PrometheusMetrics: public MetricsBase { .Name("meta_access_duration_microseconds") .Help("histogram of processing time for accessing mata") .Register(*registry_); - prometheus::Histogram &meta_access_duration_seconds_histogram_ = meta_access_duration_seconds_.Add({}, BucketBoundaries{100, 300, 500, 700, 900, 2000, 4000, 6000, 8000, 20000}); - - + prometheus::Histogram &meta_access_duration_seconds_histogram_ = + meta_access_duration_seconds_.Add({}, BucketBoundaries{100, 300, 500, 700, 900, 2000, 4000, 6000, 8000, 20000}); ////all from FaissExecutionEngine.cpp //record data loading from disk count, size, duration, IO speed @@ -306,26 +461,28 @@ class PrometheusMetrics: public MetricsBase { .Name("disk_load_duration_microseconds") .Help("Histogram of processing time for loading data from disk") .Register(*registry_); - prometheus::Histogram &faiss_disk_load_duration_seconds_histogram_ = disk_load_duration_second_.Add({{"DB","Faiss"}},BucketBoundaries{2e5, 4e5, 6e5 , 8e5}); + prometheus::Histogram &faiss_disk_load_duration_seconds_histogram_ = + disk_load_duration_second_.Add({{"DB", "Faiss"}}, BucketBoundaries{2e5, 4e5, 6e5, 8e5}); prometheus::Family &disk_load_size_bytes_ = prometheus::BuildHistogram() .Name("disk_load_size_bytes") .Help("Histogram of data size by bytes for loading data from disk") .Register(*registry_); - prometheus::Histogram &faiss_disk_load_size_bytes_histogram_ = disk_load_size_bytes_.Add({{"DB","Faiss"}},BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9}); + prometheus::Histogram &faiss_disk_load_size_bytes_histogram_ = + disk_load_size_bytes_.Add({{"DB", "Faiss"}}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9}); // prometheus::Family &disk_load_IO_speed_ = prometheus::BuildHistogram() // .Name("disk_load_IO_speed_byte_per_sec") // .Help("Histogram of IO speed for loading data from disk") // .Register(*registry_); -// prometheus::Histogram &faiss_disk_load_IO_speed_histogram_ = disk_load_IO_speed_.Add({{"DB","Faiss"}},BucketBoundaries{1000, 2000, 3000, 4000, 6000, 8000}); +// prometheus::Histogram &faiss_disk_load_IO_speed_histogram_ = +// disk_load_IO_speed_.Add({{"DB","Faiss"}},BucketBoundaries{1000, 2000, 3000, 4000, 6000, 8000}); prometheus::Family &faiss_disk_load_IO_speed_ = prometheus::BuildGauge() .Name("disk_load_IO_speed_byte_per_microsec") .Help("disk IO speed ") .Register(*registry_); - prometheus::Gauge &faiss_disk_load_IO_speed_gauge_ = faiss_disk_load_IO_speed_.Add({{"DB","Faiss"}}); - + prometheus::Gauge &faiss_disk_load_IO_speed_gauge_ = faiss_disk_load_IO_speed_.Add({{"DB", "Faiss"}}); ////all from CacheMgr.cpp //record cache access count @@ -344,9 +501,9 @@ class PrometheusMetrics: public MetricsBase { //record GPU cache usage and % prometheus::Family &gpu_cache_usage_ = prometheus::BuildGauge() - .Name("gpu_cache_usage_bytes") - .Help("current gpu cache usage by bytes") - .Register(*registry_); + .Name("gpu_cache_usage_bytes") + .Help("current gpu cache usage by bytes") + .Register(*registry_); // record query response using Quantiles = std::vector; @@ -354,18 +511,21 @@ class PrometheusMetrics: public MetricsBase { .Name("query_response_summary") .Help("query response summary") .Register(*registry_); - prometheus::Summary &query_response_summary_ = query_response_.Add({}, Quantiles{{0.95,0.00},{0.9,0.05},{0.8,0.1}}); + prometheus::Summary + &query_response_summary_ = query_response_.Add({}, Quantiles{{0.95, 0.00}, {0.9, 0.05}, {0.8, 0.1}}); prometheus::Family &query_vector_response_ = prometheus::BuildSummary() .Name("query_vector_response_summary") .Help("query each vector response summary") .Register(*registry_); - prometheus::Summary &query_vector_response_summary_ = query_vector_response_.Add({}, Quantiles{{0.95,0.00},{0.9,0.05},{0.8,0.1}}); + prometheus::Summary &query_vector_response_summary_ = + query_vector_response_.Add({}, Quantiles{{0.95, 0.00}, {0.9, 0.05}, {0.8, 0.1}}); prometheus::Family &query_vector_response_per_second_ = prometheus::BuildGauge() .Name("query_vector_response_per_microsecond") .Help("the number of vectors can be queried every second ") - .Register(*registry_); prometheus::Gauge &query_vector_response_per_second_gauge_ = query_vector_response_per_second_.Add({}); + .Register(*registry_); + prometheus::Gauge &query_vector_response_per_second_gauge_ = query_vector_response_per_second_.Add({}); prometheus::Family &query_response_per_second_ = prometheus::BuildGauge() .Name("query_response_per_microsecond") @@ -404,7 +564,6 @@ class PrometheusMetrics: public MetricsBase { .Register(*registry_); prometheus::Gauge &CPU_usage_percent_ = CPU_.Add({{"CPU", "avg"}}); - prometheus::Family &RAM_ = prometheus::BuildGauge() .Name("RAM_usage_percent") .Help("RAM usage percent by this process") @@ -427,8 +586,10 @@ class PrometheusMetrics: public MetricsBase { .Name("query_index_throughtout_per_microsecond") .Help("query index throughtout per microsecond") .Register(*registry_); - prometheus::Gauge &query_index_IVF_type_per_second_gauge_ = query_index_type_per_second_.Add({{"IndexType","IVF"}}); - prometheus::Gauge &query_index_IDMAP_type_per_second_gauge_ = query_index_type_per_second_.Add({{"IndexType","IDMAP"}}); + prometheus::Gauge + &query_index_IVF_type_per_second_gauge_ = query_index_type_per_second_.Add({{"IndexType", "IVF"}}); + prometheus::Gauge + &query_index_IDMAP_type_per_second_gauge_ = query_index_type_per_second_.Add({{"IndexType", "IDMAP"}}); prometheus::Family &connection_ = prometheus::BuildGauge() .Name("connection_number") @@ -449,7 +610,6 @@ class PrometheusMetrics: public MetricsBase { prometheus::Gauge &inoctets_gauge_ = octets_.Add({{"type", "inoctets"}}); prometheus::Gauge &outoctets_gauge_ = octets_.Add({{"type", "outoctets"}}); - prometheus::Family &GPU_temperature_ = prometheus::BuildGauge() .Name("GPU_temperature") .Help("GPU temperature") @@ -461,9 +621,6 @@ class PrometheusMetrics: public MetricsBase { .Register(*registry_); }; -} -} -} - - - +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/metrics/SystemInfo.cpp b/cpp/src/metrics/SystemInfo.cpp index 5db2448ee9..54d0c51943 100644 --- a/cpp/src/metrics/SystemInfo.cpp +++ b/cpp/src/metrics/SystemInfo.cpp @@ -16,29 +16,28 @@ // under the License. -#include "SystemInfo.h" +#include "metrics/SystemInfo.h" #include #include #include #include -#include "nvml.h" -//#include -// -//std::mutex mutex; - +#include +#include +#include namespace zilliz { namespace milvus { namespace server { -void SystemInfo::Init() { - if(initialized_) return; +void +SystemInfo::Init() { + if (initialized_) return; initialized_ = true; // initialize CPU information - FILE* file; + FILE *file; struct tms time_sample; char line[128]; last_cpu_ = times(&time_sample); @@ -46,7 +45,7 @@ void SystemInfo::Init() { last_user_cpu_ = time_sample.tms_utime; file = fopen("/proc/cpuinfo", "r"); num_processors_ = 0; - while(fgets(line, 128, file) != NULL){ + while (fgets(line, 128, file) != NULL) { if (strncmp(line, "processor", 9) == 0) num_processors_++; if (strncmp(line, "physical", 8) == 0) { num_physical_processors_ = ParseLine(line); @@ -58,24 +57,24 @@ void SystemInfo::Init() { //initialize GPU information nvmlReturn_t nvmlresult; nvmlresult = nvmlInit(); - if(NVML_SUCCESS != nvmlresult) { + if (NVML_SUCCESS != nvmlresult) { printf("System information initilization failed"); - return ; + return; } nvmlresult = nvmlDeviceGetCount(&num_device_); - if(NVML_SUCCESS != nvmlresult) { + if (NVML_SUCCESS != nvmlresult) { printf("Unable to get devidce number"); - return ; + return; } //initialize network traffic information - std::pair in_and_out_octets = Octets(); + std::pair in_and_out_octets = Octets(); in_octets_ = in_and_out_octets.first; out_octets_ = in_and_out_octets.second; net_time_ = std::chrono::system_clock::now(); } -long long +uint64_t SystemInfo::ParseLine(char *line) { // This assumes that a digit will be found and the line ends in " Kb". int i = strlen(line); @@ -83,53 +82,52 @@ SystemInfo::ParseLine(char *line) { while (*p < '0' || *p > '9') p++; line[i - 3] = '\0'; i = atoi(p); - return static_cast(i); + return static_cast(i); } -unsigned long +uint64_t SystemInfo::GetPhysicalMemory() { struct sysinfo memInfo; - sysinfo (&memInfo); - unsigned long totalPhysMem = memInfo.totalram; + sysinfo(&memInfo); + uint64_t totalPhysMem = memInfo.totalram; //Multiply in next statement to avoid int overflow on right hand side... totalPhysMem *= memInfo.mem_unit; return totalPhysMem; } -unsigned long +uint64_t SystemInfo::GetProcessUsedMemory() { //Note: this value is in KB! - FILE* file = fopen("/proc/self/status", "r"); - constexpr int64_t line_length = 128; - long long result = -1; - constexpr int64_t KB_SIZE = 1024; + FILE *file = fopen("/proc/self/status", "r"); + constexpr uint64_t line_length = 128; + uint64_t result = -1; + constexpr uint64_t KB_SIZE = 1024; char line[line_length]; - while (fgets(line, line_length, file) != NULL){ - if (strncmp(line, "VmRSS:", 6) == 0){ + while (fgets(line, line_length, file) != NULL) { + if (strncmp(line, "VmRSS:", 6) == 0) { result = ParseLine(line); break; } } fclose(file); // return value in Byte - return (result*KB_SIZE); - + return (result * KB_SIZE); } double SystemInfo::MemoryPercent() { if (!initialized_) Init(); - return (double)(GetProcessUsedMemory()*100)/(double)total_ram_; + return (double) (GetProcessUsedMemory() * 100) / (double) total_ram_; } std::vector SystemInfo::CPUCorePercent() { - std::vector prev_work_time_array; - std::vector prev_total_time_array = getTotalCpuTime(prev_work_time_array); + std::vector prev_work_time_array; + std::vector prev_total_time_array = getTotalCpuTime(prev_work_time_array); usleep(100000); - std::vector cur_work_time_array; - std::vector cur_total_time_array = getTotalCpuTime(cur_work_time_array); + std::vector cur_work_time_array; + std::vector cur_total_time_array = getTotalCpuTime(cur_work_time_array); std::vector cpu_core_percent; for (int i = 1; i < num_processors_; i++) { @@ -140,22 +138,21 @@ SystemInfo::CPUCorePercent() { return cpu_core_percent; } -std::vector -SystemInfo::getTotalCpuTime(std::vector &work_time_array) -{ - std::vector total_time_array; - FILE* file = fopen("/proc/stat", "r"); +std::vector +SystemInfo::getTotalCpuTime(std::vector &work_time_array) { + std::vector total_time_array; + FILE *file = fopen("/proc/stat", "r"); if (file == NULL) { perror("Could not open stat file"); return total_time_array; } - unsigned long long user = 0, nice = 0, system = 0, idle = 0; - unsigned long long iowait = 0, irq = 0, softirq = 0, steal = 0, guest = 0, guestnice = 0; + uint64_t user = 0, nice = 0, system = 0, idle = 0; + uint64_t iowait = 0, irq = 0, softirq = 0, steal = 0, guest = 0, guestnice = 0; for (int i = 0; i < num_processors_; i++) { char buffer[1024]; - char* ret = fgets(buffer, sizeof(buffer) - 1, file); + char *ret = fgets(buffer, sizeof(buffer) - 1, file); if (ret == NULL) { perror("Could not read stat file"); fclose(file); @@ -163,7 +160,7 @@ SystemInfo::getTotalCpuTime(std::vector &work_time_array) } sscanf(buffer, - "cpu %16llu %16llu %16llu %16llu %16llu %16llu %16llu %16llu %16llu %16llu", + "cpu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu", &user, &nice, &system, &idle, &iowait, &irq, &softirq, &steal, &guest, &guestnice); work_time_array.push_back(user + nice + system); @@ -174,9 +171,6 @@ SystemInfo::getTotalCpuTime(std::vector &work_time_array) return total_time_array; } - - - double SystemInfo::CPUPercent() { if (!initialized_) Init(); @@ -186,11 +180,10 @@ SystemInfo::CPUPercent() { now = times(&time_sample); if (now <= last_cpu_ || time_sample.tms_stime < last_sys_cpu_ || - time_sample.tms_utime < last_user_cpu_){ + time_sample.tms_utime < last_user_cpu_) { //Overflow detection. Just skip this value. percent = -1.0; - } - else{ + } else { percent = (time_sample.tms_stime - last_sys_cpu_) + (time_sample.tms_utime - last_user_cpu_); percent /= (now - last_cpu_); @@ -203,12 +196,11 @@ SystemInfo::CPUPercent() { return percent; } - -std::vector +std::vector SystemInfo::GPUMemoryTotal() { // get GPU usage percent - if(!initialized_) Init(); - std::vector result; + if (!initialized_) Init(); + std::vector result; nvmlMemory_t nvmlMemory; for (int i = 0; i < num_device_; ++i) { nvmlDevice_t device; @@ -219,21 +211,22 @@ SystemInfo::GPUMemoryTotal() { return result; } -std::vector -SystemInfo::GPUTemperature(){ - if(!initialized_) Init(); - std::vector result; +std::vector +SystemInfo::GPUTemperature() { + if (!initialized_) Init(); + std::vector result; for (int i = 0; i < num_device_; i++) { nvmlDevice_t device; nvmlDeviceGetHandleByIndex(i, &device); unsigned int temp; - nvmlDeviceGetTemperature(device, NVML_TEMPERATURE_GPU,&temp); + nvmlDeviceGetTemperature(device, NVML_TEMPERATURE_GPU, &temp); result.push_back(temp); } return result; } + std::vector -SystemInfo::CPUTemperature(){ +SystemInfo::CPUTemperature() { std::vector result; for (int i = 0; i <= num_physical_processors_; ++i) { std::string path = "/sys/class/thermal/thermal_zone" + std::to_string(i) + "/temp"; @@ -247,15 +240,14 @@ SystemInfo::CPUTemperature(){ result.push_back(temp / 1000); fclose(file); } - } -std::vector +std::vector SystemInfo::GPUMemoryUsed() { // get GPU memory used - if(!initialized_) Init(); + if (!initialized_) Init(); - std::vector result; + std::vector result; nvmlMemory_t nvmlMemory; for (int i = 0; i < num_device_; ++i) { nvmlDevice_t device; @@ -266,42 +258,41 @@ SystemInfo::GPUMemoryUsed() { return result; } -std::pair -SystemInfo::Octets(){ +std::pair +SystemInfo::Octets() { pid_t pid = getpid(); // const std::string filename = "/proc/"+std::to_string(pid)+"/net/netstat"; const std::string filename = "/proc/net/netstat"; std::ifstream file(filename); std::string lastline = ""; std::string line = ""; - while(file){ + while (file) { getline(file, line); - if(file.fail()){ + if (file.fail()) { break; } lastline = line; } std::vector space_position; size_t space_pos = lastline.find(" "); - while(space_pos != std::string::npos){ + while (space_pos != std::string::npos) { space_position.push_back(space_pos); - space_pos = lastline.find(" ",space_pos+1); + space_pos = lastline.find(" ", space_pos + 1); } // InOctets is between 6th and 7th " " and OutOctets is between 7th and 8th " " - size_t inoctets_begin = space_position[6]+1; - size_t inoctets_length = space_position[7]-inoctets_begin; - size_t outoctets_begin = space_position[7]+1; - size_t outoctets_length = space_position[8]-outoctets_begin; - std::string inoctets = lastline.substr(inoctets_begin,inoctets_length); - std::string outoctets = lastline.substr(outoctets_begin,outoctets_length); + size_t inoctets_begin = space_position[6] + 1; + size_t inoctets_length = space_position[7] - inoctets_begin; + size_t outoctets_begin = space_position[7] + 1; + size_t outoctets_length = space_position[8] - outoctets_begin; + std::string inoctets = lastline.substr(inoctets_begin, inoctets_length); + std::string outoctets = lastline.substr(outoctets_begin, outoctets_length); - - unsigned long long inoctets_bytes = std::stoull(inoctets); - unsigned long long outoctets_bytes = std::stoull(outoctets); - std::pair res(inoctets_bytes, outoctets_bytes); + uint64_t inoctets_bytes = std::stoull(inoctets); + uint64_t outoctets_bytes = std::stoull(outoctets); + std::pair res(inoctets_bytes, outoctets_bytes); return res; } -} -} -} \ No newline at end of file +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/metrics/SystemInfo.h b/cpp/src/metrics/SystemInfo.h index 9089c7cddb..802cbb0cce 100644 --- a/cpp/src/metrics/SystemInfo.h +++ b/cpp/src/metrics/SystemInfo.h @@ -18,19 +18,17 @@ #pragma once -#include "sys/types.h" -#include "sys/sysinfo.h" -#include "stdlib.h" -#include "stdio.h" -#include "string.h" -#include "sys/times.h" -#include "sys/vtimes.h" +#include +#include +#include +#include +#include +#include +#include #include - #include #include - - +#include namespace zilliz { namespace milvus { @@ -38,7 +36,7 @@ namespace server { class SystemInfo { private: - unsigned long total_ram_ = 0; + uint64_t total_ram_ = 0; clock_t last_cpu_ = clock_t(); clock_t last_sys_cpu_ = clock_t(); clock_t last_user_cpu_ = clock_t(); @@ -46,44 +44,71 @@ class SystemInfo { int num_processors_ = 0; int num_physical_processors_ = 0; //number of GPU - unsigned int num_device_ = 0; - unsigned long long in_octets_ = 0; - unsigned long long out_octets_ = 0; + uint32_t num_device_ = 0; + uint64_t in_octets_ = 0; + uint64_t out_octets_ = 0; bool initialized_ = false; public: static SystemInfo & - GetInstance(){ + GetInstance() { static SystemInfo instance; return instance; } void Init(); - int num_processor() const { return num_processors_;}; - int num_physical_processors() const { return num_physical_processors_; }; - int num_device() const {return num_device_;}; - unsigned long long get_inoctets() { return in_octets_;}; - unsigned long long get_octets() { return out_octets_;}; - std::chrono::system_clock::time_point get_nettime() { return net_time_;}; - void set_inoctets(unsigned long long value) { in_octets_ = value;}; - void set_outoctets(unsigned long long value) { out_octets_ = value;}; - void set_nettime() {net_time_ = std::chrono::system_clock::now();}; - long long ParseLine(char* line); - unsigned long GetPhysicalMemory(); - unsigned long GetProcessUsedMemory(); + + int num_processor() const { + return num_processors_; + } + + int num_physical_processors() const { + return num_physical_processors_; + } + + uint32_t num_device() const { + return num_device_; + } + + uint64_t get_inoctets() { + return in_octets_; + } + + uint64_t get_octets() { + return out_octets_; + } + + std::chrono::system_clock::time_point get_nettime() { + return net_time_; + } + + void set_inoctets(uint64_t value) { + in_octets_ = value; + } + + void set_outoctets(uint64_t value) { + out_octets_ = value; + } + + void set_nettime() { + net_time_ = std::chrono::system_clock::now(); + } + + uint64_t ParseLine(char *line); + uint64_t GetPhysicalMemory(); + uint64_t GetProcessUsedMemory(); double MemoryPercent(); double CPUPercent(); - std::pair Octets(); - std::vector GPUMemoryTotal(); - std::vector GPUMemoryUsed(); + std::pair Octets(); + std::vector GPUMemoryTotal(); + std::vector GPUMemoryUsed(); std::vector CPUCorePercent(); - std::vector getTotalCpuTime(std::vector &workTime); - std::vector GPUTemperature(); + std::vector getTotalCpuTime(std::vector &workTime); + std::vector GPUTemperature(); std::vector CPUTemperature(); - }; -} -} -} +} // namespace server +} // namespace milvus +} // namespace zilliz From 180600fb244317a42d677ebf5bdf55a8d495898e Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Fri, 27 Sep 2019 15:41:20 +0800 Subject: [PATCH 134/217] MS-565 remove jsoncons v2 Former-commit-id: a37cbd9b67c66fca6e4ef08519e5f48248985c98 --- cpp/src/core/knowhere/CMakeLists.txt | 3 - cpp/src/core/test/test_ivf.cpp | 4 +- cpp/src/db/engine/ExecutionEngineImpl.cpp | 51 ++++--- cpp/src/wrapper/ConfAdapter.cpp | 161 ++++++++++++++++++++++ cpp/src/wrapper/ConfAdapter.h | 91 ++++++++++++ cpp/src/wrapper/ConfAdapterMgr.cpp | 64 +++++++++ cpp/src/wrapper/ConfAdapterMgr.h | 63 +++++++++ cpp/src/wrapper/VecImpl.cpp | 35 ++--- cpp/src/wrapper/VecIndex.cpp | 71 +--------- cpp/src/wrapper/VecIndex.h | 7 - cpp/unittest/wrapper/wrapper_test.cpp | 157 ++++++++++++--------- 11 files changed, 519 insertions(+), 188 deletions(-) create mode 100644 cpp/src/wrapper/ConfAdapter.cpp create mode 100644 cpp/src/wrapper/ConfAdapter.h create mode 100644 cpp/src/wrapper/ConfAdapterMgr.cpp create mode 100644 cpp/src/wrapper/ConfAdapterMgr.h diff --git a/cpp/src/core/knowhere/CMakeLists.txt b/cpp/src/core/knowhere/CMakeLists.txt index 864d03583a..4d596e243a 100644 --- a/cpp/src/core/knowhere/CMakeLists.txt +++ b/cpp/src/core/knowhere/CMakeLists.txt @@ -117,7 +117,6 @@ set(CORE_INCLUDE_DIRS ${CORE_SOURCE_DIR}/knowhere ${CORE_SOURCE_DIR}/thirdparty ${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService - ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include ${ARROW_INCLUDE_DIR} ${FAISS_INCLUDE_DIR} ${OPENBLAS_INCLUDE_DIR} @@ -129,8 +128,6 @@ set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) #INSTALL(DIRECTORY # ${CORE_SOURCE_DIR}/include/knowhere -# ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons -# ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext # ${ARROW_INCLUDE_DIR}/arrow # ${FAISS_PREFIX}/include/faiss # ${OPENBLAS_INCLUDE_DIR}/ diff --git a/cpp/src/core/test/test_ivf.cpp b/cpp/src/core/test/test_ivf.cpp index 579729f2a1..1eeb1f9872 100644 --- a/cpp/src/core/test/test_ivf.cpp +++ b/cpp/src/core/test/test_ivf.cpp @@ -146,8 +146,8 @@ INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest, Values( std::make_tuple("IVF", ParameterType::ivf), std::make_tuple("GPUIVF", ParameterType::ivf), - std::make_tuple("IVFPQ", ParameterType::ivfpq), - std::make_tuple("GPUIVFPQ", ParameterType::ivfpq), +// std::make_tuple("IVFPQ", ParameterType::ivfpq), +// std::make_tuple("GPUIVFPQ", ParameterType::ivfpq), std::make_tuple("IVFSQ", ParameterType::ivfsq), std::make_tuple("GPUIVFSQ", ParameterType::ivfsq) ) diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index b9d18b4a2f..1541cb051b 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -26,6 +26,9 @@ #include "src/wrapper/VecIndex.h" #include "src/wrapper/VecImpl.h" #include "knowhere/common/Exception.h" +#include "knowhere/common/Config.h" +#include "wrapper/ConfAdapterMgr.h" +#include "wrapper/ConfAdapter.h" #include "server/Config.h" #include @@ -50,11 +53,15 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, throw Exception(DB_ERROR, "Could not create VecIndex"); } - Config build_cfg; - build_cfg["dim"] = dimension; - build_cfg["metric_type"] = (metric_type_ == MetricType::IP) ? "IP" : "L2"; - AutoGenParams(index_->GetType(), 0, build_cfg); - auto ec = std::static_pointer_cast(index_)->Build(build_cfg); + TempMetaConf temp_conf; + temp_conf.gpu_id = gpu_num_; + temp_conf.dim = dimension; + temp_conf.metric_type = (metric_type_ == MetricType::IP) ? + knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2; + auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType()); + auto conf = adapter->Match(temp_conf); + + auto ec = std::static_pointer_cast(index_)->Build(conf); if (ec != KNOWHERE_SUCCESS) { throw Exception(DB_ERROR, "Build index error"); } @@ -274,17 +281,21 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t throw Exception(DB_ERROR, "Could not create VecIndex"); } - Config build_cfg; - build_cfg["dim"] = Dimension(); - build_cfg["metric_type"] = (metric_type_ == MetricType::IP) ? "IP" : "L2"; - build_cfg["gpu_id"] = gpu_num_; - build_cfg["nlist"] = nlist_; - AutoGenParams(to_index->GetType(), Count(), build_cfg); + TempMetaConf temp_conf; + temp_conf.gpu_id = gpu_num_; + temp_conf.dim = Dimension(); + temp_conf.nlist = nlist_; + temp_conf.metric_type = (metric_type_ == MetricType::IP) ? + knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2; + temp_conf.size = Count(); + + auto adapter = AdapterMgr::GetInstance().GetAdapter(to_index->GetType()); + auto conf = adapter->Match(temp_conf); auto status = to_index->BuildAll(Count(), - from_index->GetRawVectors(), - from_index->GetRawIds(), - build_cfg); + from_index->GetRawVectors(), + from_index->GetRawIds(), + conf); if (!status.ok()) { throw Exception(DB_ERROR, status.message()); } return std::make_shared(to_index, location, engine_type, metric_type_, nlist_); @@ -302,8 +313,16 @@ Status ExecutionEngineImpl::Search(long n, } ENGINE_LOG_DEBUG << "Search Params: [k] " << k << " [nprobe] " << nprobe; - auto cfg = Config::object{{"k", k}, {"nprobe", nprobe}}; - auto status = index_->Search(n, data, distances, labels, cfg); + + // TODO(linxj): remove here. Get conf from function + TempMetaConf temp_conf; + temp_conf.k = k; + temp_conf.nprobe = nprobe; + + auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType()); + auto conf = adapter->MatchSearch(temp_conf, index_->GetType()); + + auto status = index_->Search(n, data, distances, labels, conf); if (!status.ok()) { ENGINE_LOG_ERROR << "Search error"; } diff --git a/cpp/src/wrapper/ConfAdapter.cpp b/cpp/src/wrapper/ConfAdapter.cpp new file mode 100644 index 0000000000..ea6be1acf3 --- /dev/null +++ b/cpp/src/wrapper/ConfAdapter.cpp @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include +#include "ConfAdapter.h" +#include "src/utils/Log.h" +#include "knowhere/index/vector_index/helpers/IndexParameter.h" + +// TODO: add conf checker + +namespace zilliz { +namespace milvus { +namespace engine { + +#if CUDA_VERSION > 9000 +#define GPU_MAX_NRPOBE 2048 +#else +#define GPU_MAX_NRPOBE 1024 +#endif + +void +ConfAdapter::MatchBase(knowhere::Config conf) { + if (conf->metric_type == knowhere::DEFAULT_TYPE) + conf->metric_type = knowhere::METRICTYPE::L2; + if (conf->gpu_id == knowhere::INVALID_VALUE) + conf->gpu_id = 0; +} + +knowhere::Config +ConfAdapter::Match(const TempMetaConf &metaconf) { + auto conf = std::make_shared(); + conf->d = metaconf.dim; + conf->metric_type = metaconf.metric_type; + conf->gpu_id = conf->gpu_id; + MatchBase(conf); + return conf; +} + +knowhere::Config +ConfAdapter::MatchSearch(const TempMetaConf &metaconf, const IndexType &type) { + auto conf = std::make_shared(); + conf->k = metaconf.k; + return conf; +} + +knowhere::Config +IVFConfAdapter::Match(const TempMetaConf &metaconf) { + auto conf = std::make_shared(); + conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); + conf->d = metaconf.dim; + conf->metric_type = metaconf.metric_type; + conf->gpu_id = conf->gpu_id; + MatchBase(conf); + return conf; +} + +static constexpr float TYPICAL_COUNT = 1000000.0; + +int64_t +IVFConfAdapter::MatchNlist(const int64_t &size, const int64_t &nlist) { + if (size <= TYPICAL_COUNT / 16384 + 1) { + // handle less row count, avoid nlist set to 0 + return 1; + } else if (int(size / TYPICAL_COUNT) * nlist == 0) { + // calculate a proper nlist if nlist not specified or size less than TYPICAL_COUNT + return int(size / TYPICAL_COUNT * 16384); + } + return 0; +} + +knowhere::Config +IVFConfAdapter::MatchSearch(const TempMetaConf &metaconf, const IndexType &type) { + auto conf = std::make_shared(); + conf->k = metaconf.k; + conf->nprobe = metaconf.nprobe; + + switch (type) { + case IndexType::FAISS_IVFFLAT_GPU: + case IndexType::FAISS_IVFSQ8_GPU: + case IndexType::FAISS_IVFPQ_GPU: + if (conf->nprobe > GPU_MAX_NRPOBE) { + WRAPPER_LOG_WARNING << "When search with GPU, nprobe shoud be no more than " << GPU_MAX_NRPOBE + << ", but you passed " << conf->nprobe + << ". Search with " << GPU_MAX_NRPOBE << " instead"; + conf->nprobe = GPU_MAX_NRPOBE; + + } + } + return conf; +} + +knowhere::Config +IVFSQConfAdapter::Match(const TempMetaConf &metaconf) { + auto conf = std::make_shared(); + conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); + conf->d = metaconf.dim; + conf->metric_type = metaconf.metric_type; + conf->gpu_id = conf->gpu_id; + conf->nbits = 8; + MatchBase(conf); + return conf; +} + +knowhere::Config +IVFPQConfAdapter::Match(const TempMetaConf &metaconf) { + auto conf = std::make_shared(); + conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); + conf->d = metaconf.dim; + conf->metric_type = metaconf.metric_type; + conf->gpu_id = conf->gpu_id; + conf->nbits = 8; + conf->m = 8; + MatchBase(conf); + return conf; +} + +knowhere::Config +NSGConfAdapter::Match(const TempMetaConf &metaconf) { + auto conf = std::make_shared(); + conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); + conf->d = metaconf.dim; + conf->metric_type = metaconf.metric_type; + conf->gpu_id = conf->gpu_id; + + auto scale_factor = round(metaconf.dim / 128.0); + scale_factor = scale_factor >= 4 ? 4 : scale_factor; + conf->nprobe = 6 + 10 * scale_factor; + conf->knng = 100 + 100 * scale_factor; + conf->search_length = 40 + 5 * scale_factor; + conf->out_degree = 50 + 5 * scale_factor; + conf->candidate_pool_size = 200 + 100 * scale_factor; + MatchBase(conf); + return conf; +} + +knowhere::Config +NSGConfAdapter::MatchSearch(const TempMetaConf &metaconf, const IndexType &type) { + auto conf = std::make_shared(); + conf->k = metaconf.k; + conf->search_length = metaconf.search_length; + return conf; +} + +} +} +} diff --git a/cpp/src/wrapper/ConfAdapter.h b/cpp/src/wrapper/ConfAdapter.h new file mode 100644 index 0000000000..d0f032cb61 --- /dev/null +++ b/cpp/src/wrapper/ConfAdapter.h @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include "knowhere/common/Config.h" +#include "VecIndex.h" + + +namespace zilliz { +namespace milvus { +namespace engine { + +// TODO(linxj): remove later, replace with real metaconf +constexpr int64_t TEMPMETA_DEFAULT_VALUE = -1; +struct TempMetaConf { + int64_t size = TEMPMETA_DEFAULT_VALUE; + int64_t nlist = TEMPMETA_DEFAULT_VALUE; + int64_t dim = TEMPMETA_DEFAULT_VALUE; + int64_t gpu_id = TEMPMETA_DEFAULT_VALUE; + int64_t k = TEMPMETA_DEFAULT_VALUE; + int64_t nprobe = TEMPMETA_DEFAULT_VALUE; + int64_t search_length = TEMPMETA_DEFAULT_VALUE; + knowhere::METRICTYPE metric_type = knowhere::DEFAULT_TYPE; +}; + +class ConfAdapter { + public: + virtual knowhere::Config + Match(const TempMetaConf &metaconf); + + virtual knowhere::Config + MatchSearch(const TempMetaConf &metaconf, const IndexType &type); + + protected: + static void MatchBase(knowhere::Config conf); +}; + +using ConfAdapterPtr = std::shared_ptr; + +class IVFConfAdapter : public ConfAdapter { + public: + knowhere::Config + Match(const TempMetaConf &metaconf) override; + + knowhere::Config + MatchSearch(const TempMetaConf &metaconf, const IndexType &type) override; + + protected: + static int64_t MatchNlist(const int64_t &size, const int64_t &nlist); +}; + +class IVFSQConfAdapter : public IVFConfAdapter { + public: + knowhere::Config + Match(const TempMetaConf &metaconf) override; +}; + +class IVFPQConfAdapter : public IVFConfAdapter { + public: + knowhere::Config + Match(const TempMetaConf &metaconf) override; +}; + +class NSGConfAdapter : public IVFConfAdapter { + public: + knowhere::Config + Match(const TempMetaConf &metaconf) override; + + knowhere::Config + MatchSearch(const TempMetaConf &metaconf, const IndexType &type) final; +}; + +} +} +} diff --git a/cpp/src/wrapper/ConfAdapterMgr.cpp b/cpp/src/wrapper/ConfAdapterMgr.cpp new file mode 100644 index 0000000000..cb7331a99d --- /dev/null +++ b/cpp/src/wrapper/ConfAdapterMgr.cpp @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include "src/utils/Exception.h" +#include "ConfAdapterMgr.h" + + +namespace zilliz { +namespace milvus { +namespace engine { + +ConfAdapterPtr +AdapterMgr::GetAdapter(const IndexType &indexType) { + if (!init_) RegisterAdapter(); + + auto it = table_.find(indexType); + if (it != table_.end()) { + return it->second(); + } else { + throw Exception(KNOWHERE_INVALID_ARGUMENT, "Can not find this type of confadapter"); + } +} + + +#define REGISTER_CONF_ADAPTER(T, KEY, NAME) static AdapterMgr::register_treg_##NAME##_(KEY) +void +AdapterMgr::RegisterAdapter() { + init_ = true; + + REGISTER_CONF_ADAPTER(ConfAdapter, IndexType::FAISS_IDMAP, idmap); + + REGISTER_CONF_ADAPTER(IVFConfAdapter, IndexType::FAISS_IVFFLAT_CPU, ivf_cpu); + REGISTER_CONF_ADAPTER(IVFConfAdapter, IndexType::FAISS_IVFFLAT_GPU, ivf_gpu); + REGISTER_CONF_ADAPTER(IVFConfAdapter, IndexType::FAISS_IVFFLAT_MIX, ivf_mix); + + REGISTER_CONF_ADAPTER(IVFSQConfAdapter, IndexType::FAISS_IVFSQ8_CPU, ivfsq8_cpu); + REGISTER_CONF_ADAPTER(IVFSQConfAdapter, IndexType::FAISS_IVFSQ8_GPU, ivfsq8_gpu); + REGISTER_CONF_ADAPTER(IVFSQConfAdapter, IndexType::FAISS_IVFSQ8_MIX, ivfsq8_mix); + + REGISTER_CONF_ADAPTER(IVFPQConfAdapter, IndexType::FAISS_IVFPQ_CPU, ivfpq_cpu); + REGISTER_CONF_ADAPTER(IVFPQConfAdapter, IndexType::FAISS_IVFPQ_GPU, ivfpq_gpu); + + REGISTER_CONF_ADAPTER(NSGConfAdapter, IndexType::NSG_MIX, nsg_mix); +} + +} // engine +} // milvus +} // zilliz + diff --git a/cpp/src/wrapper/ConfAdapterMgr.h b/cpp/src/wrapper/ConfAdapterMgr.h new file mode 100644 index 0000000000..fe8b4f07fa --- /dev/null +++ b/cpp/src/wrapper/ConfAdapterMgr.h @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include "VecIndex.h" +#include "ConfAdapter.h" + + +namespace zilliz { +namespace milvus { +namespace engine { + +class AdapterMgr { + public: + template + struct register_t { + explicit register_t(const IndexType &key) { + AdapterMgr::GetInstance().table_.emplace(key, [] { + return std::make_shared(); + }); + } + }; + + static AdapterMgr & + GetInstance() { + static AdapterMgr instance; + return instance; + } + + ConfAdapterPtr + GetAdapter(const IndexType &indexType); + + void + RegisterAdapter(); + + protected: + bool init_ = false; + std::map > table_; +}; + + +} // engine +} // milvus +} // zilliz + + + diff --git a/cpp/src/wrapper/VecImpl.cpp b/cpp/src/wrapper/VecImpl.cpp index 24f6a2bb0e..e536fa886f 100644 --- a/cpp/src/wrapper/VecImpl.cpp +++ b/cpp/src/wrapper/VecImpl.cpp @@ -24,6 +24,10 @@ #include "VecImpl.h" #include "DataTransfer.h" +/* + * no parameter check in this layer. + * only responible for index combination + */ namespace zilliz { namespace milvus { @@ -39,7 +43,7 @@ VecIndexImpl::BuildAll(const long &nb, const long &nt, const float *xt) { try { - dim = cfg["dim"].as(); + dim = cfg->d; auto dataset = GenDatasetWithIds(nb, dim, xb, ids); auto preprocessor = index_->BuildPreprocessor(dataset, cfg); @@ -50,9 +54,6 @@ VecIndexImpl::BuildAll(const long &nb, } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (jsoncons::json_exception &e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_INVALID_ARGUMENT, e.what()); } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_ERROR, e.what()); @@ -69,9 +70,6 @@ VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const Config } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (jsoncons::json_exception &e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_INVALID_ARGUMENT, e.what()); } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_ERROR, e.what()); @@ -82,13 +80,11 @@ VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const Config Status VecIndexImpl::Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) { try { - auto k = cfg["k"].as(); + auto k = cfg->k; auto dataset = GenDataset(nq, dim, xq); Config search_cfg = cfg; - ParameterValidation(type, search_cfg); - auto res = index_->Search(dataset, search_cfg); auto ids_array = res->array()[0]; auto dis_array = res->array()[1]; @@ -120,9 +116,6 @@ VecIndexImpl::Search(const long &nq, const float *xq, float *dist, long *ids, co } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (jsoncons::json_exception &e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_INVALID_ARGUMENT, e.what()); } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_ERROR, e.what()); @@ -208,14 +201,11 @@ BFIndex::GetRawIds() { ErrorCode BFIndex::Build(const Config &cfg) { try { - dim = cfg["dim"].as(); + dim = cfg->d; std::static_pointer_cast(index_)->Train(cfg); } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return KNOWHERE_UNEXPECTED_ERROR; - } catch (jsoncons::json_exception &e) { - WRAPPER_LOG_ERROR << e.what(); - return KNOWHERE_INVALID_ARGUMENT; } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); return KNOWHERE_ERROR; @@ -231,7 +221,7 @@ BFIndex::BuildAll(const long &nb, const long &nt, const float *xt) { try { - dim = cfg["dim"].as(); + dim = cfg->d; auto dataset = GenDatasetWithIds(nb, dim, xb, ids); std::static_pointer_cast(index_)->Train(cfg); @@ -239,9 +229,6 @@ BFIndex::BuildAll(const long &nb, } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (jsoncons::json_exception &e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_INVALID_ARGUMENT, e.what()); } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_ERROR, e.what()); @@ -258,7 +245,7 @@ IVFMixIndex::BuildAll(const long &nb, const long &nt, const float *xt) { try { - dim = cfg["dim"].as(); + dim = cfg->d; auto dataset = GenDatasetWithIds(nb, dim, xb, ids); auto preprocessor = index_->BuildPreprocessor(dataset, cfg); @@ -278,9 +265,6 @@ IVFMixIndex::BuildAll(const long &nb, } catch (KnowhereException &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (jsoncons::json_exception &e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_INVALID_ARGUMENT, e.what()); } catch (std::exception &e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_ERROR, e.what()); @@ -290,7 +274,6 @@ IVFMixIndex::BuildAll(const long &nb, Status IVFMixIndex::Load(const zilliz::knowhere::BinarySet &index_binary) { - //index_ = std::make_shared(); index_->Load(index_binary); dim = Dimension(); return Status::OK(); diff --git a/cpp/src/wrapper/VecIndex.cpp b/cpp/src/wrapper/VecIndex.cpp index cd8843e5ac..71d4ab96e6 100644 --- a/cpp/src/wrapper/VecIndex.cpp +++ b/cpp/src/wrapper/VecIndex.cpp @@ -36,8 +36,6 @@ namespace zilliz { namespace milvus { namespace engine { -static constexpr float TYPICAL_COUNT = 1000000.0; - struct FileIOReader { std::fstream fs; std::string name; @@ -101,7 +99,7 @@ FileIOWriter::operator()(void *ptr, size_t size) { VecIndexPtr GetVecIndexFactory(const IndexType &type, const Config &cfg) { std::shared_ptr index; - auto gpu_device = cfg.get_with_default("gpu_id", 0); + auto gpu_device = -1; // TODO(linxj): remove hardcode here switch (type) { case IndexType::FAISS_IDMAP: { index = std::make_shared(); @@ -112,12 +110,11 @@ GetVecIndexFactory(const IndexType &type, const Config &cfg) { break; } case IndexType::FAISS_IVFFLAT_GPU: { - // TODO(linxj): 规范化参数 index = std::make_shared(gpu_device); break; } case IndexType::FAISS_IVFFLAT_MIX: { - index = std::make_shared(0); + index = std::make_shared(gpu_device); return std::make_shared(index, IndexType::FAISS_IVFFLAT_MIX); } case IndexType::FAISS_IVFPQ_CPU: { @@ -242,70 +239,6 @@ write_index(VecIndexPtr index, const std::string &location) { return Status::OK(); } - -// TODO(linxj): redo here. -void -AutoGenParams(const IndexType &type, const long &size, zilliz::knowhere::Config &cfg) { - auto nlist = cfg.get_with_default("nlist", 0); - if (size <= TYPICAL_COUNT / 16384 + 1) { - //handle less row count, avoid nlist set to 0 - cfg["nlist"] = 1; - } else if (int(size / TYPICAL_COUNT) * nlist == 0) { - //calculate a proper nlist if nlist not specified or size less than TYPICAL_COUNT - cfg["nlist"] = int(size / TYPICAL_COUNT * 16384); - } - - if (!cfg.contains("gpu_id")) { cfg["gpu_id"] = int(0); } - if (!cfg.contains("metric_type")) { cfg["metric_type"] = "L2"; } - - switch (type) { - case IndexType::FAISS_IVFSQ8_MIX: { - if (!cfg.contains("nbits")) { cfg["nbits"] = int(8); } - break; - } - case IndexType::NSG_MIX: { - auto scale_factor = round(cfg["dim"].as() / 128.0); - scale_factor = scale_factor >= 4 ? 4 : scale_factor; - cfg["nlist"] = int(size / 1000000.0 * 8192); - if (!cfg.contains("nprobe")) { cfg["nprobe"] = 6 + 10 * scale_factor; } - if (!cfg.contains("knng")) { cfg["knng"] = 100 + 100 * scale_factor; } - if (!cfg.contains("search_length")) { cfg["search_length"] = 40 + 5 * scale_factor; } - if (!cfg.contains("out_degree")) { cfg["out_degree"] = 50 + 5 * scale_factor; } - if (!cfg.contains("candidate_pool_size")) { cfg["candidate_pool_size"] = 200 + 100 * scale_factor; } - WRAPPER_LOG_DEBUG << pretty_print(cfg); - break; - } - } -} - -#if CUDA_VERSION > 9000 -#define GPU_MAX_NRPOBE 2048 -#else -#define GPU_MAX_NRPOBE 1024 -#endif - -void -ParameterValidation(const IndexType &type, Config &cfg) { - switch (type) { - case IndexType::FAISS_IVFSQ8_GPU: - case IndexType::FAISS_IVFFLAT_GPU: - case IndexType::FAISS_IVFPQ_GPU: { - //search on GPU - if (cfg.get_with_default("nprobe", 0) != 0) { - auto nprobe = cfg["nprobe"].as(); - if (nprobe > GPU_MAX_NRPOBE) { - WRAPPER_LOG_WARNING << "When search with GPU, nprobe shoud be no more than " << GPU_MAX_NRPOBE - << ", but you passed " << nprobe - << ". Search with " << GPU_MAX_NRPOBE << " instead"; - cfg.insert_or_assign("nprobe", GPU_MAX_NRPOBE); - } - } - break; - } - default:break; - } -} - IndexType ConvertToCpuIndexType(const IndexType &type) { // TODO(linxj): add IDMAP diff --git a/cpp/src/wrapper/VecIndex.h b/cpp/src/wrapper/VecIndex.h index 3737270f42..4ead6aaf0a 100644 --- a/cpp/src/wrapper/VecIndex.h +++ b/cpp/src/wrapper/VecIndex.h @@ -30,7 +30,6 @@ namespace zilliz { namespace milvus { namespace engine { -// TODO(linxj): jsoncons => rapidjson or other. using Config = zilliz::knowhere::Config; enum class IndexType { @@ -116,12 +115,6 @@ GetVecIndexFactory(const IndexType &type, const Config &cfg = Config()); extern VecIndexPtr LoadVecIndex(const IndexType &index_type, const zilliz::knowhere::BinarySet &index_binary); -extern void -AutoGenParams(const IndexType &type, const long &size, Config &cfg); - -extern void -ParameterValidation(const IndexType &type, Config &cfg); - extern IndexType ConvertToCpuIndexType(const IndexType &type); diff --git a/cpp/unittest/wrapper/wrapper_test.cpp b/cpp/unittest/wrapper/wrapper_test.cpp index 4c50201fb3..d19f98076c 100644 --- a/cpp/unittest/wrapper/wrapper_test.cpp +++ b/cpp/unittest/wrapper/wrapper_test.cpp @@ -18,10 +18,12 @@ #include "utils/easylogging++.h" #include "src/wrapper/VecIndex.h" #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" +#include "knowhere/index/vector_index/helpers/IndexParameter.h" #include "utils.h" #include + INITIALIZE_EASYLOGGINGPP using namespace zilliz::milvus::engine; @@ -35,19 +37,83 @@ constexpr int64_t DIM = 128; constexpr int64_t NB = 100000; constexpr int64_t DEVICE_ID = 0; +class ParamGenerator { + public: + static ParamGenerator& GetInstance(){ + static ParamGenerator instance; + return instance; + } + + Config Gen(const IndexType& type) { + switch (type) { + case IndexType::FAISS_IDMAP: { + auto tempconf = std::make_shared(); + tempconf->metric_type = zilliz::knowhere::METRICTYPE::L2; + return tempconf; + } + case IndexType::FAISS_IVFFLAT_CPU: + case IndexType::FAISS_IVFFLAT_GPU: + case IndexType::FAISS_IVFFLAT_MIX: { + auto tempconf = std::make_shared(); + tempconf->nlist = 100; + tempconf->nprobe = 16; + tempconf->metric_type = zilliz::knowhere::METRICTYPE::L2; + return tempconf; + } + case IndexType::FAISS_IVFSQ8_CPU: + case IndexType::FAISS_IVFSQ8_GPU: + case IndexType::FAISS_IVFSQ8_MIX: { + auto tempconf = std::make_shared(); + tempconf->nlist = 100; + tempconf->nprobe = 16; + tempconf->nbits = 8; + tempconf->metric_type = zilliz::knowhere::METRICTYPE::L2; + return tempconf; + } + case IndexType::FAISS_IVFPQ_CPU: + case IndexType::FAISS_IVFPQ_GPU: { + auto tempconf = std::make_shared(); + tempconf->nlist = 100; + tempconf->nprobe = 16; + tempconf->nbits = 8; + tempconf->m = 8; + tempconf->metric_type = zilliz::knowhere::METRICTYPE::L2; + return tempconf; + } + case IndexType::NSG_MIX: { + auto tempconf = std::make_shared(); + tempconf->nlist = 100; + tempconf->nprobe = 16; + tempconf->search_length = 8; + tempconf->knng = 200; + tempconf->search_length = 40; // TODO(linxj): be 20 when search + tempconf->out_degree = 60; + tempconf->candidate_pool_size = 200; + tempconf->metric_type = zilliz::knowhere::METRICTYPE::L2; + return tempconf; + } + } + } +}; + class KnowhereWrapperTest - : public TestWithParam<::std::tuple> { + : public TestWithParam<::std::tuple> { protected: void SetUp() override { - zilliz::knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID, 1024*1024*200, 1024*1024*300, 2); + zilliz::knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID,1024*1024*200, 1024*1024*300, 2); std::string generator_type; - std::tie(index_type, generator_type, dim, nb, nq, k, train_cfg, search_cfg) = GetParam(); + std::tie(index_type, generator_type, dim, nb, nq, k) = GetParam(); auto generator = std::make_shared(); generator->GenData(dim, nb, nq, xb, xq, ids, k, gt_ids, gt_dis); index_ = GetVecIndexFactory(index_type); + + conf = ParamGenerator::GetInstance().Gen(index_type); + conf->k = k; + conf->d = dim; + conf->gpu_id = DEVICE_ID; } void TearDown() override { zilliz::knowhere::FaissGpuResourceMgr::GetInstance().Free(); @@ -81,8 +147,7 @@ class KnowhereWrapperTest protected: IndexType index_type; - Config train_cfg; - Config search_cfg; + Config conf; int dim = DIM; int nb = NB; @@ -102,53 +167,15 @@ class KnowhereWrapperTest INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest, Values( //["Index type", "Generator type", "dim", "nb", "nq", "k", "build config", "search config"] - std::make_tuple(IndexType::FAISS_IVFFLAT_CPU, "Default", - 64, 100000, 10, 10, - Config::object{{"nlist", 100}, {"dim", 64}, {"metric_type", "L2"}}, - Config::object{{"dim", 64}, {"k", 10}, {"nprobe", 10}} - ), - // to_gpu_test Failed - std::make_tuple(IndexType::FAISS_IVFFLAT_GPU, "Default", - DIM, NB, 10, 10, - Config::object{{"nlist", 100}, {"dim", DIM}, {"metric_type", "L2"}, {"gpu_id", DEVICE_ID}}, - Config::object{{"dim", DIM}, {"k", 10}, {"nprobe", 40}} - ), - std::make_tuple(IndexType::FAISS_IVFFLAT_MIX, "Default", - 64, 100000, 10, 10, - Config::object{{"nlist", 1000}, {"dim", 64}, {"metric_type", "L2"}}, - Config::object{{"dim", 64}, {"k", 10}, {"nprobe", 5}} - ), - std::make_tuple(IndexType::FAISS_IDMAP, "Default", - 64, 100000, 10, 10, - Config::object{{"dim", 64}, {"metric_type", "L2"}}, - Config::object{{"dim", 64}, {"k", 10}} - ), - std::make_tuple(IndexType::FAISS_IVFSQ8_CPU, "Default", - DIM, NB, 10, 10, - Config::object{{"dim", DIM}, {"nlist", 1000}, {"nbits", 8}, {"metric_type", "L2"}, {"gpu_id", DEVICE_ID}}, - Config::object{{"dim", DIM}, {"k", 10}, {"nprobe", 5}} - ), - std::make_tuple(IndexType::FAISS_IVFSQ8_GPU, "Default", - DIM, NB, 10, 10, - Config::object{{"dim", DIM}, {"nlist", 1000}, {"nbits", 8}, {"metric_type", "L2"}, {"gpu_id", DEVICE_ID}}, - Config::object{{"dim", DIM}, {"k", 10}, {"nprobe", 5}} - ), - std::make_tuple(IndexType::FAISS_IVFSQ8_MIX, "Default", - DIM, NB, 10, 10, - Config::object{{"dim", DIM}, {"nlist", 1000}, {"nbits", 8}, {"metric_type", "L2"}, {"gpu_id", DEVICE_ID}}, - Config::object{{"dim", DIM}, {"k", 10}, {"nprobe", 5}} - ) -// std::make_tuple(IndexType::NSG_MIX, "Default", -// 128, 250000, 10, 10, -// Config::object{{"dim", 128}, {"nlist", 8192}, {"nprobe", 16}, {"metric_type", "L2"}, -// {"knng", 200}, {"search_length", 40}, {"out_degree", 60}, {"candidate_pool_size", 200}}, -// Config::object{{"k", 10}, {"search_length", 20}} -// ) - //std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default", - // 64, 10000, 10, 10, - // Config::object{{"TPTNumber", 1}, {"dim", 64}}, - // Config::object{{"dim", 64}, {"k", 10}} - //) + std::make_tuple(IndexType::FAISS_IVFFLAT_CPU, "Default", 64, 100000, 10, 10), + std::make_tuple(IndexType::FAISS_IVFFLAT_GPU, "Default", DIM, NB, 10, 10), + std::make_tuple(IndexType::FAISS_IVFFLAT_MIX, "Default", 64, 100000, 10, 10), + std::make_tuple(IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10), + std::make_tuple(IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10), + std::make_tuple(IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10), +// std::make_tuple(IndexType::NSG_MIX, "Default", 128, 250000, 10, 10), +// std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 250000, 10, 10), + std::make_tuple(IndexType::FAISS_IDMAP, "Default", 64, 100000, 10, 10) ) ); @@ -159,8 +186,8 @@ TEST_P(KnowhereWrapperTest, BASE_TEST) { std::vector res_ids(elems); std::vector res_dis(elems); - index_->BuildAll(nb, xb.data(), ids.data(), train_cfg); - index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg); + index_->BuildAll(nb, xb.data(), ids.data(), conf); + index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf); AssertResult(res_ids, res_dis); } @@ -171,14 +198,14 @@ TEST_P(KnowhereWrapperTest, TO_GPU_TEST) { std::vector res_ids(elems); std::vector res_dis(elems); - index_->BuildAll(nb, xb.data(), ids.data(), train_cfg); - index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg); + index_->BuildAll(nb, xb.data(), ids.data(), conf); + index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf); AssertResult(res_ids, res_dis); { auto dev_idx = index_->CopyToGpu(DEVICE_ID); for (int i = 0; i < 10; ++i) { - dev_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg); + dev_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf); } AssertResult(res_ids, res_dis); } @@ -190,15 +217,15 @@ TEST_P(KnowhereWrapperTest, TO_GPU_TEST) { auto dev_idx = new_index->CopyToGpu(DEVICE_ID); for (int i = 0; i < 10; ++i) { - dev_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg); + dev_idx->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf); } AssertResult(res_ids, res_dis); } } -TEST_P(KnowhereWrapperTest, TO_CPU_TEST) { - // dev -} +//TEST_P(KnowhereWrapperTest, TO_CPU_TEST) { +// // dev +//} TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) { EXPECT_EQ(index_->GetType(), index_type); @@ -206,8 +233,8 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) { auto elems = nq * k; std::vector res_ids(elems); std::vector res_dis(elems); - index_->BuildAll(nb, xb.data(), ids.data(), train_cfg); - index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg); + index_->BuildAll(nb, xb.data(), ids.data(), conf); + index_->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf); AssertResult(res_ids, res_dis); { @@ -220,7 +247,7 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) { std::vector res_ids(elems); std::vector res_dis(elems); - new_index->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg); + new_index->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf); AssertResult(res_ids, res_dis); } @@ -234,7 +261,7 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) { std::vector res_ids(elems); std::vector res_dis(elems); - new_index->Search(nq, xq.data(), res_dis.data(), res_ids.data(), search_cfg); + new_index->Search(nq, xq.data(), res_dis.data(), res_ids.data(), conf); AssertResult(res_ids, res_dis); } } From 88ced5a75100ad5350bf0bc103abebf3d4c90a7a Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Fri, 27 Sep 2019 15:51:48 +0800 Subject: [PATCH 135/217] MS-595 Fix compiling error on gcc 7.4 Former-commit-id: 2d408646efb2d8bbc48803a8d39be8570751778e --- cpp/README.md | 13 ++++++++----- cpp/src/scheduler/TaskTable.h | 1 + 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index 57f3a85dcc..f5f77d25f8 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -50,10 +50,9 @@ Centos7 : $ yum install gfortran qt4 flex bison $ yum install mysql-devel mysql -Ubuntu16.04 : +Ubuntu 16.04 or 18.04: $ sudo apt-get install gfortran qt4-qmake flex bison $ sudo apt-get install libmysqlclient-dev mysql-client - ``` Verify the existence of `libmysqlclient_r.so`: @@ -66,6 +65,10 @@ $ locate libmysqlclient_r.so If not, you need to create a symbolic link: ```shell +# Locate libmysqlclient.so +$ sudo updatedb +$ locate libmysqlclient.so + # Create symbolic link $ sudo ln -s /path/to/libmysqlclient.so /path/to/libmysqlclient_r.so ``` @@ -90,7 +93,7 @@ please reinstall CMake with curl: ```shell CentOS 7: $ yum install curl-devel - Ubuntu 16.04: + Ubuntu 16.04 or 18.04: $ sudo apt-get install libcurl4-openssl-dev ``` @@ -106,7 +109,7 @@ please reinstall CMake with curl: ```shell CentOS 7: $ yum install clang -Ubuntu 16.04: +Ubuntu 16.04 or 18.04: $ sudo apt-get install clang-format clang-tidy $ ./build.sh -l @@ -123,7 +126,7 @@ $ ./build.sh -u ```shell CentOS 7: $ yum install lcov -Ubuntu 16.04: +Ubuntu 16.04 or 18.04: $ sudo apt-get install lcov $ ./build.sh -u -c diff --git a/cpp/src/scheduler/TaskTable.h b/cpp/src/scheduler/TaskTable.h index 819197c1ad..b36d2cd695 100644 --- a/cpp/src/scheduler/TaskTable.h +++ b/cpp/src/scheduler/TaskTable.h @@ -23,6 +23,7 @@ #include #include #include +#include #include "task/SearchTask.h" #include "event/Event.h" From f64380aed24ea5ee137de502d117addfb109eaf1 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Fri, 27 Sep 2019 16:08:14 +0800 Subject: [PATCH 136/217] MS-574 update config comments Former-commit-id: d9b088a15e16c43be8eae1657184d30ade8e3053 --- cpp/conf/server_config.template | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index d80dfda958..b49b5855f5 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -1,4 +1,4 @@ -# All the following configurations are default values. +# Default values are used when you make no changes to the following parameters. server_config: address: 0.0.0.0 # milvus server ip address (IPv4) @@ -25,8 +25,8 @@ metric_config: cache_config: cpu_mem_capacity: 16 # GB, CPU memory used for cache - cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered - cache_insert_data: false # whether load inserted data into cache + cpu_mem_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered + cache_insert_data: false # whether to load inserted data into cache engine_config: blas_threshold: 20 From 8a6e83643f207c122d2487754885daa2b7153979 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Fri, 27 Sep 2019 17:05:53 +0800 Subject: [PATCH 137/217] MS-574 rename some configurations Former-commit-id: a7e700ae417c28527f665ac919666d3a22a73707 --- cpp/conf/server_config.template | 12 ++- cpp/src/cache/CpuCacheMgr.cpp | 16 +-- cpp/src/cache/GpuCacheMgr.cpp | 8 +- cpp/src/server/Config.cpp | 182 ++++++++++++++++---------------- cpp/src/server/Config.h | 60 +++++------ cpp/src/server/DBWrapper.cpp | 6 +- 6 files changed, 143 insertions(+), 141 deletions(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index b49b5855f5..3d63cdfafa 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -11,25 +11,27 @@ db_config: secondary_path: # path used to store data only, split by semicolon backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database - # Keep 'dialect://:@:/', and replace other texts with real values. + # Keep 'dialect://:@:/', and replace other texts with real values # Replace 'dialect' with 'mysql' or 'sqlite' insert_buffer_size: 4 # GB, maximum insert buffer size allowed + # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory build_index_gpu: 0 # gpu id used for building index metric_config: enable_monitor: false # enable monitoring or not collector: prometheus # prometheus prometheus_config: - port: 8080 # port prometheus used to fetch metrics + port: 8080 # port prometheus uses to fetch metrics cache_config: - cpu_mem_capacity: 16 # GB, CPU memory used for cache - cpu_mem_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered + cpu_cache_capacity: 16 # GB, CPU memory used for cache + cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered cache_insert_data: false # whether to load inserted data into cache engine_config: - blas_threshold: 20 + use_blas_threshold: 20 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times + # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times resource_config: resource_pool: diff --git a/cpp/src/cache/CpuCacheMgr.cpp b/cpp/src/cache/CpuCacheMgr.cpp index d26004b2be..7f53493017 100644 --- a/cpp/src/cache/CpuCacheMgr.cpp +++ b/cpp/src/cache/CpuCacheMgr.cpp @@ -34,23 +34,23 @@ CpuCacheMgr::CpuCacheMgr() { server::Config &config = server::Config::GetInstance(); Status s; - int32_t cpu_mem_cap; - s = config.GetCacheConfigCpuMemCapacity(cpu_mem_cap); + int32_t cpu_cache_cap; + s = config.GetCacheConfigCpuCacheCapacity(cpu_cache_cap); if (!s.ok()) { SERVER_LOG_ERROR << s.message(); } - int64_t cap = cpu_mem_cap * unit; + int64_t cap = cpu_cache_cap * unit; cache_ = std::make_shared>(cap, 1UL << 32); - float cpu_mem_threshold; - s = config.GetCacheConfigCpuMemThreshold(cpu_mem_threshold); + float cpu_cache_threshold; + s = config.GetCacheConfigCpuCacheThreshold(cpu_cache_threshold); if (!s.ok()) { SERVER_LOG_ERROR << s.message(); } - if (cpu_mem_threshold > 0.0 && cpu_mem_threshold <= 1.0) { - cache_->set_freemem_percent(cpu_mem_threshold); + if (cpu_cache_threshold > 0.0 && cpu_cache_threshold <= 1.0) { + cache_->set_freemem_percent(cpu_cache_threshold); } else { - SERVER_LOG_ERROR << "Invalid cpu_mem_threshold: " << cpu_mem_threshold + SERVER_LOG_ERROR << "Invalid cpu_cache_threshold: " << cpu_cache_threshold << ", by default set to " << cache_->freemem_percent(); } } diff --git a/cpp/src/cache/GpuCacheMgr.cpp b/cpp/src/cache/GpuCacheMgr.cpp index ad68a6ebef..f19e70de6b 100644 --- a/cpp/src/cache/GpuCacheMgr.cpp +++ b/cpp/src/cache/GpuCacheMgr.cpp @@ -38,16 +38,16 @@ GpuCacheMgr::GpuCacheMgr() { server::Config &config = server::Config::GetInstance(); Status s; - int32_t gpu_mem_cap; - s = config.GetCacheConfigGpuMemCapacity(gpu_mem_cap); + int32_t gpu_cache_cap; + s = config.GetCacheConfigGpuCacheCapacity(gpu_cache_cap); if (!s.ok()) { SERVER_LOG_ERROR << s.message(); } - int32_t cap = gpu_mem_cap * G_BYTE; + int32_t cap = gpu_cache_cap * G_BYTE; cache_ = std::make_shared>(cap, 1UL << 32); float gpu_mem_threshold; - s = config.GetCacheConfigGpuMemThreshold(gpu_mem_threshold); + s = config.GetCacheConfigGpuCacheThreshold(gpu_mem_threshold); if (!s.ok()) { SERVER_LOG_ERROR << s.message(); } diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index d0b2b84539..0ad122fcad 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -135,20 +135,20 @@ Config::ValidateConfig() { if (!s.ok()) return s; /* cache config */ - int32_t cache_cpu_mem_capacity; - s = GetCacheConfigCpuMemCapacity(cache_cpu_mem_capacity); + int32_t cache_cpu_cache_capacity; + s = GetCacheConfigCpuCacheCapacity(cache_cpu_cache_capacity); if (!s.ok()) return s; - float cache_cpu_mem_threshold; - s = GetCacheConfigCpuMemThreshold(cache_cpu_mem_threshold); + float cache_cpu_cache_threshold; + s = GetCacheConfigCpuCacheThreshold(cache_cpu_cache_threshold); if (!s.ok()) return s; - int32_t cache_gpu_mem_capacity; - s = GetCacheConfigGpuMemCapacity(cache_gpu_mem_capacity); + int32_t cache_gpu_cache_capacity; + s = GetCacheConfigGpuCacheCapacity(cache_gpu_cache_capacity); if (!s.ok()) return s; - float cache_gpu_mem_threshold; - s = GetCacheConfigGpuMemThreshold(cache_gpu_mem_threshold); + float cache_gpu_cache_threshold; + s = GetCacheConfigGpuCacheThreshold(cache_gpu_cache_threshold); if (!s.ok()) return s; bool cache_insert_data; @@ -156,8 +156,8 @@ Config::ValidateConfig() { if (!s.ok()) return s; /* engine config */ - int32_t engine_blas_threshold; - s = GetEngineConfigBlasThreshold(engine_blas_threshold); + int32_t engine_use_blas_threshold; + s = GetEngineConfigUseBlasThreshold(engine_use_blas_threshold); if (!s.ok()) return s; int32_t engine_omp_thread_num; @@ -226,23 +226,23 @@ Config::ResetDefaultConfig() { if (!s.ok()) return s; /* cache config */ - s = SetCacheConfigCpuMemCapacity(CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); + s = SetCacheConfigCpuCacheCapacity(CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT); if (!s.ok()) return s; - s = SetCacheConfigCpuMemThreshold(CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT); + s = SetCacheConfigCpuCacheThreshold(CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT); if (!s.ok()) return s; - s = SetCacheConfigGpuMemCapacity(CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT); + s = SetCacheConfigGpuCacheCapacity(CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); if (!s.ok()) return s; - s = SetCacheConfigGpuMemThreshold(CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT); + s = SetCacheConfigGpuCacheThreshold(CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); if (!s.ok()) return s; s = SetCacheConfigCacheInsertData(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); if (!s.ok()) return s; /* engine config */ - s = SetEngineConfigBlasThreshold(CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); + s = SetEngineConfigUseBlasThreshold(CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); if (!s.ok()) return s; s = SetEngineConfigOmpThreadNum(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); @@ -417,17 +417,17 @@ Config::CheckMetricConfigPrometheusPort(const std::string &value) { } Status -Config::CheckCacheConfigCpuMemCapacity(const std::string &value) { +Config::CheckCacheConfigCpuCacheCapacity(const std::string &value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_mem_capacity: " + value); + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_cache_capacity: " + value); } else { uint64_t cpu_cache_capacity = std::stoi(value) * GB; uint64_t total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (cpu_cache_capacity >= total_mem) { - return Status(SERVER_INVALID_ARGUMENT, "Cache config cpu_mem_capacity exceed system memory: " + value); + return Status(SERVER_INVALID_ARGUMENT, "Cache config cpu_cache_capacity exceed system memory: " + value); } else if (cpu_cache_capacity > (double) total_mem * 0.9) { - std::cerr << "Warning: cpu_mem_capacity value is too big" << std::endl; + std::cerr << "Warning: cpu_cache_capacity value is too big" << std::endl; } int32_t buffer_value; @@ -435,29 +435,29 @@ Config::CheckCacheConfigCpuMemCapacity(const std::string &value) { if (!s.ok()) return s; int64_t insert_buffer_size = buffer_value * GB; if (insert_buffer_size + cpu_cache_capacity >= total_mem) { - return Status(SERVER_INVALID_ARGUMENT, "Sum of cpu_mem_capacity and buffer_size exceed system memory"); + return Status(SERVER_INVALID_ARGUMENT, "Sum of cpu_cache_capacity and buffer_size exceed system memory"); } } return Status::OK(); } Status -Config::CheckCacheConfigCpuMemThreshold(const std::string &value) { +Config::CheckCacheConfigCpuCacheThreshold(const std::string &value) { if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_mem_threshold: " + value); + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_cache_threshold: " + value); } else { - float cpu_mem_threshold = std::stof(value); - if (cpu_mem_threshold <= 0.0 || cpu_mem_threshold >= 1.0) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_mem_threshold: " + value); + float cpu_cache_threshold = std::stof(value); + if (cpu_cache_threshold <= 0.0 || cpu_cache_threshold >= 1.0) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_cache_threshold: " + value); } } return Status::OK(); } Status -Config::CheckCacheConfigGpuMemCapacity(const std::string &value) { +Config::CheckCacheConfigGpuCacheCapacity(const std::string &value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::cerr << "ERROR: gpu_cache_capacity " << value << " is not a number" << std::endl; + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_cache_capacity: " + value); } else { uint64_t gpu_cache_capacity = std::stoi(value) * GB; int gpu_index; @@ -469,22 +469,22 @@ Config::CheckCacheConfigGpuMemCapacity(const std::string &value) { "Fail to get GPU memory for GPU device: " + std::to_string(gpu_index)); } else if (gpu_cache_capacity >= gpu_memory) { return Status(SERVER_INVALID_ARGUMENT, - "Cache config gpu_mem_capacity exceed GPU memory: " + std::to_string(gpu_memory)); + "Cache config gpu_cache_capacity exceed GPU memory: " + std::to_string(gpu_memory)); } else if (gpu_cache_capacity > (double) gpu_memory * 0.9) { - std::cerr << "Warning: gpu_mem_capacity value is too big" << std::endl; + std::cerr << "Warning: gpu_cache_capacity value is too big" << std::endl; } } return Status::OK(); } Status -Config::CheckCacheConfigGpuMemThreshold(const std::string &value) { +Config::CheckCacheConfigGpuCacheThreshold(const std::string &value) { if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_mem_threshold: " + value); + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_cache_threshold: " + value); } else { - float gpu_mem_threshold = std::stof(value); - if (gpu_mem_threshold <= 0.0 || gpu_mem_threshold >= 1.0) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_mem_threshold: " + value); + float gpu_cache_threshold = std::stof(value); + if (gpu_cache_threshold <= 0.0 || gpu_cache_threshold >= 1.0) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_cache_threshold: " + value); } } return Status::OK(); @@ -499,9 +499,9 @@ Config::CheckCacheConfigCacheInsertData(const std::string &value) { } Status -Config::CheckEngineConfigBlasThreshold(const std::string &value) { +Config::CheckEngineConfigUseBlasThreshold(const std::string &value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config blas threshold: " + value); + return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config use_blas_threshold: " + value); } return Status::OK(); } @@ -729,45 +729,45 @@ Config::GetMetricConfigStrPrometheusPort() { //////////////////////////////////////////////////////////////////////////////// /* cache config */ std::string -Config::GetCacheConfigStrCpuMemCapacity() { +Config::GetCacheConfigStrCpuCacheCapacity() { std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value).ok()) { - value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_CAPACITY, - CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value); + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value).ok()) { + value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_CACHE_CAPACITY, + CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value); } return value; } std::string -Config::GetCacheConfigStrCpuMemThreshold() { +Config::GetCacheConfigStrCpuCacheThreshold() { std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value).ok()) { - value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, - CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value); + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value).ok()) { + value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_CACHE_THRESHOLD, + CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); } return value; } std::string -Config::GetCacheConfigStrGpuMemCapacity() { +Config::GetCacheConfigStrGpuCacheCapacity() { std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value).ok()) { - value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_CAPACITY, - CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value); + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value).ok()) { + value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_CACHE_CAPACITY, + CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value); } return value; } std::string -Config::GetCacheConfigStrGpuMemThreshold() { +Config::GetCacheConfigStrGpuCacheThreshold() { std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value).ok()) { - value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, - CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value); + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value).ok()) { + value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_CACHE_THRESHOLD, + CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value); } return value; } @@ -786,12 +786,12 @@ Config::GetCacheConfigStrCacheInsertData() { //////////////////////////////////////////////////////////////////////////////// /* engine config */ std::string -Config::GetEngineConfigStrBlasThreshold() { +Config::GetEngineConfigStrUseBlasThreshold() { std::string value; - if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value).ok()) { - value = GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_BLAS_THRESHOLD, - CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value); + if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value).ok()) { + value = GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_USE_BLAS_THRESHOLD, + CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); + SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value); } return value; } @@ -922,36 +922,36 @@ Config::GetMetricConfigPrometheusPort(std::string &value) { } Status -Config::GetCacheConfigCpuMemCapacity(int32_t &value) { - std::string str = GetCacheConfigStrCpuMemCapacity(); - Status s = CheckCacheConfigCpuMemCapacity(str); +Config::GetCacheConfigCpuCacheCapacity(int32_t &value) { + std::string str = GetCacheConfigStrCpuCacheCapacity(); + Status s = CheckCacheConfigCpuCacheCapacity(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); } Status -Config::GetCacheConfigCpuMemThreshold(float &value) { - std::string str = GetCacheConfigStrCpuMemThreshold(); - Status s = CheckCacheConfigCpuMemThreshold(str); +Config::GetCacheConfigCpuCacheThreshold(float &value) { + std::string str = GetCacheConfigStrCpuCacheThreshold(); + Status s = CheckCacheConfigCpuCacheThreshold(str); if (!s.ok()) return s; value = std::stof(str); return Status::OK(); } Status -Config::GetCacheConfigGpuMemCapacity(int32_t &value) { - std::string str = GetCacheConfigStrGpuMemCapacity(); - Status s = CheckCacheConfigGpuMemCapacity(str); +Config::GetCacheConfigGpuCacheCapacity(int32_t &value) { + std::string str = GetCacheConfigStrGpuCacheCapacity(); + Status s = CheckCacheConfigGpuCacheCapacity(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); } Status -Config::GetCacheConfigGpuMemThreshold(float &value) { - std::string str = GetCacheConfigStrGpuMemThreshold(); - Status s = CheckCacheConfigGpuMemThreshold(str); +Config::GetCacheConfigGpuCacheThreshold(float &value) { + std::string str = GetCacheConfigStrGpuCacheThreshold(); + Status s = CheckCacheConfigGpuCacheThreshold(str); if (!s.ok()) return s; value = std::stof(str); return Status::OK(); @@ -968,9 +968,9 @@ Config::GetCacheConfigCacheInsertData(bool &value) { } Status -Config::GetEngineConfigBlasThreshold(int32_t &value) { - std::string str = GetEngineConfigStrBlasThreshold(); - Status s = CheckEngineConfigBlasThreshold(str); +Config::GetEngineConfigUseBlasThreshold(int32_t &value) { + std::string str = GetEngineConfigStrUseBlasThreshold(); + Status s = CheckEngineConfigUseBlasThreshold(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -1116,34 +1116,34 @@ Config::SetMetricConfigPrometheusPort(const std::string &value) { /* cache config */ Status -Config::SetCacheConfigCpuMemCapacity(const std::string &value) { - Status s = CheckCacheConfigCpuMemCapacity(value); +Config::SetCacheConfigCpuCacheCapacity(const std::string &value) { + Status s = CheckCacheConfigCpuCacheCapacity(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_MEM_CAPACITY, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_CAPACITY, value); return Status::OK(); } Status -Config::SetCacheConfigCpuMemThreshold(const std::string &value) { - Status s = CheckCacheConfigCpuMemThreshold(value); +Config::SetCacheConfigCpuCacheThreshold(const std::string &value) { + Status s = CheckCacheConfigCpuCacheThreshold(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_MEM_THRESHOLD, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); return Status::OK(); } Status -Config::SetCacheConfigGpuMemCapacity(const std::string &value) { - Status s = CheckCacheConfigGpuMemCapacity(value); +Config::SetCacheConfigGpuCacheCapacity(const std::string &value) { + Status s = CheckCacheConfigGpuCacheCapacity(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_MEM_CAPACITY, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_CAPACITY, value); return Status::OK(); } Status -Config::SetCacheConfigGpuMemThreshold(const std::string &value) { - Status s = CheckCacheConfigGpuMemThreshold(value); +Config::SetCacheConfigGpuCacheThreshold(const std::string &value) { + Status s = CheckCacheConfigGpuCacheThreshold(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_MEM_THRESHOLD, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value); return Status::OK(); } @@ -1157,10 +1157,10 @@ Config::SetCacheConfigCacheInsertData(const std::string &value) { /* engine config */ Status -Config::SetEngineConfigBlasThreshold(const std::string &value) { - Status s = CheckEngineConfigBlasThreshold(value); +Config::SetEngineConfigUseBlasThreshold(const std::string &value) { + Status s = CheckEngineConfigUseBlasThreshold(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_BLAS_THRESHOLD, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value); return Status::OK(); } diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index af23dd67b6..c2484a850a 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -60,14 +60,14 @@ static const char *CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0"; /* cache config */ static const char *CONFIG_CACHE = "cache_config"; -static const char *CONFIG_CACHE_CPU_MEM_CAPACITY = "cpu_mem_capacity"; -static const char *CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT = "16"; -static const char *CONFIG_CACHE_GPU_MEM_CAPACITY = "gpu_mem_capacity"; -static const char *CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT = "0"; -static const char *CONFIG_CACHE_CPU_MEM_THRESHOLD = "cpu_mem_threshold"; -static const char *CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT = "0.85"; -static const char *CONFIG_CACHE_GPU_MEM_THRESHOLD = "gpu_mem_threshold"; -static const char *CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT = "0.85"; +static const char *CONFIG_CACHE_CPU_CACHE_CAPACITY = "cpu_cache_capacity"; +static const char *CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT = "16"; +static const char *CONFIG_CACHE_GPU_CACHE_CAPACITY = "gpu_cache_capacity"; +static const char *CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT = "0"; +static const char *CONFIG_CACHE_CPU_CACHE_THRESHOLD = "cpu_mem_threshold"; +static const char *CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT = "0.85"; +static const char *CONFIG_CACHE_GPU_CACHE_THRESHOLD = "gpu_mem_threshold"; +static const char *CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT = "0.85"; static const char *CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; static const char *CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; @@ -83,8 +83,8 @@ static const char *CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT = "8080"; /* engine config */ static const char *CONFIG_ENGINE = "engine_config"; -static const char *CONFIG_ENGINE_BLAS_THRESHOLD = "blas_threshold"; -static const char *CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT = "20"; +static const char *CONFIG_ENGINE_USE_BLAS_THRESHOLD = "use_blas_threshold"; +static const char *CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT = "20"; static const char *CONFIG_ENGINE_OMP_THREAD_NUM = "omp_thread_num"; static const char *CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; @@ -137,14 +137,14 @@ class Config { Status CheckMetricConfigPrometheusPort(const std::string &value); /* cache config */ - Status CheckCacheConfigCpuMemCapacity(const std::string &value); - Status CheckCacheConfigCpuMemThreshold(const std::string &value); - Status CheckCacheConfigGpuMemCapacity(const std::string &value); - Status CheckCacheConfigGpuMemThreshold(const std::string &value); + Status CheckCacheConfigCpuCacheCapacity(const std::string &value); + Status CheckCacheConfigCpuCacheThreshold(const std::string &value); + Status CheckCacheConfigGpuCacheCapacity(const std::string &value); + Status CheckCacheConfigGpuCacheThreshold(const std::string &value); Status CheckCacheConfigCacheInsertData(const std::string &value); /* engine config */ - Status CheckEngineConfigBlasThreshold(const std::string &value); + Status CheckEngineConfigUseBlasThreshold(const std::string &value); Status CheckEngineConfigOmpThreadNum(const std::string &value); /* resource config */ @@ -173,14 +173,14 @@ class Config { std::string GetMetricConfigStrPrometheusPort(); /* cache config */ - std::string GetCacheConfigStrCpuMemCapacity(); - std::string GetCacheConfigStrCpuMemThreshold(); - std::string GetCacheConfigStrGpuMemCapacity(); - std::string GetCacheConfigStrGpuMemThreshold(); + std::string GetCacheConfigStrCpuCacheCapacity(); + std::string GetCacheConfigStrCpuCacheThreshold(); + std::string GetCacheConfigStrGpuCacheCapacity(); + std::string GetCacheConfigStrGpuCacheThreshold(); std::string GetCacheConfigStrCacheInsertData(); /* engine config */ - std::string GetEngineConfigStrBlasThreshold(); + std::string GetEngineConfigStrUseBlasThreshold(); std::string GetEngineConfigStrOmpThreadNum(); /* resource config */ @@ -208,14 +208,14 @@ class Config { Status GetMetricConfigPrometheusPort(std::string &value); /* cache config */ - Status GetCacheConfigCpuMemCapacity(int32_t &value); - Status GetCacheConfigCpuMemThreshold(float &value); - Status GetCacheConfigGpuMemCapacity(int32_t &value); - Status GetCacheConfigGpuMemThreshold(float &value); + Status GetCacheConfigCpuCacheCapacity(int32_t &value); + Status GetCacheConfigCpuCacheThreshold(float &value); + Status GetCacheConfigGpuCacheCapacity(int32_t &value); + Status GetCacheConfigGpuCacheThreshold(float &value); Status GetCacheConfigCacheInsertData(bool &value); /* engine config */ - Status GetEngineConfigBlasThreshold(int32_t &value); + Status GetEngineConfigUseBlasThreshold(int32_t &value); Status GetEngineConfigOmpThreadNum(int32_t &value); /* resource config */ @@ -244,14 +244,14 @@ class Config { Status SetMetricConfigPrometheusPort(const std::string &value); /* cache config */ - Status SetCacheConfigCpuMemCapacity(const std::string &value); - Status SetCacheConfigCpuMemThreshold(const std::string &value); - Status SetCacheConfigGpuMemCapacity(const std::string &value); - Status SetCacheConfigGpuMemThreshold(const std::string &value); + Status SetCacheConfigCpuCacheCapacity(const std::string &value); + Status SetCacheConfigCpuCacheThreshold(const std::string &value); + Status SetCacheConfigGpuCacheCapacity(const std::string &value); + Status SetCacheConfigGpuCacheThreshold(const std::string &value); Status SetCacheConfigCacheInsertData(const std::string &value); /* engine config */ - Status SetEngineConfigBlasThreshold(const std::string &value); + Status SetEngineConfigUseBlasThreshold(const std::string &value); Status SetEngineConfigOmpThreadNum(const std::string &value); /* resource config */ diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 66104bfe5a..d764570966 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -92,10 +92,10 @@ Status DBWrapper::StartService() { } //init faiss global variable - int32_t blas_threshold; - s = config.GetEngineConfigBlasThreshold(blas_threshold); + int32_t use_blas_threshold; + s = config.GetEngineConfigUseBlasThreshold(use_blas_threshold); if (!s.ok()) return s; - faiss::distance_compute_blas_threshold = blas_threshold; + faiss::distance_compute_blas_threshold = use_blas_threshold; //set archive config engine::ArchiveConf::CriteriaT criterial; From c93202258fc01ac847cea2ac3cfcbff764fac517 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 27 Sep 2019 18:57:46 +0800 Subject: [PATCH 138/217] format unittest code Former-commit-id: f5979db268dc71e79ef556617fb8992dda04a422 --- cpp/CMakeLists.txt | 2 +- cpp/build-support/lint_exclusions.txt | 6 +- cpp/build.sh | 31 +- cpp/src/db/meta/MetaFactory.cpp | 5 +- cpp/unittest/CMakeLists.txt | 2 + cpp/unittest/db/db_tests.cpp | 331 +++++++++--------- cpp/unittest/db/engine_test.cpp | 71 ++-- cpp/unittest/db/mem_test.cpp | 152 ++++---- cpp/unittest/db/meta_tests.cpp | 174 ++++----- cpp/unittest/db/misc_test.cpp | 52 +-- cpp/unittest/db/mysql_db_test.cpp | 130 +++---- cpp/unittest/db/mysql_meta_test.cpp | 180 +++++----- cpp/unittest/db/search_test.cpp | 182 +++++----- cpp/unittest/db/utils.cpp | 132 ++++--- cpp/unittest/db/utils.h | 33 +- cpp/unittest/main.cpp | 6 +- cpp/unittest/metrics/metricbase_test.cpp | 16 +- cpp/unittest/metrics/metrics_test.cpp | 104 +++--- cpp/unittest/metrics/prometheus_test.cpp | 21 +- cpp/unittest/metrics/utils.cpp | 34 +- cpp/unittest/metrics/utils.h | 15 +- cpp/unittest/scheduler/algorithm_test.cpp | 9 +- cpp/unittest/scheduler/event_test.cpp | 6 +- cpp/unittest/scheduler/node_test.cpp | 34 +- cpp/unittest/scheduler/normal_test.cpp | 23 +- .../scheduler/resource_factory_test.cpp | 17 +- cpp/unittest/scheduler/resource_mgr_test.cpp | 15 +- cpp/unittest/scheduler/resource_test.cpp | 27 +- cpp/unittest/scheduler/schedinst_test.cpp | 12 +- cpp/unittest/scheduler/scheduler_test.cpp | 55 ++- cpp/unittest/scheduler/task_test.cpp | 9 +- cpp/unittest/scheduler/tasktable_test.cpp | 181 +++++----- cpp/unittest/server/cache_test.cpp | 116 +++--- cpp/unittest/server/config_test.cpp | 50 +-- cpp/unittest/server/rpc_test.cpp | 108 +++--- cpp/unittest/server/util_test.cpp | 240 +++++++------ cpp/unittest/wrapper/utils.cpp | 27 +- cpp/unittest/wrapper/utils.h | 2 - cpp/unittest/wrapper/wrapper_test.cpp | 2 +- cpp/version.h.macro | 17 + 40 files changed, 1353 insertions(+), 1276 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index cc807d43ed..d2092eb018 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -189,7 +189,7 @@ add_custom_target(lint --exclude_globs ${LINT_EXCLUSIONS_FILE} --source_dir - ${CMAKE_CURRENT_SOURCE_DIR}/src + ${CMAKE_CURRENT_SOURCE_DIR} ${MILVUS_LINT_QUIET}) # diff --git a/cpp/build-support/lint_exclusions.txt b/cpp/build-support/lint_exclusions.txt index 27bd780f42..593ab6b1cf 100644 --- a/cpp/build-support/lint_exclusions.txt +++ b/cpp/build-support/lint_exclusions.txt @@ -3,6 +3,10 @@ *cmake_build* *src/thirdparty* *src/core/thirdparty* +*thirdparty* *src/grpc* *easylogging++* -*SqliteMetaImpl.cpp \ No newline at end of file +*SqliteMetaImpl.cpp +*src/core* +*src/wrapper* +*unittest/wrapper* \ No newline at end of file diff --git a/cpp/build.sh b/cpp/build.sh index 4087db4592..974110c08f 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -99,21 +99,26 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then # cpplint check make lint if [ $? -ne 0 ]; then - echo "ERROR! cpplint check not pass" - exit 1 - fi - # clang-format check - make check-clang-format - if [ $? -ne 0 ]; then - echo "ERROR! clang-format check failed" - exit 1 - fi - # clang-tidy check - make check-clang-tidy - if [ $? -ne 0 ]; then - echo "ERROR! clang-tidy check failed" + echo "ERROR! cpplint check failed" exit 1 fi + echo "cpplint check pass!" + +# # clang-format check +# make check-clang-format +# if [ $? -ne 0 ]; then +# echo "ERROR! clang-format check failed" +# exit 1 +# fi +# echo "clang-format check pass!" +# +# # clang-tidy check +# make check-clang-tidy +# if [ $? -ne 0 ]; then +# echo "ERROR! clang-tidy check failed" +# exit 1 +# fi +# echo "clang-tidy check pass!" else # compile and build make -j 4 || exit 1 diff --git a/cpp/src/db/meta/MetaFactory.cpp b/cpp/src/db/meta/MetaFactory.cpp index dd0b2ae7ea..6a305a4280 100644 --- a/cpp/src/db/meta/MetaFactory.cpp +++ b/cpp/src/db/meta/MetaFactory.cpp @@ -40,9 +40,8 @@ MetaFactory::BuildOption(const std::string &path) { if (p == "") { srand(time(nullptr)); std::stringstream ss; - uint32_t rd = 0; - rand_r(&rd); - ss << "/tmp/" << rd; + uint32_t seed = 1; + ss << "/tmp/" << rand_r(&seed); p = ss.str(); } diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 0df8bfd791..0f454ccc43 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -118,6 +118,8 @@ set(unittest_libs cublas ) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) +message(STATUS "CMAKE_CURRENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}") foreach(dir ${CORE_INCLUDE_DIRS}) include_directories(${dir}) endforeach() diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index f2f8121538..365fc3ab2d 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "utils.h" +#include "db/utils.h" #include "db/DB.h" #include "db/DBImpl.h" #include "db/meta/MetaConsts.h" @@ -28,139 +28,142 @@ #include #include -using namespace zilliz::milvus; - namespace { - static const char* TABLE_NAME = "test_group"; - static constexpr int64_t TABLE_DIM = 256; - static constexpr int64_t VECTOR_COUNT = 25000; - static constexpr int64_t INSERT_LOOP = 1000; - static constexpr int64_t SECONDS_EACH_HOUR = 3600; - static constexpr int64_t DAY_SECONDS = 24 * 60 * 60; +namespace ms = zilliz::milvus; - engine::meta::TableSchema BuildTableSchema() { - engine::meta::TableSchema table_info; - table_info.dimension_ = TABLE_DIM; - table_info.table_id_ = TABLE_NAME; - return table_info; - } - - void BuildVectors(int64_t n, std::vector& vectors) { - vectors.clear(); - vectors.resize(n*TABLE_DIM); - float* data = vectors.data(); - for(int i = 0; i < n; i++) { - for(int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); - data[TABLE_DIM * i] += i / 2000.; - } - } - - std::string CurrentTmDate(int64_t offset_day = 0) { - time_t tt; - time( &tt ); - tt = tt + 8*SECONDS_EACH_HOUR; - tt = tt + 24*SECONDS_EACH_HOUR*offset_day; - tm* t= gmtime( &tt ); - - std::string str = std::to_string(t->tm_year + 1900) + "-" + std::to_string(t->tm_mon + 1) - + "-" + std::to_string(t->tm_mday); - - return str; - } - - void - ConvertTimeRangeToDBDates(const std::string &start_value, - const std::string &end_value, - std::vector &dates) { - dates.clear(); - - time_t tt_start, tt_end; - tm tm_start, tm_end; - if (!zilliz::milvus::server::CommonUtil::TimeStrToTime(start_value, tt_start, tm_start)) { - return; - } - - if (!zilliz::milvus::server::CommonUtil::TimeStrToTime(end_value, tt_end, tm_end)) { - return; - } - - long days = (tt_end > tt_start) ? (tt_end - tt_start) / DAY_SECONDS : (tt_start - tt_end) / - DAY_SECONDS; - if (days == 0) { - return; - } - - for (long i = 0; i < days; i++) { - time_t tt_day = tt_start + DAY_SECONDS * i; - tm tm_day; - zilliz::milvus::server::CommonUtil::ConvertTime(tt_day, tm_day); - - long date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 + - tm_day.tm_mday;//according to db logic - dates.push_back(date); - } - } +static const char *TABLE_NAME = "test_group"; +static constexpr int64_t TABLE_DIM = 256; +static constexpr int64_t VECTOR_COUNT = 25000; +static constexpr int64_t INSERT_LOOP = 1000; +static constexpr int64_t SECONDS_EACH_HOUR = 3600; +static constexpr int64_t DAY_SECONDS = 24 * 60 * 60; +ms::engine::meta::TableSchema +BuildTableSchema() { + ms::engine::meta::TableSchema table_info; + table_info.dimension_ = TABLE_DIM; + table_info.table_id_ = TABLE_NAME; + return table_info; } +void +BuildVectors(int64_t n, std::vector &vectors) { + vectors.clear(); + vectors.resize(n * TABLE_DIM); + float *data = vectors.data(); + for (int i = 0; i < n; i++) { + for (int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); + data[TABLE_DIM * i] += i / 2000.; + } +} + +std::string +CurrentTmDate(int64_t offset_day = 0) { + time_t tt; + time(&tt); + tt = tt + 8 * SECONDS_EACH_HOUR; + tt = tt + 24 * SECONDS_EACH_HOUR * offset_day; + tm t; + gmtime_r(&tt, &t); + + std::string str = std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1) + + "-" + std::to_string(t.tm_mday); + + return str; +} + +void +ConvertTimeRangeToDBDates(const std::string &start_value, + const std::string &end_value, + std::vector &dates) { + dates.clear(); + + time_t tt_start, tt_end; + tm tm_start, tm_end; + if (!zilliz::milvus::server::CommonUtil::TimeStrToTime(start_value, tt_start, tm_start)) { + return; + } + + if (!zilliz::milvus::server::CommonUtil::TimeStrToTime(end_value, tt_end, tm_end)) { + return; + } + + int64_t days = (tt_end > tt_start) ? (tt_end - tt_start) / DAY_SECONDS : (tt_start - tt_end) / + DAY_SECONDS; + if (days == 0) { + return; + } + + for (int64_t i = 0; i < days; i++) { + time_t tt_day = tt_start + DAY_SECONDS * i; + tm tm_day; + zilliz::milvus::server::CommonUtil::ConvertTime(tt_day, tm_day); + + int64_t date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 + + tm_day.tm_mday;//according to db logic + dates.push_back(date); + } +} + +} // namespace + TEST_F(DBTest, CONFIG_TEST) { { - ASSERT_ANY_THROW(engine::ArchiveConf conf("wrong")); + ASSERT_ANY_THROW(ms::engine::ArchiveConf conf("wrong")); /* EXPECT_DEATH(engine::ArchiveConf conf("wrong"), ""); */ } { - engine::ArchiveConf conf("delete"); + ms::engine::ArchiveConf conf("delete"); ASSERT_EQ(conf.GetType(), "delete"); auto criterias = conf.GetCriterias(); - ASSERT_TRUE(criterias.size() == 0); + ASSERT_EQ(criterias.size(), 0); } { - engine::ArchiveConf conf("swap"); + ms::engine::ArchiveConf conf("swap"); ASSERT_EQ(conf.GetType(), "swap"); auto criterias = conf.GetCriterias(); - ASSERT_TRUE(criterias.size() == 0); + ASSERT_EQ(criterias.size(), 0); } { - ASSERT_ANY_THROW(engine::ArchiveConf conf1("swap", "disk:")); - ASSERT_ANY_THROW(engine::ArchiveConf conf2("swap", "disk:a")); - engine::ArchiveConf conf("swap", "disk:1024"); + ASSERT_ANY_THROW(ms::engine::ArchiveConf conf1("swap", "disk:")); + ASSERT_ANY_THROW(ms::engine::ArchiveConf conf2("swap", "disk:a")); + ms::engine::ArchiveConf conf("swap", "disk:1024"); auto criterias = conf.GetCriterias(); - ASSERT_TRUE(criterias.size() == 1); - ASSERT_TRUE(criterias["disk"] == 1024); + ASSERT_EQ(criterias.size(), 1); + ASSERT_EQ(criterias["disk"], 1024); } { - ASSERT_ANY_THROW(engine::ArchiveConf conf1("swap", "days:")); - ASSERT_ANY_THROW(engine::ArchiveConf conf2("swap", "days:a")); - engine::ArchiveConf conf("swap", "days:100"); + ASSERT_ANY_THROW(ms::engine::ArchiveConf conf1("swap", "days:")); + ASSERT_ANY_THROW(ms::engine::ArchiveConf conf2("swap", "days:a")); + ms::engine::ArchiveConf conf("swap", "days:100"); auto criterias = conf.GetCriterias(); - ASSERT_TRUE(criterias.size() == 1); - ASSERT_TRUE(criterias["days"] == 100); + ASSERT_EQ(criterias.size(), 1); + ASSERT_EQ(criterias["days"], 100); } { - ASSERT_ANY_THROW(engine::ArchiveConf conf1("swap", "days:")); - ASSERT_ANY_THROW(engine::ArchiveConf conf2("swap", "days:a")); - engine::ArchiveConf conf("swap", "days:100;disk:200"); + ASSERT_ANY_THROW(ms::engine::ArchiveConf conf1("swap", "days:")); + ASSERT_ANY_THROW(ms::engine::ArchiveConf conf2("swap", "days:a")); + ms::engine::ArchiveConf conf("swap", "days:100;disk:200"); auto criterias = conf.GetCriterias(); - ASSERT_TRUE(criterias.size() == 2); - ASSERT_TRUE(criterias["days"] == 100); - ASSERT_TRUE(criterias["disk"] == 200); + ASSERT_EQ(criterias.size(), 2); + ASSERT_EQ(criterias["days"], 100); + ASSERT_EQ(criterias["disk"], 200); } } - TEST_F(DBTest, DB_TEST) { - engine::meta::TableSchema table_info = BuildTableSchema(); + ms::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - engine::meta::TableSchema table_info_get; + ms::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - engine::IDNumbers vector_ids; - engine::IDNumbers target_ids; + ms::engine::IDNumbers vector_ids; + ms::engine::IDNumbers target_ids; int64_t nb = 50; std::vector xb; @@ -171,7 +174,7 @@ TEST_F(DBTest, DB_TEST) { BuildVectors(qb, qxb); std::thread search([&]() { - engine::QueryResults results; + ms::engine::QueryResults results; int k = 10; std::this_thread::sleep_for(std::chrono::seconds(2)); @@ -180,18 +183,18 @@ TEST_F(DBTest, DB_TEST) { uint64_t count = 0; uint64_t prev_count = 0; - for (auto j=0; j<10; ++j) { + for (auto j = 0; j < 10; ++j) { ss.str(""); db_->Size(count); prev_count = count; START_TIMER; stat = db_->Query(TABLE_NAME, k, qb, 10, qxb.data(), results); - ss << "Search " << j << " With Size " << count/engine::meta::M << " M"; + ss << "Search " << j << " With Size " << count / ms::engine::meta::M << " M"; STOP_TIMER(ss.str()); ASSERT_TRUE(stat.ok()); - for (auto k=0; kInsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); ASSERT_EQ(target_ids.size(), qb); } else { @@ -222,14 +225,14 @@ TEST_F(DBTest, DB_TEST) { uint64_t count; stat = db_->GetTableRowCount(TABLE_NAME, count); ASSERT_TRUE(stat.ok()); - ASSERT_TRUE(count > 0); -}; + ASSERT_GT(count, 0); +} TEST_F(DBTest, SEARCH_TEST) { - engine::meta::TableSchema table_info = BuildTableSchema(); + ms::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - engine::meta::TableSchema table_info_get; + ms::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); @@ -239,63 +242,63 @@ TEST_F(DBTest, SEARCH_TEST) { size_t nb = VECTOR_COUNT; size_t nq = 10; size_t k = 5; - std::vector xb(nb*TABLE_DIM); - std::vector xq(nq*TABLE_DIM); - std::vector ids(nb); + std::vector xb(nb * TABLE_DIM); + std::vector xq(nq * TABLE_DIM); + std::vector ids(nb); std::random_device rd; std::mt19937 gen(rd()); std::uniform_real_distribution<> dis_xt(-1.0, 1.0); - for (size_t i = 0; i < nb*TABLE_DIM; i++) { + for (size_t i = 0; i < nb * TABLE_DIM; i++) { xb[i] = dis_xt(gen); - if (i < nb){ + if (i < nb) { ids[i] = i; } } - for (size_t i = 0; i < nq*TABLE_DIM; i++) { + for (size_t i = 0; i < nq * TABLE_DIM; i++) { xq[i] = dis_xt(gen); } // result data //std::vector nns_gt(k*nq); - std::vector nns(k*nq); // nns = nearst neg search + std::vector nns(k * nq); // nns = nearst neg search //std::vector dis_gt(k*nq); - std::vector dis(k*nq); + std::vector dis(k * nq); // insert data const int batch_size = 100; for (int j = 0; j < nb / batch_size; ++j) { - stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data()+batch_size*j*TABLE_DIM, ids); - if (j == 200){ sleep(1);} + stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data() + batch_size * j * TABLE_DIM, ids); + if (j == 200) { sleep(1); } ASSERT_TRUE(stat.ok()); } - engine::TableIndex index; - index.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; + ms::engine::TableIndex index; + index.engine_type_ = (int) ms::engine::EngineType::FAISS_IDMAP; db_->CreateIndex(TABLE_NAME, index); // wait until build index finish { - engine::QueryResults results; + ms::engine::QueryResults results; stat = db_->Query(TABLE_NAME, k, nq, 10, xq.data(), results); ASSERT_TRUE(stat.ok()); } {//search by specify index file - engine::meta::DatesT dates; + ms::engine::meta::DatesT dates; std::vector file_ids = {"1", "2", "3", "4", "5", "6"}; - engine::QueryResults results; + ms::engine::QueryResults results; stat = db_->Query(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, results); ASSERT_TRUE(stat.ok()); } // TODO(linxj): add groundTruth assert -}; +} TEST_F(DBTest, PRELOADTABLE_TEST) { - engine::meta::TableSchema table_info = BuildTableSchema(); + ms::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - engine::meta::TableSchema table_info_get; + ms::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); @@ -306,28 +309,27 @@ TEST_F(DBTest, PRELOADTABLE_TEST) { BuildVectors(nb, xb); int loop = 5; - for (auto i=0; iInsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); ASSERT_EQ(vector_ids.size(), nb); } - engine::TableIndex index; - index.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; + ms::engine::TableIndex index; + index.engine_type_ = (int) ms::engine::EngineType::FAISS_IDMAP; db_->CreateIndex(TABLE_NAME, index); // wait until build index finish - int64_t prev_cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); + int64_t prev_cache_usage = ms::cache::CpuCacheMgr::GetInstance()->CacheUsage(); stat = db_->PreloadTable(TABLE_NAME); ASSERT_TRUE(stat.ok()); - int64_t cur_cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); + int64_t cur_cache_usage = ms::cache::CpuCacheMgr::GetInstance()->CacheUsage(); ASSERT_TRUE(prev_cache_usage < cur_cache_usage); - } TEST_F(DBTest, SHUTDOWN_TEST) { db_->Stop(); - engine::meta::TableSchema table_info = BuildTableSchema(); + ms::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); ASSERT_FALSE(stat.ok()); @@ -338,7 +340,7 @@ TEST_F(DBTest, SHUTDOWN_TEST) { stat = db_->HasTable(table_info.table_id_, has_table); ASSERT_FALSE(stat.ok()); - engine::IDNumbers ids; + ms::engine::IDNumbers ids; stat = db_->InsertVectors(table_info.table_id_, 0, nullptr, ids); ASSERT_FALSE(stat.ok()); @@ -349,15 +351,15 @@ TEST_F(DBTest, SHUTDOWN_TEST) { stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_FALSE(stat.ok()); - engine::TableIndex index; + ms::engine::TableIndex index; stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_FALSE(stat.ok()); stat = db_->DescribeIndex(table_info.table_id_, index); ASSERT_FALSE(stat.ok()); - engine::meta::DatesT dates; - engine::QueryResults results; + ms::engine::meta::DatesT dates; + ms::engine::QueryResults results; stat = db_->Query(table_info.table_id_, 1, 1, 1, nullptr, dates, results); ASSERT_FALSE(stat.ok()); std::vector file_ids; @@ -369,24 +371,24 @@ TEST_F(DBTest, SHUTDOWN_TEST) { } TEST_F(DBTest, INDEX_TEST) { - engine::meta::TableSchema table_info = BuildTableSchema(); + ms::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); int64_t nb = VECTOR_COUNT; std::vector xb; BuildVectors(nb, xb); - engine::IDNumbers vector_ids; + ms::engine::IDNumbers vector_ids; db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); ASSERT_EQ(vector_ids.size(), nb); - engine::TableIndex index; - index.engine_type_ = (int)engine::EngineType::FAISS_IVFSQ8; - index.metric_type_ = (int)engine::MetricType::IP; + ms::engine::TableIndex index; + index.engine_type_ = (int) ms::engine::EngineType::FAISS_IVFSQ8; + index.metric_type_ = (int) ms::engine::MetricType::IP; stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); - engine::TableIndex index_out; + ms::engine::TableIndex index_out; stat = db_->DescribeIndex(table_info.table_id_, index_out); ASSERT_TRUE(stat.ok()); ASSERT_EQ(index.engine_type_, index_out.engine_type_); @@ -398,23 +400,22 @@ TEST_F(DBTest, INDEX_TEST) { } TEST_F(DBTest2, ARHIVE_DISK_CHECK) { - - engine::meta::TableSchema table_info = BuildTableSchema(); + ms::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - std::vector table_schema_array; + std::vector table_schema_array; stat = db_->AllTables(table_schema_array); ASSERT_TRUE(stat.ok()); bool bfound = false; - for(auto& schema : table_schema_array) { - if(schema.table_id_ == TABLE_NAME) { + for (auto &schema : table_schema_array) { + if (schema.table_id_ == TABLE_NAME) { bfound = true; break; } } ASSERT_TRUE(bfound); - engine::meta::TableSchema table_info_get; + ms::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); @@ -428,8 +429,8 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) { BuildVectors(nb, xb); int loop = INSERT_LOOP; - for (auto i=0; iInsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); std::this_thread::sleep_for(std::chrono::microseconds(1)); } @@ -438,14 +439,14 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) { db_->Size(size); LOG(DEBUG) << "size=" << size; - ASSERT_LE(size, 1 * engine::meta::G); -}; + ASSERT_LE(size, 1 * ms::engine::meta::G); +} TEST_F(DBTest2, DELETE_TEST) { - engine::meta::TableSchema table_info = BuildTableSchema(); + ms::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - engine::meta::TableSchema table_info_get; + ms::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); @@ -461,25 +462,25 @@ TEST_F(DBTest2, DELETE_TEST) { std::vector xb; BuildVectors(nb, xb); - engine::IDNumbers vector_ids; + ms::engine::IDNumbers vector_ids; stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); - engine::TableIndex index; + ms::engine::TableIndex index; stat = db_->CreateIndex(TABLE_NAME, index); - std::vector dates; + std::vector dates; stat = db_->DeleteTable(TABLE_NAME, dates); std::this_thread::sleep_for(std::chrono::seconds(2)); ASSERT_TRUE(stat.ok()); db_->HasTable(TABLE_NAME, has_table); ASSERT_FALSE(has_table); -}; +} TEST_F(DBTest2, DELETE_BY_RANGE_TEST) { - engine::meta::TableSchema table_info = BuildTableSchema(); + ms::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - engine::meta::TableSchema table_info_get; + ms::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); @@ -496,15 +497,15 @@ TEST_F(DBTest2, DELETE_BY_RANGE_TEST) { std::vector xb; BuildVectors(nb, xb); - engine::IDNumbers vector_ids; + ms::engine::IDNumbers vector_ids; stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); - engine::TableIndex index; + ms::engine::TableIndex index; stat = db_->CreateIndex(TABLE_NAME, index); db_->Size(size); ASSERT_NE(size, 0UL); - std::vector dates; + std::vector dates; std::string start_value = CurrentTmDate(); std::string end_value = CurrentTmDate(1); ConvertTimeRangeToDBDates(start_value, end_value, dates); @@ -515,4 +516,4 @@ TEST_F(DBTest2, DELETE_BY_RANGE_TEST) { uint64_t row_count = 0; db_->GetTableRowCount(TABLE_NAME, row_count); ASSERT_EQ(row_count, 0UL); -} \ No newline at end of file +} diff --git a/cpp/unittest/db/engine_test.cpp b/cpp/unittest/db/engine_test.cpp index f55d8d9e27..517b2fd506 100644 --- a/cpp/unittest/db/engine_test.cpp +++ b/cpp/unittest/db/engine_test.cpp @@ -21,67 +21,66 @@ #include "db/engine/EngineFactory.h" #include "db/engine/ExecutionEngineImpl.h" -#include "utils.h" +#include "db/utils.h" -using namespace zilliz::milvus; +namespace { + +namespace ms = zilliz::milvus; + +} TEST_F(EngineTest, FACTORY_TEST) { { - auto engine_ptr = engine::EngineFactory::Build( + auto engine_ptr = ms::engine::EngineFactory::Build( 512, "/tmp/milvus_index_1", - engine::EngineType::INVALID, - engine::MetricType::IP, - 1024 - ); + ms::engine::EngineType::INVALID, + ms::engine::MetricType::IP, + 1024); ASSERT_TRUE(engine_ptr == nullptr); } { - auto engine_ptr = engine::EngineFactory::Build( + auto engine_ptr = ms::engine::EngineFactory::Build( 512, "/tmp/milvus_index_1", - engine::EngineType::FAISS_IDMAP, - engine::MetricType::IP, - 1024 - ); + ms::engine::EngineType::FAISS_IDMAP, + ms::engine::MetricType::IP, + 1024); ASSERT_TRUE(engine_ptr != nullptr); } { - auto engine_ptr = engine::EngineFactory::Build( + auto engine_ptr = ms::engine::EngineFactory::Build( 512, "/tmp/milvus_index_1", - engine::EngineType::FAISS_IVFFLAT, - engine::MetricType::IP, - 1024 - ); + ms::engine::EngineType::FAISS_IVFFLAT, + ms::engine::MetricType::IP, + 1024); ASSERT_TRUE(engine_ptr != nullptr); } { - auto engine_ptr = engine::EngineFactory::Build( + auto engine_ptr = ms::engine::EngineFactory::Build( 512, "/tmp/milvus_index_1", - engine::EngineType::FAISS_IVFSQ8, - engine::MetricType::IP, - 1024 - ); + ms::engine::EngineType::FAISS_IVFSQ8, + ms::engine::MetricType::IP, + 1024); ASSERT_TRUE(engine_ptr != nullptr); } { - auto engine_ptr = engine::EngineFactory::Build( + auto engine_ptr = ms::engine::EngineFactory::Build( 512, "/tmp/milvus_index_1", - engine::EngineType::NSG_MIX, - engine::MetricType::IP, - 1024 - ); + ms::engine::EngineType::NSG_MIX, + ms::engine::MetricType::IP, + 1024); ASSERT_TRUE(engine_ptr != nullptr); } @@ -90,27 +89,26 @@ TEST_F(EngineTest, FACTORY_TEST) { TEST_F(EngineTest, ENGINE_IMPL_TEST) { uint16_t dimension = 64; std::string file_path = "/tmp/milvus_index_1"; - auto engine_ptr = engine::EngineFactory::Build( + auto engine_ptr = ms::engine::EngineFactory::Build( dimension, file_path, - engine::EngineType::FAISS_IVFFLAT, - engine::MetricType::IP, - 1024 - ); + ms::engine::EngineType::FAISS_IVFFLAT, + ms::engine::MetricType::IP, + 1024); std::vector data; - std::vector ids; + std::vector ids; const int row_count = 10000; data.reserve(row_count*dimension); ids.reserve(row_count); - for(long i = 0; i < row_count; i++) { + for (int64_t i = 0; i < row_count; i++) { ids.push_back(i); - for(uint16_t k = 0; k < dimension; k++) { + for (uint16_t k = 0; k < dimension; k++) { data.push_back(i*dimension + k); } } - auto status = engine_ptr->AddWithIds((long)ids.size(), data.data(), ids.data()); + auto status = engine_ptr->AddWithIds((int64_t)ids.size(), data.data(), ids.data()); ASSERT_TRUE(status.ok()); ASSERT_EQ(engine_ptr->Dimension(), dimension); @@ -127,5 +125,4 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) { // // auto engine_build = new_engine->BuildIndex("/tmp/milvus_index_2", engine::EngineType::FAISS_IVFSQ8); // //ASSERT_TRUE(status.ok()); - } diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index b85718a865..93500acce7 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -25,7 +25,7 @@ #include "db/engine/EngineFactory.h" #include "db/meta/MetaConsts.h" #include "metrics/Metrics.h" -#include "utils.h" +#include "db/utils.h" #include #include @@ -35,30 +35,32 @@ #include #include -using namespace zilliz::milvus; - namespace { -static std::string TABLE_NAME = "test_group"; +namespace ms = zilliz::milvus; + static constexpr int64_t TABLE_DIM = 256; -std::string GenTableName() { +std::string +GetTableName() { auto now = std::chrono::system_clock::now(); auto micros = std::chrono::duration_cast( - now.time_since_epoch()).count(); - TABLE_NAME = std::to_string(micros); - return TABLE_NAME; + now.time_since_epoch()).count(); + static std::string table_name = std::to_string(micros); + return table_name; } -engine::meta::TableSchema BuildTableSchema() { - engine::meta::TableSchema table_info; +ms::engine::meta::TableSchema +BuildTableSchema() { + ms::engine::meta::TableSchema table_info; table_info.dimension_ = TABLE_DIM; - table_info.table_id_ = GenTableName(); - table_info.engine_type_ = (int) engine::EngineType::FAISS_IDMAP; + table_info.table_id_ = GetTableName(); + table_info.engine_type_ = (int) ms::engine::EngineType::FAISS_IDMAP; return table_info; } -void BuildVectors(int64_t n, std::vector &vectors) { +void +BuildVectors(int64_t n, std::vector &vectors) { vectors.clear(); vectors.resize(n * TABLE_DIM); float *data = vectors.data(); @@ -67,15 +69,15 @@ void BuildVectors(int64_t n, std::vector &vectors) { data[TABLE_DIM * i + j] = drand48(); } } -} +} // namespace TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) { - engine::meta::TableSchema table_schema = BuildTableSchema(); + ms::engine::meta::TableSchema table_schema = BuildTableSchema(); auto status = impl_->CreateTable(table_schema); ASSERT_TRUE(status.ok()); - engine::meta::TableFileSchema table_file_schema; - table_file_schema.table_id_ = TABLE_NAME; + ms::engine::meta::TableFileSchema table_file_schema; + table_file_schema.table_id_ = GetTableName(); status = impl_->CreateTableFile(table_file_schema); ASSERT_TRUE(status.ok()); @@ -83,17 +85,17 @@ TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) { std::vector vectors; BuildVectors(n, vectors); - engine::VectorSource source(n, vectors.data()); + ms::engine::VectorSource source(n, vectors.data()); size_t num_vectors_added; - engine::ExecutionEnginePtr execution_engine_ = - engine::EngineFactory::Build(table_file_schema.dimension_, - table_file_schema.location_, - (engine::EngineType) table_file_schema.engine_type_, - (engine::MetricType)table_file_schema.metric_type_, - table_schema.nlist_); + ms::engine::ExecutionEnginePtr execution_engine_ = + ms::engine::EngineFactory::Build(table_file_schema.dimension_, + table_file_schema.location_, + (ms::engine::EngineType) table_file_schema.engine_type_, + (ms::engine::MetricType) table_file_schema.metric_type_, + table_schema.nlist_); - engine::IDNumbers vector_ids; + ms::engine::IDNumbers vector_ids; status = source.Add(execution_engine_, table_file_schema, 50, num_vectors_added, vector_ids); ASSERT_TRUE(status.ok()); vector_ids = source.GetVectorIds(); @@ -113,19 +115,19 @@ TEST_F(MemManagerTest, VECTOR_SOURCE_TEST) { TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) { auto options = GetOptions(); - engine::meta::TableSchema table_schema = BuildTableSchema(); + ms::engine::meta::TableSchema table_schema = BuildTableSchema(); auto status = impl_->CreateTable(table_schema); ASSERT_TRUE(status.ok()); - engine::MemTableFile mem_table_file(TABLE_NAME, impl_, options); + ms::engine::MemTableFile mem_table_file(GetTableName(), impl_, options); int64_t n_100 = 100; std::vector vectors_100; BuildVectors(n_100, vectors_100); - engine::VectorSourcePtr source = std::make_shared(n_100, vectors_100.data()); + ms::engine::VectorSourcePtr source = std::make_shared(n_100, vectors_100.data()); - engine::IDNumbers vector_ids; + ms::engine::IDNumbers vector_ids; status = mem_table_file.Add(source, vector_ids); ASSERT_TRUE(status.ok()); @@ -137,11 +139,11 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) { size_t singleVectorMem = sizeof(float) * TABLE_DIM; ASSERT_EQ(mem_table_file.GetCurrentMem(), n_100 * singleVectorMem); - int64_t n_max = engine::MAX_TABLE_FILE_MEM / singleVectorMem; + int64_t n_max = ms::engine::MAX_TABLE_FILE_MEM / singleVectorMem; std::vector vectors_128M; BuildVectors(n_max, vectors_128M); - engine::VectorSourcePtr source_128M = std::make_shared(n_max, vectors_128M.data()); + ms::engine::VectorSourcePtr source_128M = std::make_shared(n_max, vectors_128M.data()); vector_ids.clear(); status = mem_table_file.Add(source_128M, vector_ids); @@ -154,7 +156,7 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) { TEST_F(MemManagerTest, MEM_TABLE_TEST) { auto options = GetOptions(); - engine::meta::TableSchema table_schema = BuildTableSchema(); + ms::engine::meta::TableSchema table_schema = BuildTableSchema(); auto status = impl_->CreateTable(table_schema); ASSERT_TRUE(status.ok()); @@ -162,27 +164,27 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) { std::vector vectors_100; BuildVectors(n_100, vectors_100); - engine::VectorSourcePtr source_100 = std::make_shared(n_100, vectors_100.data()); + ms::engine::VectorSourcePtr source_100 = std::make_shared(n_100, vectors_100.data()); - engine::MemTable mem_table(TABLE_NAME, impl_, options); + ms::engine::MemTable mem_table(GetTableName(), impl_, options); - engine::IDNumbers vector_ids; + ms::engine::IDNumbers vector_ids; status = mem_table.Add(source_100, vector_ids); ASSERT_TRUE(status.ok()); vector_ids = source_100->GetVectorIds(); ASSERT_EQ(vector_ids.size(), 100); - engine::MemTableFilePtr mem_table_file; + ms::engine::MemTableFilePtr mem_table_file; mem_table.GetCurrentMemTableFile(mem_table_file); size_t singleVectorMem = sizeof(float) * TABLE_DIM; ASSERT_EQ(mem_table_file->GetCurrentMem(), n_100 * singleVectorMem); - int64_t n_max = engine::MAX_TABLE_FILE_MEM / singleVectorMem; + int64_t n_max = ms::engine::MAX_TABLE_FILE_MEM / singleVectorMem; std::vector vectors_128M; BuildVectors(n_max, vectors_128M); vector_ids.clear(); - engine::VectorSourcePtr source_128M = std::make_shared(n_max, vectors_128M.data()); + ms::engine::VectorSourcePtr source_128M = std::make_shared(n_max, vectors_128M.data()); status = mem_table.Add(source_128M, vector_ids); ASSERT_TRUE(status.ok()); @@ -198,7 +200,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) { std::vector vectors_1G; BuildVectors(n_1G, vectors_1G); - engine::VectorSourcePtr source_1G = std::make_shared(n_1G, vectors_1G.data()); + ms::engine::VectorSourcePtr source_1G = std::make_shared(n_1G, vectors_1G.data()); vector_ids.clear(); status = mem_table.Add(source_1G, vector_ids); @@ -207,7 +209,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) { vector_ids = source_1G->GetVectorIds(); ASSERT_EQ(vector_ids.size(), n_1G); - int expectedTableFileCount = 2 + std::ceil((n_1G - n_100) * singleVectorMem / engine::MAX_TABLE_FILE_MEM); + int expectedTableFileCount = 2 + std::ceil((n_1G - n_100) * singleVectorMem / ms::engine::MAX_TABLE_FILE_MEM); ASSERT_EQ(mem_table.GetTableFileCount(), expectedTableFileCount); status = mem_table.Serialize(); @@ -215,11 +217,11 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) { } TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) { - engine::meta::TableSchema table_info = BuildTableSchema(); + ms::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = TABLE_NAME; + ms::engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = GetTableName(); stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -228,12 +230,12 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) { std::vector xb; BuildVectors(nb, xb); - engine::IDNumbers vector_ids; - for(int64_t i = 0; i < nb; i++) { + ms::engine::IDNumbers vector_ids; + for (int64_t i = 0; i < nb; i++) { vector_ids.push_back(i); } - stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids); ASSERT_TRUE(stat.ok()); std::this_thread::sleep_for(std::chrono::seconds(3));//ensure raw data write to disk @@ -256,19 +258,19 @@ TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) { int topk = 10, nprobe = 10; for (auto &pair : search_vectors) { auto &search = pair.second; - engine::QueryResults results; - stat = db_->Query(TABLE_NAME, topk, 1, nprobe, search.data(), results); + ms::engine::QueryResults results; + stat = db_->Query(GetTableName(), topk, 1, nprobe, search.data(), results); ASSERT_EQ(results[0][0].first, pair.first); ASSERT_LT(results[0][0].second, 1e-4); } } TEST_F(MemManagerTest2, INSERT_TEST) { - engine::meta::TableSchema table_info = BuildTableSchema(); + ms::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = TABLE_NAME; + ms::engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = GetTableName(); stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -280,8 +282,8 @@ TEST_F(MemManagerTest2, INSERT_TEST) { int64_t nb = 40960; std::vector xb; BuildVectors(nb, xb); - engine::IDNumbers vector_ids; - stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + ms::engine::IDNumbers vector_ids; + stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids); ASSERT_TRUE(stat.ok()); } auto end_time = METRICS_NOW_TIME; @@ -290,17 +292,17 @@ TEST_F(MemManagerTest2, INSERT_TEST) { } TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) { - engine::meta::TableSchema table_info = BuildTableSchema(); + ms::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = TABLE_NAME; + ms::engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = GetTableName(); stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - engine::IDNumbers vector_ids; - engine::IDNumbers target_ids; + ms::engine::IDNumbers vector_ids; + ms::engine::IDNumbers target_ids; int64_t nb = 40960; std::vector xb; @@ -311,7 +313,7 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) { BuildVectors(qb, qxb); std::thread search([&]() { - engine::QueryResults results; + ms::engine::QueryResults results; int k = 10; std::this_thread::sleep_for(std::chrono::seconds(2)); @@ -326,8 +328,8 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) { prev_count = count; START_TIMER; - stat = db_->Query(TABLE_NAME, k, qb, 10, qxb.data(), results); - ss << "Search " << j << " With Size " << count / engine::meta::M << " M"; + stat = db_->Query(GetTableName(), k, qb, 10, qxb.data(), results); + ss << "Search " << j << " With Size " << count / ms::engine::meta::M << " M"; STOP_TIMER(ss.str()); ASSERT_TRUE(stat.ok()); @@ -349,29 +351,28 @@ TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) { for (auto i = 0; i < loop; ++i) { if (i == 0) { - db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); + db_->InsertVectors(GetTableName(), qb, qxb.data(), target_ids); ASSERT_EQ(target_ids.size(), qb); } else { - db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids); } std::this_thread::sleep_for(std::chrono::microseconds(1)); } search.join(); -}; +} TEST_F(MemManagerTest2, VECTOR_IDS_TEST) { - engine::meta::TableSchema table_info = BuildTableSchema(); + ms::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = TABLE_NAME; + ms::engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = GetTableName(); stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - engine::IDNumbers vector_ids; - + ms::engine::IDNumbers vector_ids; int64_t nb = 100000; std::vector xb; @@ -382,7 +383,7 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) { vector_ids[i] = i; } - stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids); ASSERT_EQ(vector_ids[0], 0); ASSERT_TRUE(stat.ok()); @@ -394,7 +395,7 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) { for (auto i = 0; i < nb; i++) { vector_ids[i] = i + nb; } - stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids); ASSERT_EQ(vector_ids[0], nb); ASSERT_TRUE(stat.ok()); @@ -406,15 +407,15 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) { for (auto i = 0; i < nb; i++) { vector_ids[i] = i + nb / 2; } - stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); - ASSERT_EQ(vector_ids[0], nb/2); + stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids); + ASSERT_EQ(vector_ids[0], nb / 2); ASSERT_TRUE(stat.ok()); nb = 65536; //128M xb.clear(); BuildVectors(nb, xb); vector_ids.clear(); - stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids); ASSERT_TRUE(stat.ok()); nb = 100; @@ -425,8 +426,9 @@ TEST_F(MemManagerTest2, VECTOR_IDS_TEST) { for (auto i = 0; i < nb; i++) { vector_ids[i] = i + nb; } - stat = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + stat = db_->InsertVectors(GetTableName(), nb, xb.data(), vector_ids); for (auto i = 0; i < nb; i++) { ASSERT_EQ(vector_ids[i], i + nb); } } + diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 23897408bc..39217ea298 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "utils.h" +#include "db/utils.h" #include "db/meta/SqliteMetaImpl.h" #include "db/Utils.h" #include "db/meta/MetaConsts.h" @@ -25,13 +25,16 @@ #include #include -using namespace zilliz::milvus; -using namespace zilliz::milvus::engine; +namespace { + +namespace ms = zilliz::milvus; + +} // namespace TEST_F(MetaTest, TABLE_TEST) { auto table_id = "meta_test_table"; - meta::TableSchema table; + ms::engine::meta::TableSchema table; table.table_id_ = table_id; auto status = impl_->CreateTable(table); ASSERT_TRUE(status.ok()); @@ -49,7 +52,7 @@ TEST_F(MetaTest, TABLE_TEST) { table.table_id_ = table_id; status = impl_->CreateTable(table); - ASSERT_EQ(status.code(), DB_ALREADY_EXIST); + ASSERT_EQ(status.code(), ms::DB_ALREADY_EXIST); table.table_id_ = ""; status = impl_->CreateTable(table); @@ -59,16 +62,16 @@ TEST_F(MetaTest, TABLE_TEST) { TEST_F(MetaTest, TABLE_FILE_TEST) { auto table_id = "meta_test_table"; - meta::TableSchema table; + ms::engine::meta::TableSchema table; table.table_id_ = table_id; table.dimension_ = 256; auto status = impl_->CreateTable(table); - meta::TableFileSchema table_file; + ms::engine::meta::TableFileSchema table_file; table_file.table_id_ = table.table_id_; status = impl_->CreateTableFile(table_file); ASSERT_TRUE(status.ok()); - ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); + ASSERT_EQ(table_file.file_type_, ms::engine::meta::TableFileSchema::NEW); uint64_t cnt = 0; status = impl_->Count(table_id, cnt); @@ -77,30 +80,30 @@ TEST_F(MetaTest, TABLE_FILE_TEST) { auto file_id = table_file.file_id_; - auto new_file_type = meta::TableFileSchema::INDEX; + auto new_file_type = ms::engine::meta::TableFileSchema::INDEX; table_file.file_type_ = new_file_type; status = impl_->UpdateTableFile(table_file); ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type_, new_file_type); - meta::DatesT dates; - dates.push_back(utils::GetDate()); + ms::engine::meta::DatesT dates; + dates.push_back(ms::engine::utils::GetDate()); status = impl_->DropPartitionsByDates(table_file.table_id_, dates); ASSERT_TRUE(status.ok()); dates.clear(); - for (auto i=2; i < 10; ++i) { - dates.push_back(utils::GetDateWithDelta(-1*i)); + for (auto i = 2; i < 10; ++i) { + dates.push_back(ms::engine::utils::GetDateWithDelta(-1 * i)); } status = impl_->DropPartitionsByDates(table_file.table_id_, dates); ASSERT_TRUE(status.ok()); - table_file.date_ = utils::GetDateWithDelta(-2); + table_file.date_ = ms::engine::utils::GetDateWithDelta(-2); status = impl_->UpdateTableFile(table_file); ASSERT_TRUE(status.ok()); - ASSERT_EQ(table_file.date_, utils::GetDateWithDelta(-2)); - ASSERT_FALSE(table_file.file_type_ == meta::TableFileSchema::TO_DELETE); + ASSERT_EQ(table_file.date_, ms::engine::utils::GetDateWithDelta(-2)); + ASSERT_FALSE(table_file.file_type_ == ms::engine::meta::TableFileSchema::TO_DELETE); dates.clear(); dates.push_back(table_file.date_); @@ -108,7 +111,7 @@ TEST_F(MetaTest, TABLE_FILE_TEST) { ASSERT_TRUE(status.ok()); std::vector ids = {table_file.id_}; - meta::TableFilesSchema files; + ms::engine::meta::TableFilesSchema files; status = impl_->GetTableFiles(table_file.table_id_, ids, files); ASSERT_TRUE(status.ok()); ASSERT_EQ(files.size(), 0UL); @@ -116,33 +119,34 @@ TEST_F(MetaTest, TABLE_FILE_TEST) { TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { srand(time(0)); - DBMetaOptions options; + ms::engine::DBMetaOptions options; options.path_ = "/tmp/milvus_test"; - int days_num = rand() % 100; + unsigned int seed = 1; + int days_num = rand_r(&seed) % 100; std::stringstream ss; ss << "days:" << days_num; - options.archive_conf_ = ArchiveConf("delete", ss.str()); + options.archive_conf_ = ms::engine::ArchiveConf("delete", ss.str()); - meta::SqliteMetaImpl impl(options); + ms::engine::meta::SqliteMetaImpl impl(options); auto table_id = "meta_test_table"; - meta::TableSchema table; + ms::engine::meta::TableSchema table; table.table_id_ = table_id; auto status = impl.CreateTable(table); - meta::TableFilesSchema files; - meta::TableFileSchema table_file; + ms::engine::meta::TableFilesSchema files; + ms::engine::meta::TableFileSchema table_file; table_file.table_id_ = table.table_id_; auto cnt = 100; - long ts = utils::GetMicroSecTimeStamp(); + int64_t ts = ms::engine::utils::GetMicroSecTimeStamp(); std::vector days; std::vector ids; - for (auto i=0; i ids; - for (auto i=0; i= 5) { - ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW); + ASSERT_EQ(file.file_type_, ms::engine::meta::TableFileSchema::NEW); } ++i; } @@ -214,7 +218,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { TEST_F(MetaTest, TABLE_FILES_TEST) { auto table_id = "meta_test_group"; - meta::TableSchema table; + ms::engine::meta::TableSchema table; table.table_id_ = table_id; auto status = impl_->CreateTable(table); @@ -226,51 +230,51 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { uint64_t to_index_files_cnt = 6; uint64_t index_files_cnt = 7; - meta::TableFileSchema table_file; + ms::engine::meta::TableFileSchema table_file; table_file.table_id_ = table.table_id_; - for (auto i=0; iCreateTableFile(table_file); - table_file.file_type_ = meta::TableFileSchema::NEW_MERGE; + table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_MERGE; status = impl_->UpdateTableFile(table_file); } - for (auto i=0; iCreateTableFile(table_file); - table_file.file_type_ = meta::TableFileSchema::NEW_INDEX; + table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_INDEX; status = impl_->UpdateTableFile(table_file); } - for (auto i=0; iCreateTableFile(table_file); - table_file.file_type_ = meta::TableFileSchema::BACKUP; + table_file.file_type_ = ms::engine::meta::TableFileSchema::BACKUP; table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } - for (auto i=0; iCreateTableFile(table_file); - table_file.file_type_ = meta::TableFileSchema::NEW; + table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW; status = impl_->UpdateTableFile(table_file); } - for (auto i=0; iCreateTableFile(table_file); - table_file.file_type_ = meta::TableFileSchema::RAW; + table_file.file_type_ = ms::engine::meta::TableFileSchema::RAW; table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } - for (auto i=0; iCreateTableFile(table_file); - table_file.file_type_ = meta::TableFileSchema::TO_INDEX; + table_file.file_type_ = ms::engine::meta::TableFileSchema::TO_INDEX; table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } - for (auto i=0; iCreateTableFile(table_file); - table_file.file_type_ = meta::TableFileSchema::INDEX; + table_file.file_type_ = ms::engine::meta::TableFileSchema::INDEX; table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } @@ -278,36 +282,36 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { uint64_t total_row_count = 0; status = impl_->Count(table_id, total_row_count); ASSERT_TRUE(status.ok()); - ASSERT_EQ(total_row_count, raw_files_cnt+to_index_files_cnt+index_files_cnt); + ASSERT_EQ(total_row_count, raw_files_cnt + to_index_files_cnt + index_files_cnt); - meta::TableFilesSchema files; + ms::engine::meta::TableFilesSchema files; status = impl_->FilesToIndex(files); ASSERT_EQ(files.size(), to_index_files_cnt); - meta::DatePartionedTableFilesSchema dated_files; + ms::engine::meta::DatePartionedTableFilesSchema dated_files; status = impl_->FilesToMerge(table.table_id_, dated_files); ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt); status = impl_->FilesToIndex(files); ASSERT_EQ(files.size(), to_index_files_cnt); - meta::DatesT dates = {table_file.date_}; + ms::engine::meta::DatesT dates = {table_file.date_}; std::vector ids; status = impl_->FilesToSearch(table_id, ids, dates, dated_files); ASSERT_EQ(dated_files[table_file.date_].size(), - to_index_files_cnt+raw_files_cnt+index_files_cnt); + to_index_files_cnt + raw_files_cnt + index_files_cnt); - status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files); + status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files); ASSERT_EQ(dated_files[table_file.date_].size(), - to_index_files_cnt+raw_files_cnt+index_files_cnt); + to_index_files_cnt + raw_files_cnt + index_files_cnt); - status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files); + status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files); ASSERT_EQ(dated_files[table_file.date_].size(), - to_index_files_cnt+raw_files_cnt+index_files_cnt); + to_index_files_cnt + raw_files_cnt + index_files_cnt); ids.push_back(size_t(9999999999)); status = impl_->FilesToSearch(table_id, ids, dates, dated_files); - ASSERT_EQ(dated_files[table_file.date_].size(),0); + ASSERT_EQ(dated_files[table_file.date_].size(), 0); std::vector file_types; std::vector file_ids; @@ -316,26 +320,26 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { ASSERT_FALSE(status.ok()); file_types = { - meta::TableFileSchema::NEW, - meta::TableFileSchema::NEW_MERGE, - meta::TableFileSchema::NEW_INDEX, - meta::TableFileSchema::TO_INDEX, - meta::TableFileSchema::INDEX, - meta::TableFileSchema::RAW, - meta::TableFileSchema::BACKUP, + ms::engine::meta::TableFileSchema::NEW, + ms::engine::meta::TableFileSchema::NEW_MERGE, + ms::engine::meta::TableFileSchema::NEW_INDEX, + ms::engine::meta::TableFileSchema::TO_INDEX, + ms::engine::meta::TableFileSchema::INDEX, + ms::engine::meta::TableFileSchema::RAW, + ms::engine::meta::TableFileSchema::BACKUP, }; status = impl_->FilesByType(table.table_id_, file_types, file_ids); ASSERT_TRUE(status.ok()); uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt + - backup_files_cnt + new_files_cnt + raw_files_cnt + - to_index_files_cnt + index_files_cnt; + backup_files_cnt + new_files_cnt + raw_files_cnt + + to_index_files_cnt + index_files_cnt; ASSERT_EQ(file_ids.size(), total_cnt); status = impl_->DeleteTableFiles(table_id); ASSERT_TRUE(status.ok()); status = impl_->CreateTableFile(table_file); - table_file.file_type_ = meta::TableFileSchema::NEW; + table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW; status = impl_->UpdateTableFile(table_file); status = impl_->CleanUp(); ASSERT_TRUE(status.ok()); @@ -350,11 +354,11 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { TEST_F(MetaTest, INDEX_TEST) { auto table_id = "index_test"; - meta::TableSchema table; + ms::engine::meta::TableSchema table; table.table_id_ = table_id; auto status = impl_->CreateTable(table); - TableIndex index; + ms::engine::TableIndex index; index.metric_type_ = 2; index.nlist_ = 1234; index.engine_type_ = 3; @@ -365,12 +369,12 @@ TEST_F(MetaTest, INDEX_TEST) { status = impl_->UpdateTableFlag(table_id, flag); ASSERT_TRUE(status.ok()); - engine::meta::TableSchema table_info; + ms::engine::meta::TableSchema table_info; table_info.table_id_ = table_id; status = impl_->DescribeTable(table_info); ASSERT_EQ(table_info.flag_, flag); - TableIndex index_out; + ms::engine::TableIndex index_out; status = impl_->DescribeTableIndex(table_id, index_out); ASSERT_EQ(index_out.metric_type_, index.metric_type_); ASSERT_EQ(index_out.nlist_, index.nlist_); @@ -385,4 +389,4 @@ TEST_F(MetaTest, INDEX_TEST) { status = impl_->UpdateTableFilesToIndex(table_id); ASSERT_TRUE(status.ok()); -} \ No newline at end of file +} diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index 762e9944ee..d6b024930f 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -27,43 +27,47 @@ #include #include -using namespace zilliz::milvus; +namespace { + +namespace ms = zilliz::milvus; + +} // namespace TEST(DBMiscTest, EXCEPTION_TEST) { - Exception ex1(100, "error"); + ms::Exception ex1(100, "error"); std::string what = ex1.what(); ASSERT_EQ(what, "error"); ASSERT_EQ(ex1.code(), 100); - InvalidArgumentException ex2; - ASSERT_EQ(ex2.code(), SERVER_INVALID_ARGUMENT); + ms::InvalidArgumentException ex2; + ASSERT_EQ(ex2.code(), ms::SERVER_INVALID_ARGUMENT); } TEST(DBMiscTest, OPTIONS_TEST) { try { - engine::ArchiveConf archive("$$##"); - } catch (std::exception& ex) { + ms::engine::ArchiveConf archive("$$##"); + } catch (std::exception &ex) { ASSERT_TRUE(true); } { - engine::ArchiveConf archive("delete", "no"); + ms::engine::ArchiveConf archive("delete", "no"); ASSERT_TRUE(archive.GetCriterias().empty()); } { - engine::ArchiveConf archive("delete", "1:2"); + ms::engine::ArchiveConf archive("delete", "1:2"); ASSERT_TRUE(archive.GetCriterias().empty()); } { - engine::ArchiveConf archive("delete", "1:2:3"); + ms::engine::ArchiveConf archive("delete", "1:2:3"); ASSERT_TRUE(archive.GetCriterias().empty()); } { - engine::ArchiveConf archive("delete"); - engine::ArchiveConf::CriteriaT criterial = { + ms::engine::ArchiveConf archive("delete"); + ms::engine::ArchiveConf::CriteriaT criterial = { {"disk", 1024}, {"days", 100} }; @@ -76,29 +80,29 @@ TEST(DBMiscTest, OPTIONS_TEST) { } TEST(DBMiscTest, META_TEST) { - engine::DBMetaOptions options; + ms::engine::DBMetaOptions options; options.path_ = "/tmp/milvus_test"; - engine::meta::SqliteMetaImpl impl(options); + ms::engine::meta::SqliteMetaImpl impl(options); time_t tt; - time( &tt ); + time(&tt); int delta = 10; - engine::meta::DateT dt = engine::utils::GetDate(tt, delta); + ms::engine::meta::DateT dt = ms::engine::utils::GetDate(tt, delta); ASSERT_GT(dt, 0); } TEST(DBMiscTest, UTILS_TEST) { - engine::DBMetaOptions options; + ms::engine::DBMetaOptions options; options.path_ = "/tmp/milvus_test/main"; options.slave_paths_.push_back("/tmp/milvus_test/slave_1"); options.slave_paths_.push_back("/tmp/milvus_test/slave_2"); const std::string TABLE_NAME = "test_tbl"; - auto status = engine::utils::CreateTablePath(options, TABLE_NAME); + auto status = ms::engine::utils::CreateTablePath(options, TABLE_NAME); ASSERT_TRUE(status.ok()); ASSERT_TRUE(boost::filesystem::exists(options.path_)); - for(auto& path : options.slave_paths_) { - ASSERT_TRUE(boost::filesystem::exists(path)); + for (auto &path : options.slave_paths_) { + ASSERT_TRUE(boost::filesystem::exists(path)); } // options.slave_paths.push_back("/"); @@ -109,18 +113,18 @@ TEST(DBMiscTest, UTILS_TEST) { // status = engine::utils::CreateTablePath(options, TABLE_NAME); // ASSERT_FALSE(status.ok()); - engine::meta::TableFileSchema file; + ms::engine::meta::TableFileSchema file; file.id_ = 50; file.table_id_ = TABLE_NAME; file.file_type_ = 3; file.date_ = 155000; - status = engine::utils::GetTableFilePath(options, file); + status = ms::engine::utils::GetTableFilePath(options, file); ASSERT_FALSE(status.ok()); ASSERT_TRUE(file.location_.empty()); - status = engine::utils::DeleteTablePath(options, TABLE_NAME); + status = ms::engine::utils::DeleteTablePath(options, TABLE_NAME); ASSERT_TRUE(status.ok()); - status = engine::utils::DeleteTableFilePath(options, file); + status = ms::engine::utils::DeleteTableFilePath(options, file); ASSERT_TRUE(status.ok()); -} \ No newline at end of file +} diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp index 53ce51ca45..2a7456785e 100644 --- a/cpp/unittest/db/mysql_db_test.cpp +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "utils.h" +#include "db/utils.h" #include "db/DB.h" #include "db/DBImpl.h" #include "db/meta/MetaConsts.h" @@ -26,48 +26,49 @@ #include #include -using namespace zilliz::milvus; - namespace { - static const char* TABLE_NAME = "test_group"; - static constexpr int64_t TABLE_DIM = 256; - static constexpr int64_t VECTOR_COUNT = 25000; - static constexpr int64_t INSERT_LOOP = 1000; +namespace ms = zilliz::milvus; - engine::meta::TableSchema BuildTableSchema() { - engine::meta::TableSchema table_info; - table_info.dimension_ = TABLE_DIM; - table_info.table_id_ = TABLE_NAME; - table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; - return table_info; - } - - void BuildVectors(int64_t n, std::vector& vectors) { - vectors.clear(); - vectors.resize(n*TABLE_DIM); - float* data = vectors.data(); - for(int i = 0; i < n; i++) { - for(int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); - data[TABLE_DIM * i] += i / 2000.; - } - } +static const char *TABLE_NAME = "test_group"; +static constexpr int64_t TABLE_DIM = 256; +static constexpr int64_t VECTOR_COUNT = 25000; +static constexpr int64_t INSERT_LOOP = 1000; +ms::engine::meta::TableSchema +BuildTableSchema() { + ms::engine::meta::TableSchema table_info; + table_info.dimension_ = TABLE_DIM; + table_info.table_id_ = TABLE_NAME; + table_info.engine_type_ = (int) ms::engine::EngineType::FAISS_IDMAP; + return table_info; } +void +BuildVectors(int64_t n, std::vector &vectors) { + vectors.clear(); + vectors.resize(n * TABLE_DIM); + float *data = vectors.data(); + for (int i = 0; i < n; i++) { + for (int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); + data[TABLE_DIM * i] += i / 2000.; + } +} + +} // namespace TEST_F(MySqlDBTest, DB_TEST) { - engine::meta::TableSchema table_info = BuildTableSchema(); + ms::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - engine::meta::TableSchema table_info_get; + ms::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - engine::IDNumbers vector_ids; - engine::IDNumbers target_ids; + ms::engine::IDNumbers vector_ids; + ms::engine::IDNumbers target_ids; int64_t nb = 50; std::vector xb; @@ -81,7 +82,7 @@ TEST_F(MySqlDBTest, DB_TEST) { ASSERT_EQ(target_ids.size(), qb); std::thread search([&]() { - engine::QueryResults results; + ms::engine::QueryResults results; int k = 10; std::this_thread::sleep_for(std::chrono::seconds(5)); @@ -90,22 +91,22 @@ TEST_F(MySqlDBTest, DB_TEST) { uint64_t count = 0; uint64_t prev_count = 0; - for (auto j=0; j<10; ++j) { + for (auto j = 0; j < 10; ++j) { ss.str(""); db_->Size(count); prev_count = count; START_TIMER; stat = db_->Query(TABLE_NAME, k, qb, 10, qxb.data(), results); - ss << "Search " << j << " With Size " << count/engine::meta::M << " M"; + ss << "Search " << j << " With Size " << count / ms::engine::meta::M << " M"; STOP_TIMER(ss.str()); ASSERT_TRUE(stat.ok()); - for (auto k=0; kInsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); // ASSERT_EQ(target_ids.size(), qb); @@ -139,13 +140,13 @@ TEST_F(MySqlDBTest, DB_TEST) { } search.join(); -}; +} TEST_F(MySqlDBTest, SEARCH_TEST) { - engine::meta::TableSchema table_info = BuildTableSchema(); + ms::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - engine::meta::TableSchema table_info_get; + ms::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); @@ -155,68 +156,68 @@ TEST_F(MySqlDBTest, SEARCH_TEST) { size_t nb = VECTOR_COUNT; size_t nq = 10; size_t k = 5; - std::vector xb(nb*TABLE_DIM); - std::vector xq(nq*TABLE_DIM); - std::vector ids(nb); + std::vector xb(nb * TABLE_DIM); + std::vector xq(nq * TABLE_DIM); + std::vector ids(nb); std::random_device rd; std::mt19937 gen(rd()); std::uniform_real_distribution<> dis_xt(-1.0, 1.0); - for (size_t i = 0; i < nb*TABLE_DIM; i++) { + for (size_t i = 0; i < nb * TABLE_DIM; i++) { xb[i] = dis_xt(gen); - if (i < nb){ + if (i < nb) { ids[i] = i; } } - for (size_t i = 0; i < nq*TABLE_DIM; i++) { + for (size_t i = 0; i < nq * TABLE_DIM; i++) { xq[i] = dis_xt(gen); } // result data //std::vector nns_gt(k*nq); - std::vector nns(k*nq); // nns = nearst neg search + std::vector nns(k * nq); // nns = nearst neg search //std::vector dis_gt(k*nq); - std::vector dis(k*nq); + std::vector dis(k * nq); // insert data const int batch_size = 100; for (int j = 0; j < nb / batch_size; ++j) { - stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data()+batch_size*j*TABLE_DIM, ids); - if (j == 200){ sleep(1);} + stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data() + batch_size * j * TABLE_DIM, ids); + if (j == 200) { sleep(1); } ASSERT_TRUE(stat.ok()); } sleep(2); // wait until build index finish - engine::QueryResults results; + ms::engine::QueryResults results; stat = db_->Query(TABLE_NAME, k, nq, 10, xq.data(), results); ASSERT_TRUE(stat.ok()); -}; +} TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) { - engine::meta::TableSchema table_info = BuildTableSchema(); + ms::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); - std::vector table_schema_array; + std::vector table_schema_array; stat = db_->AllTables(table_schema_array); ASSERT_TRUE(stat.ok()); bool bfound = false; - for(auto& schema : table_schema_array) { - if(schema.table_id_ == TABLE_NAME) { + for (auto &schema : table_schema_array) { + if (schema.table_id_ == TABLE_NAME) { bfound = true; break; } } ASSERT_TRUE(bfound); - engine::meta::TableSchema table_info_get; + ms::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - engine::IDNumbers vector_ids; - engine::IDNumbers target_ids; + ms::engine::IDNumbers vector_ids; + ms::engine::IDNumbers target_ids; uint64_t size; db_->Size(size); @@ -226,7 +227,7 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) { BuildVectors(nb, xb); int loop = INSERT_LOOP; - for (auto i=0; iInsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); std::this_thread::sleep_for(std::chrono::microseconds(1)); } @@ -235,15 +236,15 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) { db_->Size(size); LOG(DEBUG) << "size=" << size; - ASSERT_LE(size, 1 * engine::meta::G); -}; + ASSERT_LE(size, 1 * ms::engine::meta::G); +} TEST_F(MySqlDBTest, DELETE_TEST) { - engine::meta::TableSchema table_info = BuildTableSchema(); + ms::engine::meta::TableSchema table_info = BuildTableSchema(); auto stat = db_->CreateTable(table_info); // std::cout << stat.ToString() << std::endl; - engine::meta::TableSchema table_info_get; + ms::engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); @@ -252,7 +253,7 @@ TEST_F(MySqlDBTest, DELETE_TEST) { db_->HasTable(TABLE_NAME, has_table); ASSERT_TRUE(has_table); - engine::IDNumbers vector_ids; + ms::engine::IDNumbers vector_ids; uint64_t size; db_->Size(size); @@ -262,7 +263,7 @@ TEST_F(MySqlDBTest, DELETE_TEST) { BuildVectors(nb, xb); int loop = 20; - for (auto i=0; iInsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); std::this_thread::sleep_for(std::chrono::microseconds(1)); } @@ -276,4 +277,5 @@ TEST_F(MySqlDBTest, DELETE_TEST) { // // db_->HasTable(TABLE_NAME, has_table); // ASSERT_FALSE(has_table); -}; +} + diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index 46b030235e..5605b9eb14 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "utils.h" +#include "db/utils.h" #include "db/meta/MySQLMetaImpl.h" #include "db/Utils.h" #include "db/meta/MetaConsts.h" @@ -27,13 +27,17 @@ #include #include -using namespace zilliz::milvus; -using namespace zilliz::milvus::engine; +namespace { + +namespace ms = zilliz::milvus; + +} + TEST_F(MySqlMetaTest, TABLE_TEST) { auto table_id = "meta_test_table"; - meta::TableSchema table; + ms::engine::meta::TableSchema table; table.table_id_ = table_id; auto status = impl_->CreateTable(table); ASSERT_TRUE(status.ok()); @@ -51,7 +55,7 @@ TEST_F(MySqlMetaTest, TABLE_TEST) { table.table_id_ = table_id; status = impl_->CreateTable(table); - ASSERT_EQ(status.code(), DB_ALREADY_EXIST); + ASSERT_EQ(status.code(), ms::DB_ALREADY_EXIST); table.table_id_ = ""; status = impl_->CreateTable(table); @@ -64,20 +68,19 @@ TEST_F(MySqlMetaTest, TABLE_TEST) { TEST_F(MySqlMetaTest, TABLE_FILE_TEST) { auto table_id = "meta_test_table"; - meta::TableSchema table; + ms::engine::meta::TableSchema table; table.table_id_ = table_id; table.dimension_ = 256; auto status = impl_->CreateTable(table); - - meta::TableFileSchema table_file; + ms::engine::meta::TableFileSchema table_file; table_file.table_id_ = table.table_id_; status = impl_->CreateTableFile(table_file); ASSERT_TRUE(status.ok()); - ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); + ASSERT_EQ(table_file.file_type_, ms::engine::meta::TableFileSchema::NEW); - meta::DatesT dates; - dates.push_back(utils::GetDate()); + ms::engine::meta::DatesT dates; + dates.push_back(ms::engine::utils::GetDate()); status = impl_->DropPartitionsByDates(table_file.table_id_, dates); ASSERT_TRUE(status.ok()); @@ -88,7 +91,7 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) { auto file_id = table_file.file_id_; - auto new_file_type = meta::TableFileSchema::INDEX; + auto new_file_type = ms::engine::meta::TableFileSchema::INDEX; table_file.file_type_ = new_file_type; status = impl_->UpdateTableFile(table_file); @@ -96,17 +99,17 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) { ASSERT_EQ(table_file.file_type_, new_file_type); dates.clear(); - for (auto i=2; i < 10; ++i) { - dates.push_back(utils::GetDateWithDelta(-1*i)); + for (auto i = 2; i < 10; ++i) { + dates.push_back(ms::engine::utils::GetDateWithDelta(-1 * i)); } status = impl_->DropPartitionsByDates(table_file.table_id_, dates); ASSERT_TRUE(status.ok()); - table_file.date_ = utils::GetDateWithDelta(-2); + table_file.date_ = ms::engine::utils::GetDateWithDelta(-2); status = impl_->UpdateTableFile(table_file); ASSERT_TRUE(status.ok()); - ASSERT_EQ(table_file.date_, utils::GetDateWithDelta(-2)); - ASSERT_FALSE(table_file.file_type_ == meta::TableFileSchema::TO_DELETE); + ASSERT_EQ(table_file.date_, ms::engine::utils::GetDateWithDelta(-2)); + ASSERT_FALSE(table_file.file_type_ == ms::engine::meta::TableFileSchema::TO_DELETE); dates.clear(); dates.push_back(table_file.date_); @@ -114,41 +117,42 @@ TEST_F(MySqlMetaTest, TABLE_FILE_TEST) { ASSERT_TRUE(status.ok()); std::vector ids = {table_file.id_}; - meta::TableFilesSchema files; + ms::engine::meta::TableFilesSchema files; status = impl_->GetTableFiles(table_file.table_id_, ids, files); ASSERT_EQ(files.size(), 0UL); } TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) { srand(time(0)); - DBMetaOptions options = GetOptions().meta_; + ms::engine::DBMetaOptions options = GetOptions().meta_; - int days_num = rand() % 100; + unsigned int seed = 1; + int days_num = rand_r(&seed) % 100; std::stringstream ss; ss << "days:" << days_num; - options.archive_conf_ = ArchiveConf("delete", ss.str()); - int mode = DBOptions::MODE::SINGLE; - meta::MySQLMetaImpl impl(options, mode); + options.archive_conf_ = ms::engine::ArchiveConf("delete", ss.str()); + int mode = ms::engine::DBOptions::MODE::SINGLE; + ms::engine::meta::MySQLMetaImpl impl(options, mode); auto table_id = "meta_test_table"; - meta::TableSchema table; + ms::engine::meta::TableSchema table; table.table_id_ = table_id; auto status = impl.CreateTable(table); - meta::TableFilesSchema files; - meta::TableFileSchema table_file; + ms::engine::meta::TableFilesSchema files; + ms::engine::meta::TableFileSchema table_file; table_file.table_id_ = table.table_id_; auto cnt = 100; - long ts = utils::GetMicroSecTimeStamp(); + int64_t ts = ms::engine::utils::GetMicroSecTimeStamp(); std::vector days; std::vector ids; - for (auto i=0; i file_types = { - (int) meta::TableFileSchema::NEW, + (int) ms::engine::meta::TableFileSchema::NEW, }; std::vector file_ids; status = impl.FilesByType(table_id, file_types, file_ids); @@ -184,32 +188,32 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) { } TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) { - DBMetaOptions options = GetOptions().meta_; + ms::engine::DBMetaOptions options = GetOptions().meta_; - options.archive_conf_ = ArchiveConf("delete", "disk:11"); - int mode = DBOptions::MODE::SINGLE; - auto impl = meta::MySQLMetaImpl(options, mode); + options.archive_conf_ = ms::engine::ArchiveConf("delete", "disk:11"); + int mode = ms::engine::DBOptions::MODE::SINGLE; + auto impl = ms::engine::meta::MySQLMetaImpl(options, mode); auto table_id = "meta_test_group"; - meta::TableSchema table; + ms::engine::meta::TableSchema table; table.table_id_ = table_id; auto status = impl.CreateTable(table); - meta::TableSchema table_schema; + ms::engine::meta::TableSchema table_schema; table_schema.table_id_ = ""; status = impl.CreateTable(table_schema); - meta::TableFilesSchema files; - meta::TableFileSchema table_file; + ms::engine::meta::TableFilesSchema files; + ms::engine::meta::TableFileSchema table_file; table_file.table_id_ = table.table_id_; auto cnt = 10; auto each_size = 2UL; std::vector ids; - for (auto i=0; i= 5) { - ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW); + ASSERT_EQ(file.file_type_, ms::engine::meta::TableFileSchema::NEW); } ++i; } @@ -236,7 +240,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) { TEST_F(MySqlMetaTest, TABLE_FILES_TEST) { auto table_id = "meta_test_group"; - meta::TableSchema table; + ms::engine::meta::TableSchema table; table.table_id_ = table_id; auto status = impl_->CreateTable(table); @@ -248,51 +252,51 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) { uint64_t to_index_files_cnt = 6; uint64_t index_files_cnt = 7; - meta::TableFileSchema table_file; + ms::engine::meta::TableFileSchema table_file; table_file.table_id_ = table.table_id_; - for (auto i=0; iCreateTableFile(table_file); - table_file.file_type_ = meta::TableFileSchema::NEW_MERGE; + table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_MERGE; status = impl_->UpdateTableFile(table_file); } - for (auto i=0; iCreateTableFile(table_file); - table_file.file_type_ = meta::TableFileSchema::NEW_INDEX; + table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW_INDEX; status = impl_->UpdateTableFile(table_file); } - for (auto i=0; iCreateTableFile(table_file); - table_file.file_type_ = meta::TableFileSchema::BACKUP; + table_file.file_type_ = ms::engine::meta::TableFileSchema::BACKUP; table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } - for (auto i=0; iCreateTableFile(table_file); - table_file.file_type_ = meta::TableFileSchema::NEW; + table_file.file_type_ = ms::engine::meta::TableFileSchema::NEW; status = impl_->UpdateTableFile(table_file); } - for (auto i=0; iCreateTableFile(table_file); - table_file.file_type_ = meta::TableFileSchema::RAW; + table_file.file_type_ = ms::engine::meta::TableFileSchema::RAW; table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } - for (auto i=0; iCreateTableFile(table_file); - table_file.file_type_ = meta::TableFileSchema::TO_INDEX; + table_file.file_type_ = ms::engine::meta::TableFileSchema::TO_INDEX; table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } - for (auto i=0; iCreateTableFile(table_file); - table_file.file_type_ = meta::TableFileSchema::INDEX; + table_file.file_type_ = ms::engine::meta::TableFileSchema::INDEX; table_file.row_count_ = 1; status = impl_->UpdateTableFile(table_file); } @@ -300,36 +304,36 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) { uint64_t total_row_count = 0; status = impl_->Count(table_id, total_row_count); ASSERT_TRUE(status.ok()); - ASSERT_EQ(total_row_count, raw_files_cnt+to_index_files_cnt+index_files_cnt); + ASSERT_EQ(total_row_count, raw_files_cnt + to_index_files_cnt + index_files_cnt); - meta::TableFilesSchema files; + ms::engine::meta::TableFilesSchema files; status = impl_->FilesToIndex(files); ASSERT_EQ(files.size(), to_index_files_cnt); - meta::DatePartionedTableFilesSchema dated_files; + ms::engine::meta::DatePartionedTableFilesSchema dated_files; status = impl_->FilesToMerge(table.table_id_, dated_files); ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt); status = impl_->FilesToIndex(files); ASSERT_EQ(files.size(), to_index_files_cnt); - meta::DatesT dates = {table_file.date_}; + ms::engine::meta::DatesT dates = {table_file.date_}; std::vector ids; status = impl_->FilesToSearch(table_id, ids, dates, dated_files); ASSERT_EQ(dated_files[table_file.date_].size(), - to_index_files_cnt+raw_files_cnt+index_files_cnt); + to_index_files_cnt + raw_files_cnt + index_files_cnt); - status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files); + status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files); ASSERT_EQ(dated_files[table_file.date_].size(), - to_index_files_cnt+raw_files_cnt+index_files_cnt); + to_index_files_cnt + raw_files_cnt + index_files_cnt); - status = impl_->FilesToSearch(table_id, ids, meta::DatesT(), dated_files); + status = impl_->FilesToSearch(table_id, ids, ms::engine::meta::DatesT(), dated_files); ASSERT_EQ(dated_files[table_file.date_].size(), - to_index_files_cnt+raw_files_cnt+index_files_cnt); + to_index_files_cnt + raw_files_cnt + index_files_cnt); ids.push_back(size_t(9999999999)); status = impl_->FilesToSearch(table_id, ids, dates, dated_files); - ASSERT_EQ(dated_files[table_file.date_].size(),0); + ASSERT_EQ(dated_files[table_file.date_].size(), 0); std::vector file_types; std::vector file_ids; @@ -338,19 +342,19 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) { ASSERT_FALSE(status.ok()); file_types = { - meta::TableFileSchema::NEW, - meta::TableFileSchema::NEW_MERGE, - meta::TableFileSchema::NEW_INDEX, - meta::TableFileSchema::TO_INDEX, - meta::TableFileSchema::INDEX, - meta::TableFileSchema::RAW, - meta::TableFileSchema::BACKUP, + ms::engine::meta::TableFileSchema::NEW, + ms::engine::meta::TableFileSchema::NEW_MERGE, + ms::engine::meta::TableFileSchema::NEW_INDEX, + ms::engine::meta::TableFileSchema::TO_INDEX, + ms::engine::meta::TableFileSchema::INDEX, + ms::engine::meta::TableFileSchema::RAW, + ms::engine::meta::TableFileSchema::BACKUP, }; status = impl_->FilesByType(table.table_id_, file_types, file_ids); ASSERT_TRUE(status.ok()); uint64_t total_cnt = new_index_files_cnt + new_merge_files_cnt + - backup_files_cnt + new_files_cnt + raw_files_cnt + - to_index_files_cnt + index_files_cnt; + backup_files_cnt + new_files_cnt + raw_files_cnt + + to_index_files_cnt + index_files_cnt; ASSERT_EQ(file_ids.size(), total_cnt); status = impl_->DeleteTableFiles(table_id); @@ -366,11 +370,11 @@ TEST_F(MySqlMetaTest, TABLE_FILES_TEST) { TEST_F(MySqlMetaTest, INDEX_TEST) { auto table_id = "index_test"; - meta::TableSchema table; + ms::engine::meta::TableSchema table; table.table_id_ = table_id; auto status = impl_->CreateTable(table); - TableIndex index; + ms::engine::TableIndex index; index.metric_type_ = 2; index.nlist_ = 1234; index.engine_type_ = 3; @@ -381,12 +385,12 @@ TEST_F(MySqlMetaTest, INDEX_TEST) { status = impl_->UpdateTableFlag(table_id, flag); ASSERT_TRUE(status.ok()); - engine::meta::TableSchema table_info; + ms::engine::meta::TableSchema table_info; table_info.table_id_ = table_id; status = impl_->DescribeTable(table_info); ASSERT_EQ(table_info.flag_, flag); - TableIndex index_out; + ms::engine::TableIndex index_out; status = impl_->DescribeTableIndex(table_id, index_out); ASSERT_EQ(index_out.metric_type_, index.metric_type_); ASSERT_EQ(index_out.nlist_, index.nlist_); diff --git a/cpp/unittest/db/search_test.cpp b/cpp/unittest/db/search_test.cpp index 33917ea1b7..d0d55a11c6 100644 --- a/cpp/unittest/db/search_test.cpp +++ b/cpp/unittest/db/search_test.cpp @@ -22,53 +22,55 @@ #include "scheduler/task/SearchTask.h" #include "utils/TimeRecorder.h" -using namespace zilliz::milvus; - namespace { +namespace ms = zilliz::milvus; + static constexpr uint64_t NQ = 15; static constexpr uint64_t TOP_K = 64; -void BuildResult(uint64_t nq, - uint64_t topk, - bool ascending, - std::vector &output_ids, - std::vector &output_distence) { +void +BuildResult(uint64_t nq, + uint64_t topk, + bool ascending, + std::vector &output_ids, + std::vector &output_distence) { output_ids.clear(); - output_ids.resize(nq*topk); + output_ids.resize(nq * topk); output_distence.clear(); - output_distence.resize(nq*topk); + output_distence.resize(nq * topk); - for(uint64_t i = 0; i < nq; i++) { - for(uint64_t j = 0; j < topk; j++) { - output_ids[i * topk + j] = (long)(drand48()*100000); + for (uint64_t i = 0; i < nq; i++) { + for (uint64_t j = 0; j < topk; j++) { + output_ids[i * topk + j] = (int64_t) (drand48() * 100000); output_distence[i * topk + j] = ascending ? (j + drand48()) : ((topk - j) + drand48()); } } } -void CheckResult(const scheduler::Id2DistanceMap& src_1, - const scheduler::Id2DistanceMap& src_2, - const scheduler::Id2DistanceMap& target, - bool ascending) { - for(uint64_t i = 0; i < target.size() - 1; i++) { - if(ascending) { +void +CheckResult(const ms::scheduler::Id2DistanceMap &src_1, + const ms::scheduler::Id2DistanceMap &src_2, + const ms::scheduler::Id2DistanceMap &target, + bool ascending) { + for (uint64_t i = 0; i < target.size() - 1; i++) { + if (ascending) { ASSERT_LE(target[i].second, target[i + 1].second); } else { ASSERT_GE(target[i].second, target[i + 1].second); } } - using ID2DistMap = std::map; + using ID2DistMap = std::map; ID2DistMap src_map_1, src_map_2; - for(const auto& pair : src_1) { + for (const auto &pair : src_1) { src_map_1.insert(pair); } - for(const auto& pair : src_2) { + for (const auto &pair : src_2) { src_map_2.insert(pair); } - for(const auto& pair : target) { + for (const auto &pair : target) { ASSERT_TRUE(src_map_1.find(pair.first) != src_map_1.end() || src_map_2.find(pair.first) != src_map_2.end()); float dist = src_map_1.find(pair.first) != src_map_1.end() ? src_map_1[pair.first] : src_map_2[pair.first]; @@ -76,40 +78,42 @@ void CheckResult(const scheduler::Id2DistanceMap& src_1, } } -void CheckCluster(const std::vector& target_ids, - const std::vector& target_distence, - const scheduler::ResultSet& src_result, - int64_t nq, - int64_t topk) { +void +CheckCluster(const std::vector &target_ids, + const std::vector &target_distence, + const ms::scheduler::ResultSet &src_result, + int64_t nq, + int64_t topk) { ASSERT_EQ(src_result.size(), nq); - for(int64_t i = 0; i < nq; i++) { - auto& res = src_result[i]; + for (int64_t i = 0; i < nq; i++) { + auto &res = src_result[i]; ASSERT_EQ(res.size(), topk); - if(res.empty()) { + if (res.empty()) { continue; } - ASSERT_EQ(res[0].first, target_ids[i*topk]); - ASSERT_EQ(res[topk - 1].first, target_ids[i*topk + topk - 1]); + ASSERT_EQ(res[0].first, target_ids[i * topk]); + ASSERT_EQ(res[topk - 1].first, target_ids[i * topk + topk - 1]); } } -void CheckTopkResult(const scheduler::ResultSet& src_result, - bool ascending, - int64_t nq, - int64_t topk) { +void +CheckTopkResult(const ms::scheduler::ResultSet &src_result, + bool ascending, + int64_t nq, + int64_t topk) { ASSERT_EQ(src_result.size(), nq); - for(int64_t i = 0; i < nq; i++) { - auto& res = src_result[i]; + for (int64_t i = 0; i < nq; i++) { + auto &res = src_result[i]; ASSERT_EQ(res.size(), topk); - if(res.empty()) { + if (res.empty()) { continue; } - for(int64_t k = 0; k < topk - 1; k++) { - if(ascending) { + for (int64_t k = 0; k < topk - 1; k++) { + if (ascending) { ASSERT_LE(res[k].second, res[k + 1].second); } else { ASSERT_GE(res[k].second, res[k + 1].second); @@ -118,87 +122,87 @@ void CheckTopkResult(const scheduler::ResultSet& src_result, } } -} +} // namespace TEST(DBSearchTest, TOPK_TEST) { bool ascending = true; - std::vector target_ids; + std::vector target_ids; std::vector target_distence; - scheduler::ResultSet src_result; - auto status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result); + ms::scheduler::ResultSet src_result; + auto status = ms::scheduler::XSearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result); ASSERT_FALSE(status.ok()); ASSERT_TRUE(src_result.empty()); BuildResult(NQ, TOP_K, ascending, target_ids, target_distence); - status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result); + status = ms::scheduler::XSearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result); ASSERT_TRUE(status.ok()); ASSERT_EQ(src_result.size(), NQ); - scheduler::ResultSet target_result; - status = scheduler::XSearchTask::TopkResult(target_result, TOP_K, ascending, target_result); + ms::scheduler::ResultSet target_result; + status = ms::scheduler::XSearchTask::TopkResult(target_result, TOP_K, ascending, target_result); ASSERT_TRUE(status.ok()); - status = scheduler::XSearchTask::TopkResult(target_result, TOP_K, ascending, src_result); + status = ms::scheduler::XSearchTask::TopkResult(target_result, TOP_K, ascending, src_result); ASSERT_FALSE(status.ok()); - status = scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result); + status = ms::scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result); ASSERT_TRUE(status.ok()); ASSERT_TRUE(src_result.empty()); ASSERT_EQ(target_result.size(), NQ); - std::vector src_ids; + std::vector src_ids; std::vector src_distence; uint64_t wrong_topk = TOP_K - 10; BuildResult(NQ, wrong_topk, ascending, src_ids, src_distence); - status = scheduler::XSearchTask::ClusterResult(src_ids, src_distence, NQ, wrong_topk, src_result); + status = ms::scheduler::XSearchTask::ClusterResult(src_ids, src_distence, NQ, wrong_topk, src_result); ASSERT_TRUE(status.ok()); - status = scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result); + status = ms::scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result); ASSERT_TRUE(status.ok()); - for(uint64_t i = 0; i < NQ; i++) { + for (uint64_t i = 0; i < NQ; i++) { ASSERT_EQ(target_result[i].size(), TOP_K); } wrong_topk = TOP_K + 10; BuildResult(NQ, wrong_topk, ascending, src_ids, src_distence); - status = scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result); + status = ms::scheduler::XSearchTask::TopkResult(src_result, TOP_K, ascending, target_result); ASSERT_TRUE(status.ok()); - for(uint64_t i = 0; i < NQ; i++) { + for (uint64_t i = 0; i < NQ; i++) { ASSERT_EQ(target_result[i].size(), TOP_K); } } TEST(DBSearchTest, MERGE_TEST) { bool ascending = true; - std::vector target_ids; + std::vector target_ids; std::vector target_distence; - std::vector src_ids; + std::vector src_ids; std::vector src_distence; - scheduler::ResultSet src_result, target_result; + ms::scheduler::ResultSet src_result, target_result; uint64_t src_count = 5, target_count = 8; BuildResult(1, src_count, ascending, src_ids, src_distence); BuildResult(1, target_count, ascending, target_ids, target_distence); - auto status = scheduler::XSearchTask::ClusterResult(src_ids, src_distence, 1, src_count, src_result); + auto status = ms::scheduler::XSearchTask::ClusterResult(src_ids, src_distence, 1, src_count, src_result); ASSERT_TRUE(status.ok()); - status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, 1, target_count, target_result); + status = ms::scheduler::XSearchTask::ClusterResult(target_ids, target_distence, 1, target_count, target_result); ASSERT_TRUE(status.ok()); { - scheduler::Id2DistanceMap src = src_result[0]; - scheduler::Id2DistanceMap target = target_result[0]; - status = scheduler::XSearchTask::MergeResult(src, target, 10, ascending); + ms::scheduler::Id2DistanceMap src = src_result[0]; + ms::scheduler::Id2DistanceMap target = target_result[0]; + status = ms::scheduler::XSearchTask::MergeResult(src, target, 10, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), 10); CheckResult(src_result[0], target_result[0], target, ascending); } { - scheduler::Id2DistanceMap src = src_result[0]; - scheduler::Id2DistanceMap target; - status = scheduler::XSearchTask::MergeResult(src, target, 10, ascending); + ms::scheduler::Id2DistanceMap src = src_result[0]; + ms::scheduler::Id2DistanceMap target; + status = ms::scheduler::XSearchTask::MergeResult(src, target, 10, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), src_count); ASSERT_TRUE(src.empty()); @@ -206,18 +210,18 @@ TEST(DBSearchTest, MERGE_TEST) { } { - scheduler::Id2DistanceMap src = src_result[0]; - scheduler::Id2DistanceMap target = target_result[0]; - status = scheduler::XSearchTask::MergeResult(src, target, 30, ascending); + ms::scheduler::Id2DistanceMap src = src_result[0]; + ms::scheduler::Id2DistanceMap target = target_result[0]; + status = ms::scheduler::XSearchTask::MergeResult(src, target, 30, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), src_count + target_count); CheckResult(src_result[0], target_result[0], target, ascending); } { - scheduler::Id2DistanceMap target = src_result[0]; - scheduler::Id2DistanceMap src = target_result[0]; - status = scheduler::XSearchTask::MergeResult(src, target, 30, ascending); + ms::scheduler::Id2DistanceMap target = src_result[0]; + ms::scheduler::Id2DistanceMap src = target_result[0]; + status = ms::scheduler::XSearchTask::MergeResult(src, target, 30, ascending); ASSERT_TRUE(status.ok()); ASSERT_EQ(target.size(), src_count + target_count); CheckResult(src_result[0], target_result[0], target, ascending); @@ -226,17 +230,17 @@ TEST(DBSearchTest, MERGE_TEST) { TEST(DBSearchTest, PARALLEL_CLUSTER_TEST) { bool ascending = true; - std::vector target_ids; + std::vector target_ids; std::vector target_distence; - scheduler::ResultSet src_result; + ms::scheduler::ResultSet src_result; auto DoCluster = [&](int64_t nq, int64_t topk) { - TimeRecorder rc("DoCluster"); + ms::TimeRecorder rc("DoCluster"); src_result.clear(); BuildResult(nq, topk, ascending, target_ids, target_distence); rc.RecordSection("build id/dietance map"); - auto status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result); + auto status = ms::scheduler::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result); ASSERT_TRUE(status.ok()); ASSERT_EQ(src_result.size(), nq); @@ -257,29 +261,33 @@ TEST(DBSearchTest, PARALLEL_CLUSTER_TEST) { } TEST(DBSearchTest, PARALLEL_TOPK_TEST) { - std::vector target_ids; + std::vector target_ids; std::vector target_distence; - scheduler::ResultSet src_result; + ms::scheduler::ResultSet src_result; - std::vector insufficient_ids; + std::vector insufficient_ids; std::vector insufficient_distence; - scheduler::ResultSet insufficient_result; + ms::scheduler::ResultSet insufficient_result; - auto DoTopk = [&](int64_t nq, int64_t topk,int64_t insufficient_topk, bool ascending) { + auto DoTopk = [&](int64_t nq, int64_t topk, int64_t insufficient_topk, bool ascending) { src_result.clear(); insufficient_result.clear(); - TimeRecorder rc("DoCluster"); + ms::TimeRecorder rc("DoCluster"); BuildResult(nq, topk, ascending, target_ids, target_distence); - auto status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result); + auto status = ms::scheduler::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result); rc.RecordSection("cluster result"); BuildResult(nq, insufficient_topk, ascending, insufficient_ids, insufficient_distence); - status = scheduler::XSearchTask::ClusterResult(target_ids, target_distence, nq, insufficient_topk, insufficient_result); + status = ms::scheduler::XSearchTask::ClusterResult(target_ids, + target_distence, + nq, + insufficient_topk, + insufficient_result); rc.RecordSection("cluster result"); - scheduler::XSearchTask::TopkResult(insufficient_result, topk, ascending, src_result); + ms::scheduler::XSearchTask::TopkResult(insufficient_result, topk, ascending, src_result); ASSERT_TRUE(status.ok()); rc.RecordSection("topk"); @@ -292,4 +300,4 @@ TEST(DBSearchTest, PARALLEL_TOPK_TEST) { // DoTopk(9987, 12, 10, false); // DoTopk(77777, 1000, 1, false); // DoTopk(5432, 8899, 8899, true); -} \ No newline at end of file +} diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 08e3c8f090..2ad7c4f743 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -18,9 +18,11 @@ #include #include +#include +#include #include -#include "utils.h" +#include "db/utils.h" #include "cache/GpuCacheMgr.h" #include "cache/CpuCacheMgr.h" #include "db/DBFactory.h" @@ -29,83 +31,98 @@ INITIALIZE_EASYLOGGINGPP -using namespace zilliz::milvus; +namespace { -static std::string uri; +namespace ms = zilliz::milvus; class DBTestEnvironment : public ::testing::Environment { -public: + public: + explicit DBTestEnvironment(const std::string& uri) + : uri_(uri) { + } -// explicit DBTestEnvironment(std::string uri) : uri_(uri) {} - - static std::string getURI() { - return uri; + std::string getURI() const { + return uri_; } void SetUp() override { getURI(); } + private: + std::string uri_; }; +DBTestEnvironment* test_env = nullptr; + +} // namespace + + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// -void BaseTest::InitLog() { +void +BaseTest::InitLog() { el::Configurations defaultConf; defaultConf.setToDefault(); defaultConf.set(el::Level::Debug, - el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)"); + el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)"); el::Loggers::reconfigureLogger("default", defaultConf); } -void BaseTest::SetUp() { +void +BaseTest::SetUp() { InitLog(); - zilliz::knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(0, 1024*1024*200, 1024*1024*300, 2); + zilliz::knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(0, 1024 * 1024 * 200, 1024 * 1024 * 300, 2); } -void BaseTest::TearDown() { +void +BaseTest::TearDown() { zilliz::milvus::cache::CpuCacheMgr::GetInstance()->ClearCache(); zilliz::milvus::cache::GpuCacheMgr::GetInstance(0)->ClearCache(); zilliz::knowhere::FaissGpuResourceMgr::GetInstance().Free(); } -engine::DBOptions BaseTest::GetOptions() { - auto options = engine::DBFactory::BuildOption(); +ms::engine::DBOptions +BaseTest::GetOptions() { + auto options = ms::engine::DBFactory::BuildOption(); options.meta_.path_ = "/tmp/milvus_test"; options.meta_.backend_uri_ = "sqlite://:@:/"; return options; } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// -void DBTest::SetUp() { +void +DBTest::SetUp() { BaseTest::SetUp(); - auto res_mgr = scheduler::ResMgrInst::GetInstance(); + auto res_mgr = ms::scheduler::ResMgrInst::GetInstance(); res_mgr->Clear(); - res_mgr->Add(scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false)); - res_mgr->Add(scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, false)); - res_mgr->Add(scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true)); + res_mgr->Add(ms::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false)); + res_mgr->Add(ms::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, false)); + res_mgr->Add(ms::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true)); - auto default_conn = scheduler::Connection("IO", 500.0); - auto PCIE = scheduler::Connection("IO", 11000.0); + auto default_conn = ms::scheduler::Connection("IO", 500.0); + auto PCIE = ms::scheduler::Connection("IO", 11000.0); res_mgr->Connect("disk", "cpu", default_conn); res_mgr->Connect("cpu", "gtx1660", PCIE); res_mgr->Start(); - scheduler::SchedInst::GetInstance()->Start(); + ms::scheduler::SchedInst::GetInstance()->Start(); - scheduler::JobMgrInst::GetInstance()->Start(); + ms::scheduler::JobMgrInst::GetInstance()->Start(); auto options = GetOptions(); - db_ = engine::DBFactory::Build(options); + db_ = ms::engine::DBFactory::Build(options); } -void DBTest::TearDown() { +void +DBTest::TearDown() { db_->Stop(); db_->DropAll(); - scheduler::JobMgrInst::GetInstance()->Stop(); - scheduler::SchedInst::GetInstance()->Stop(); - scheduler::ResMgrInst::GetInstance()->Stop(); + ms::scheduler::JobMgrInst::GetInstance()->Stop(); + ms::scheduler::SchedInst::GetInstance()->Stop(); + ms::scheduler::ResMgrInst::GetInstance()->Stop(); BaseTest::TearDown(); @@ -114,23 +131,26 @@ void DBTest::TearDown() { } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// -engine::DBOptions DBTest2::GetOptions() { - auto options = engine::DBFactory::BuildOption(); +ms::engine::DBOptions +DBTest2::GetOptions() { + auto options = ms::engine::DBFactory::BuildOption(); options.meta_.path_ = "/tmp/milvus_test"; - options.meta_.archive_conf_ = engine::ArchiveConf("delete", "disk:1"); + options.meta_.archive_conf_ = ms::engine::ArchiveConf("delete", "disk:1"); options.meta_.backend_uri_ = "sqlite://:@:/"; return options; } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// -void MetaTest::SetUp() { +void +MetaTest::SetUp() { BaseTest::SetUp(); auto options = GetOptions(); - impl_ = std::make_shared(options.meta_); + impl_ = std::make_shared(options.meta_); } -void MetaTest::TearDown() { +void +MetaTest::TearDown() { impl_->DropAll(); BaseTest::TearDown(); @@ -140,27 +160,26 @@ void MetaTest::TearDown() { } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// -engine::DBOptions MySqlDBTest::GetOptions() { - auto options = engine::DBFactory::BuildOption(); +ms::engine::DBOptions +MySqlDBTest::GetOptions() { + auto options = ms::engine::DBFactory::BuildOption(); options.meta_.path_ = "/tmp/milvus_test"; - options.meta_.backend_uri_ = DBTestEnvironment::getURI(); - - if(options.meta_.backend_uri_.empty()) { - options.meta_.backend_uri_ = "mysql://root:Fantast1c@192.168.1.194:3306/"; - } + options.meta_.backend_uri_ = test_env->getURI(); return options; } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// -void MySqlMetaTest::SetUp() { +void +MySqlMetaTest::SetUp() { BaseTest::SetUp(); auto options = GetOptions(); - impl_ = std::make_shared(options.meta_, options.mode_); + impl_ = std::make_shared(options.meta_, options.mode_); } -void MySqlMetaTest::TearDown() { +void +MySqlMetaTest::TearDown() { impl_->DropAll(); BaseTest::TearDown(); @@ -169,29 +188,26 @@ void MySqlMetaTest::TearDown() { boost::filesystem::remove_all(options.meta_.path_); } -engine::DBOptions MySqlMetaTest::GetOptions() { - auto options = engine::DBFactory::BuildOption(); +ms::engine::DBOptions +MySqlMetaTest::GetOptions() { + auto options = ms::engine::DBFactory::BuildOption(); options.meta_.path_ = "/tmp/milvus_test"; - options.meta_.backend_uri_ = DBTestEnvironment::getURI(); - - if(options.meta_.backend_uri_.empty()) { - options.meta_.backend_uri_ = "mysql://root:Fantast1c@192.168.1.194:3306/"; - } + options.meta_.backend_uri_ = test_env->getURI(); return options; } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// -int main(int argc, char **argv) { +int +main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); + + std::string uri; if (argc > 1) { uri = argv[1]; } -// if(uri.empty()) { -// uri = "mysql://root:Fantast1c@192.168.1.194:3306/"; -// } -// std::cout << uri << std::endl; - ::testing::AddGlobalTestEnvironment(new DBTestEnvironment); + test_env = new DBTestEnvironment(uri); + ::testing::AddGlobalTestEnvironment(test_env); return RUN_ALL_TESTS(); } diff --git a/cpp/unittest/db/utils.h b/cpp/unittest/db/utils.h index 7a1320ef6b..dee9dcf843 100644 --- a/cpp/unittest/db/utils.h +++ b/cpp/unittest/db/utils.h @@ -20,7 +20,7 @@ #include #include -//#include +#include #include "db/DB.h" #include "db/meta/SqliteMetaImpl.h" @@ -28,7 +28,6 @@ #include "scheduler/SchedInst.h" #include "scheduler/ResourceFactory.h" - #define TIMING #ifdef TIMING @@ -36,8 +35,7 @@ #define START_TIMER start = std::chrono::high_resolution_clock::now(); #define STOP_TIMER(name) LOG(DEBUG) << "RUNTIME of " << name << ": " << \ std::chrono::duration_cast( \ - std::chrono::high_resolution_clock::now()-start \ - ).count() << " ms "; + std::chrono::high_resolution_clock::now()-start).count() << " ms "; #else #define INIT_TIMER #define START_TIMER @@ -45,11 +43,11 @@ #endif class BaseTest : public ::testing::Test { -protected: + protected: void InitLog(); - virtual void SetUp() override; - virtual void TearDown() override; + void SetUp() override; + void TearDown() override; virtual zilliz::milvus::engine::DBOptions GetOptions(); }; @@ -58,14 +56,14 @@ class DBTest : public BaseTest { protected: zilliz::milvus::engine::DBPtr db_; - virtual void SetUp() override; - virtual void TearDown() override; + void SetUp() override; + void TearDown() override; }; ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DBTest2 : public DBTest { protected: - virtual zilliz::milvus::engine::DBOptions GetOptions() override; + zilliz::milvus::engine::DBOptions GetOptions() override; }; ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -77,14 +75,14 @@ class MetaTest : public BaseTest { protected: std::shared_ptr impl_; - virtual void SetUp() override; - virtual void TearDown() override; + void SetUp() override; + void TearDown() override; }; ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// class MySqlDBTest : public DBTest { -protected: - zilliz::milvus::engine::DBOptions GetOptions(); + protected: + zilliz::milvus::engine::DBOptions GetOptions() override; }; ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -92,12 +90,11 @@ class MySqlMetaTest : public BaseTest { protected: std::shared_ptr impl_; - virtual void SetUp() override; - virtual void TearDown() override; - zilliz::milvus::engine::DBOptions GetOptions(); + void SetUp() override; + void TearDown() override; + zilliz::milvus::engine::DBOptions GetOptions() override; }; - ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// class MemManagerTest : public MetaTest { }; diff --git a/cpp/unittest/main.cpp b/cpp/unittest/main.cpp index 7a28bbf45f..d17cf9da58 100644 --- a/cpp/unittest/main.cpp +++ b/cpp/unittest/main.cpp @@ -19,13 +19,11 @@ #include #include "utils/easylogging++.h" -#include "utils/CommonUtil.h" INITIALIZE_EASYLOGGINGPP -using namespace zilliz::milvus; - -int main(int argc, char **argv) { +int +main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/cpp/unittest/metrics/metricbase_test.cpp b/cpp/unittest/metrics/metricbase_test.cpp index 31766403be..7f77312480 100644 --- a/cpp/unittest/metrics/metricbase_test.cpp +++ b/cpp/unittest/metrics/metricbase_test.cpp @@ -21,12 +21,16 @@ #include #include -using namespace zilliz::milvus; +namespace { -TEST(MetricbaseTest, METRICBASE_TEST){ - server::MetricsBase instance = server::MetricsBase::GetInstance(); +namespace ms = zilliz::milvus; + +} // namespace + +TEST(MetricbaseTest, METRICBASE_TEST) { + ms::server::MetricsBase instance = ms::server::MetricsBase::GetInstance(); instance.Init(); - server::SystemInfo::GetInstance().Init(); + ms::server::SystemInfo::GetInstance().Init(); instance.AddVectorsSuccessTotalIncrement(); instance.AddVectorsFailTotalIncrement(); instance.AddVectorsDurationHistogramOberve(1.0); @@ -60,10 +64,10 @@ TEST(MetricbaseTest, METRICBASE_TEST){ instance.QueryResponsePerSecondGaugeSet(1.0); instance.GPUPercentGaugeSet(); instance.GPUMemoryUsageGaugeSet(); - instance.AddVectorsPerSecondGaugeSet(1,1,1); + instance.AddVectorsPerSecondGaugeSet(1, 1, 1); instance.QueryIndexTypePerSecondSet("IVF", 1.0); instance.ConnectionGaugeIncrement(); instance.ConnectionGaugeDecrement(); instance.KeepingAliveCounterIncrement(); instance.OctetsSet(); -} \ No newline at end of file +} diff --git a/cpp/unittest/metrics/metrics_test.cpp b/cpp/unittest/metrics/metrics_test.cpp index 14c1ead3f3..ec950fedcc 100644 --- a/cpp/unittest/metrics/metrics_test.cpp +++ b/cpp/unittest/metrics/metrics_test.cpp @@ -15,75 +15,73 @@ // specific language governing permissions and limitations // under the License. -#include #include #include #include #include #include #include -//#include "prometheus/registry.h" -//#include "prometheus/exposer.h" -#include -#include +#include "cache/CpuCacheMgr.h" +#include "server/Config.h" #include "metrics/Metrics.h" -#include "utils.h" +#include "metrics/utils.h" #include "db/DB.h" #include "db/meta/SqliteMetaImpl.h" +namespace { -using namespace zilliz::milvus; +namespace ms = zilliz::milvus; +} // namespace TEST_F(MetricTest, METRIC_TEST) { - server::Config::GetInstance().SetMetricConfigCollector("zabbix"); - server::Metrics::GetInstance(); - server::Config::GetInstance().SetMetricConfigCollector("prometheus"); - server::Metrics::GetInstance(); + ms::server::Config::GetInstance().SetMetricConfigCollector("zabbix"); + ms::server::Metrics::GetInstance(); + ms::server::Config::GetInstance().SetMetricConfigCollector("prometheus"); + ms::server::Metrics::GetInstance(); - server::SystemInfo::GetInstance().Init(); + ms::server::SystemInfo::GetInstance().Init(); // server::Metrics::GetInstance().Init(); // server::Metrics::GetInstance().exposer_ptr()->RegisterCollectable(server::Metrics::GetInstance().registry_ptr()); - server::Metrics::GetInstance().Init(); + ms::server::Metrics::GetInstance().Init(); // server::PrometheusMetrics::GetInstance().exposer_ptr()->RegisterCollectable(server::PrometheusMetrics::GetInstance().registry_ptr()); - zilliz::milvus::cache::CpuCacheMgr::GetInstance()->SetCapacity(1UL*1024*1024*1024); - std::cout<CacheCapacity()<SetCapacity(1UL * 1024 * 1024 * 1024); + std::cout << zilliz::milvus::cache::CpuCacheMgr::GetInstance()->CacheCapacity() << std::endl; - static const char* group_name = "test_group"; + static const char *group_name = "test_group"; static const int group_dim = 256; - engine::meta::TableSchema group_info; + ms::engine::meta::TableSchema group_info; group_info.dimension_ = group_dim; group_info.table_id_ = group_name; auto stat = db_->CreateTable(group_info); - engine::meta::TableSchema group_info_get; + ms::engine::meta::TableSchema group_info_get; group_info_get.table_id_ = group_name; stat = db_->DescribeTable(group_info_get); - - engine::IDNumbers vector_ids; - engine::IDNumbers target_ids; + ms::engine::IDNumbers vector_ids; + ms::engine::IDNumbers target_ids; int d = 256; int nb = 50; float *xb = new float[d * nb]; - for(int i = 0; i < nb; i++) { - for(int j = 0; j < d; j++) xb[d * i + j] = drand48(); + for (int i = 0; i < nb; i++) { + for (int j = 0; j < d; j++) xb[d * i + j] = drand48(); xb[d * i] += i / 2000.; } int qb = 5; float *qxb = new float[d * qb]; - for(int i = 0; i < qb; i++) { - for(int j = 0; j < d; j++) qxb[d * i + j] = drand48(); + for (int i = 0; i < qb; i++) { + for (int j = 0; j < d; j++) qxb[d * i + j] = drand48(); qxb[d * i] += i / 2000.; } std::thread search([&]() { - engine::QueryResults results; + ms::engine::QueryResults results; int k = 10; std::this_thread::sleep_for(std::chrono::seconds(2)); @@ -92,23 +90,23 @@ TEST_F(MetricTest, METRIC_TEST) { uint64_t count = 0; uint64_t prev_count = 0; - for (auto j=0; j<10; ++j) { + for (auto j = 0; j < 10; ++j) { ss.str(""); db_->Size(count); prev_count = count; START_TIMER; // stat = db_->Query(group_name, k, qb, qxb, results); - ss << "Search " << j << " With Size " << (float)(count*group_dim*sizeof(float))/(1024*1024) << " M"; + ss << "Search " << j << " With Size " << (float) (count * group_dim * sizeof(float)) / (1024 * 1024) + << " M"; - for (auto k=0; k= prev_count); std::this_thread::sleep_for(std::chrono::seconds(1)); @@ -117,8 +115,8 @@ TEST_F(MetricTest, METRIC_TEST) { int loop = 10000; - for (auto i=0; iInsertVectors(group_name, qb, qxb, target_ids); ASSERT_EQ(target_ids.size(), qb); } else { @@ -129,37 +127,37 @@ TEST_F(MetricTest, METRIC_TEST) { search.join(); - delete [] xb; - delete [] qxb; -}; + delete[] xb; + delete[] qxb; +} -TEST_F(MetricTest, COLLECTOR_METRICS_TEST){ - auto status = Status::OK(); - server::CollectInsertMetrics insert_metrics0(0, status); - status = Status(DB_ERROR, "error"); - server::CollectInsertMetrics insert_metrics1(0, status); +TEST_F(MetricTest, COLLECTOR_METRICS_TEST) { + auto status = ms::Status::OK(); + ms::server::CollectInsertMetrics insert_metrics0(0, status); + status = ms::Status(ms::DB_ERROR, "error"); + ms::server::CollectInsertMetrics insert_metrics1(0, status); - server::CollectQueryMetrics query_metrics(10); + ms::server::CollectQueryMetrics query_metrics(10); - server::CollectMergeFilesMetrics merge_metrics(); + ms::server::CollectMergeFilesMetrics merge_metrics(); - server::CollectBuildIndexMetrics build_index_metrics(); + ms::server::CollectBuildIndexMetrics build_index_metrics(); - server::CollectExecutionEngineMetrics execution_metrics(10); + ms::server::CollectExecutionEngineMetrics execution_metrics(10); - server::CollectSerializeMetrics serialize_metrics(10); + ms::server::CollectSerializeMetrics serialize_metrics(10); - server::CollectAddMetrics add_metrics(10, 128); + ms::server::CollectAddMetrics add_metrics(10, 128); - server::CollectDurationMetrics duration_metrics_raw(engine::meta::TableFileSchema::RAW); - server::CollectDurationMetrics duration_metrics_index(engine::meta::TableFileSchema::TO_INDEX); - server::CollectDurationMetrics duration_metrics_delete(engine::meta::TableFileSchema::TO_DELETE); + ms::server::CollectDurationMetrics duration_metrics_raw(ms::engine::meta::TableFileSchema::RAW); + ms::server::CollectDurationMetrics duration_metrics_index(ms::engine::meta::TableFileSchema::TO_INDEX); + ms::server::CollectDurationMetrics duration_metrics_delete(ms::engine::meta::TableFileSchema::TO_DELETE); - server::CollectSearchTaskMetrics search_metrics_raw(engine::meta::TableFileSchema::RAW); - server::CollectSearchTaskMetrics search_metrics_index(engine::meta::TableFileSchema::TO_INDEX); - server::CollectSearchTaskMetrics search_metrics_delete(engine::meta::TableFileSchema::TO_DELETE); + ms::server::CollectSearchTaskMetrics search_metrics_raw(ms::engine::meta::TableFileSchema::RAW); + ms::server::CollectSearchTaskMetrics search_metrics_index(ms::engine::meta::TableFileSchema::TO_INDEX); + ms::server::CollectSearchTaskMetrics search_metrics_delete(ms::engine::meta::TableFileSchema::TO_DELETE); - server::MetricCollector metric_collector(); + ms::server::MetricCollector metric_collector(); } diff --git a/cpp/unittest/metrics/prometheus_test.cpp b/cpp/unittest/metrics/prometheus_test.cpp index a634a6ff9c..4847e61627 100644 --- a/cpp/unittest/metrics/prometheus_test.cpp +++ b/cpp/unittest/metrics/prometheus_test.cpp @@ -22,15 +22,19 @@ #include #include -using namespace zilliz::milvus; +namespace { -TEST(PrometheusTest, PROMETHEUS_TEST){ - server::Config::GetInstance().SetMetricConfigEnableMonitor("on"); +namespace ms = zilliz::milvus; - server::PrometheusMetrics instance = server::PrometheusMetrics::GetInstance(); +} // namespace + +TEST(PrometheusTest, PROMETHEUS_TEST) { + ms::server::Config::GetInstance().SetMetricConfigEnableMonitor("on"); + + ms::server::PrometheusMetrics instance = ms::server::PrometheusMetrics::GetInstance(); instance.Init(); instance.SetStartup(true); - server::SystemInfo::GetInstance().Init(); + ms::server::SystemInfo::GetInstance().Init(); instance.AddVectorsSuccessTotalIncrement(); instance.AddVectorsFailTotalIncrement(); instance.AddVectorsDurationHistogramOberve(1.0); @@ -64,7 +68,7 @@ TEST(PrometheusTest, PROMETHEUS_TEST){ instance.QueryResponsePerSecondGaugeSet(1.0); instance.GPUPercentGaugeSet(); instance.GPUMemoryUsageGaugeSet(); - instance.AddVectorsPerSecondGaugeSet(1,1,1); + instance.AddVectorsPerSecondGaugeSet(1, 1, 1); instance.QueryIndexTypePerSecondSet("IVF", 1.0); instance.QueryIndexTypePerSecondSet("IDMap", 1.0); instance.ConnectionGaugeIncrement(); @@ -76,10 +80,9 @@ TEST(PrometheusTest, PROMETHEUS_TEST){ instance.GPUTemperature(); instance.CPUTemperature(); - server::Config::GetInstance().SetMetricConfigEnableMonitor("off"); + ms::server::Config::GetInstance().SetMetricConfigEnableMonitor("off"); instance.Init(); instance.CPUCoreUsagePercentSet(); instance.GPUTemperature(); instance.CPUTemperature(); - -} \ No newline at end of file +} diff --git a/cpp/unittest/metrics/utils.cpp b/cpp/unittest/metrics/utils.cpp index 8d59a1e9c2..8275222766 100644 --- a/cpp/unittest/metrics/utils.cpp +++ b/cpp/unittest/metrics/utils.cpp @@ -18,32 +18,38 @@ #include #include +#include #include -#include "utils.h" +#include "metrics/utils.h" #include "db/DBFactory.h" INITIALIZE_EASYLOGGINGPP -using namespace zilliz::milvus; +namespace { -static std::string uri; +namespace ms = zilliz::milvus; class DBTestEnvironment : public ::testing::Environment { -public: + public: + explicit DBTestEnvironment(const std::string& uri) : uri_(uri) {} -// explicit DBTestEnvironment(std::string uri) : uri_(uri) {} - - static std::string getURI() { - return uri; + std::string getURI() const { + return uri_; } void SetUp() override { getURI(); } + private: + std::string uri_; }; +DBTestEnvironment* test_env = nullptr; + +} // namespace + void MetricTest::InitLog() { el::Configurations defaultConf; defaultConf.setToDefault(); @@ -52,8 +58,8 @@ void MetricTest::InitLog() { el::Loggers::reconfigureLogger("default", defaultConf); } -engine::DBOptions MetricTest::GetOptions() { - auto options = engine::DBFactory::BuildOption(); +ms::engine::DBOptions MetricTest::GetOptions() { + auto options = ms::engine::DBFactory::BuildOption(); options.meta_.path_ = "/tmp/milvus_test"; options.meta_.backend_uri_ = "sqlite://:@:/"; return options; @@ -62,7 +68,7 @@ engine::DBOptions MetricTest::GetOptions() { void MetricTest::SetUp() { InitLog(); auto options = GetOptions(); - db_ = engine::DBFactory::Build(options); + db_ = ms::engine::DBFactory::Build(options); } void MetricTest::TearDown() { @@ -72,10 +78,12 @@ void MetricTest::TearDown() { int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); + + std::string uri; if (argc > 1) { uri = argv[1]; } -// std::cout << uri << std::endl; - ::testing::AddGlobalTestEnvironment(new DBTestEnvironment); + test_env = new DBTestEnvironment(uri); + ::testing::AddGlobalTestEnvironment(test_env); return RUN_ALL_TESTS(); } diff --git a/cpp/unittest/metrics/utils.h b/cpp/unittest/metrics/utils.h index 37c21c82f9..3daeeb8fa6 100644 --- a/cpp/unittest/metrics/utils.h +++ b/cpp/unittest/metrics/utils.h @@ -26,7 +26,6 @@ #include "db/meta/SqliteMetaImpl.h" #include "db/meta/MySQLMetaImpl.h" - #define TIMING #ifdef TIMING @@ -34,15 +33,15 @@ #define START_TIMER start = std::chrono::high_resolution_clock::now(); #define STOP_TIMER(name) LOG(DEBUG) << "RUNTIME of " << name << ": " << \ std::chrono::duration_cast( \ - std::chrono::high_resolution_clock::now()-start \ - ).count() << " ms "; + std::chrono::high_resolution_clock::now()-start).count() << " ms "; #else #define INIT_TIMER #define START_TIMER #define STOP_TIMER(name) #endif -void ASSERT_STATS(zilliz::milvus::Status& stat); +void +ASSERT_STATS(zilliz::milvus::Status &stat); //class TestEnv : public ::testing::Environment { //public: @@ -66,11 +65,11 @@ void ASSERT_STATS(zilliz::milvus::Status& stat); // ::testing::AddGlobalTestEnvironment(new TestEnv); class MetricTest : public ::testing::Test { -protected: + protected: zilliz::milvus::engine::DBPtr db_; void InitLog(); - virtual void SetUp() override; - virtual void TearDown() override; + void SetUp() override; + void TearDown() override; virtual zilliz::milvus::engine::DBOptions GetOptions(); -}; \ No newline at end of file +}; diff --git a/cpp/unittest/scheduler/algorithm_test.cpp b/cpp/unittest/scheduler/algorithm_test.cpp index fe619c4981..56574a7463 100644 --- a/cpp/unittest/scheduler/algorithm_test.cpp +++ b/cpp/unittest/scheduler/algorithm_test.cpp @@ -101,11 +101,8 @@ TEST_F(AlgorithmTest, SHORTESTPATH_TEST) { std::cout << sp[sp.size() - 1] << std::endl; sp.pop_back(); } - - } - -} -} -} \ No newline at end of file +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/unittest/scheduler/event_test.cpp b/cpp/unittest/scheduler/event_test.cpp index 34d4c2ce23..73fffc8f6e 100644 --- a/cpp/unittest/scheduler/event_test.cpp +++ b/cpp/unittest/scheduler/event_test.cpp @@ -60,7 +60,7 @@ TEST(EventTest, TASKTABLE_UPDATED_EVENT) { std::cout << *EventPtr(event); } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/unittest/scheduler/node_test.cpp b/cpp/unittest/scheduler/node_test.cpp index b87611d3ca..3323453f0b 100644 --- a/cpp/unittest/scheduler/node_test.cpp +++ b/cpp/unittest/scheduler/node_test.cpp @@ -19,37 +19,40 @@ #include "scheduler/resource/Node.h" #include +namespace { -using namespace zilliz::milvus::scheduler; +namespace ms = zilliz::milvus::scheduler; + +} // namespace class NodeTest : public ::testing::Test { -protected: + protected: void SetUp() override { - node1_ = std::make_shared(); - node2_ = std::make_shared(); - node3_ = std::make_shared(); - isolated_node1_ = std::make_shared(); - isolated_node2_ = std::make_shared(); + node1_ = std::make_shared(); + node2_ = std::make_shared(); + node3_ = std::make_shared(); + isolated_node1_ = std::make_shared(); + isolated_node2_ = std::make_shared(); - auto pcie = Connection("PCIe", 11.0); + auto pcie = ms::Connection("PCIe", 11.0); node1_->AddNeighbour(node2_, pcie); node1_->AddNeighbour(node3_, pcie); node2_->AddNeighbour(node1_, pcie); } - NodePtr node1_; - NodePtr node2_; - NodePtr node3_; - NodePtr isolated_node1_; - NodePtr isolated_node2_; + ms::NodePtr node1_; + ms::NodePtr node2_; + ms::NodePtr node3_; + ms::NodePtr isolated_node1_; + ms::NodePtr isolated_node2_; }; TEST_F(NodeTest, ADD_NEIGHBOUR) { ASSERT_EQ(isolated_node1_->GetNeighbours().size(), 0); ASSERT_EQ(isolated_node2_->GetNeighbours().size(), 0); - auto pcie = Connection("PCIe", 11.0); + auto pcie = ms::Connection("PCIe", 11.0); isolated_node1_->AddNeighbour(isolated_node2_, pcie); ASSERT_EQ(isolated_node1_->GetNeighbours().size(), 1); ASSERT_EQ(isolated_node2_->GetNeighbours().size(), 0); @@ -58,7 +61,7 @@ TEST_F(NodeTest, ADD_NEIGHBOUR) { TEST_F(NodeTest, REPEAT_ADD_NEIGHBOUR) { ASSERT_EQ(isolated_node1_->GetNeighbours().size(), 0); ASSERT_EQ(isolated_node2_->GetNeighbours().size(), 0); - auto pcie = Connection("PCIe", 11.0); + auto pcie = ms::Connection("PCIe", 11.0); isolated_node1_->AddNeighbour(isolated_node2_, pcie); isolated_node1_->AddNeighbour(isolated_node2_, pcie); ASSERT_EQ(isolated_node1_->GetNeighbours().size(), 1); @@ -97,3 +100,4 @@ TEST_F(NodeTest, DUMP) { std::cout << node2_->Dump(); ASSERT_FALSE(node2_->Dump().empty()); } + diff --git a/cpp/unittest/scheduler/normal_test.cpp b/cpp/unittest/scheduler/normal_test.cpp index 535fa96731..e24b43c6f8 100644 --- a/cpp/unittest/scheduler/normal_test.cpp +++ b/cpp/unittest/scheduler/normal_test.cpp @@ -25,35 +25,37 @@ #include "utils/Log.h" #include +namespace { -using namespace zilliz::milvus::scheduler; +namespace ms = zilliz::milvus::scheduler; +} // namespace TEST(NormalTest, INST_TEST) { // ResourceMgr only compose resources, provide unified event - auto res_mgr = ResMgrInst::GetInstance(); + auto res_mgr = ms::ResMgrInst::GetInstance(); - res_mgr->Add(ResourceFactory::Create("disk", "DISK", 0, true, false)); - res_mgr->Add(ResourceFactory::Create("cpu", "CPU", 0, true, true)); + res_mgr->Add(ms::ResourceFactory::Create("disk", "DISK", 0, true, false)); + res_mgr->Add(ms::ResourceFactory::Create("cpu", "CPU", 0, true, true)); - auto IO = Connection("IO", 500.0); + auto IO = ms::Connection("IO", 500.0); res_mgr->Connect("disk", "cpu", IO); - auto scheduler = SchedInst::GetInstance(); + auto scheduler = ms::SchedInst::GetInstance(); res_mgr->Start(); scheduler->Start(); const uint64_t NUM_TASK = 1000; - std::vector> tasks; - TableFileSchemaPtr dummy = nullptr; + std::vector> tasks; + ms::TableFileSchemaPtr dummy = nullptr; auto disks = res_mgr->GetDiskResources(); ASSERT_FALSE(disks.empty()); if (auto observe = disks[0].lock()) { for (uint64_t i = 0; i < NUM_TASK; ++i) { - auto task = std::make_shared(dummy); - task->label() = std::make_shared(); + auto task = std::make_shared(dummy); + task->label() = std::make_shared(); tasks.push_back(task); observe->task_table().Put(task); } @@ -67,5 +69,4 @@ TEST(NormalTest, INST_TEST) { scheduler->Stop(); res_mgr->Stop(); - } diff --git a/cpp/unittest/scheduler/resource_factory_test.cpp b/cpp/unittest/scheduler/resource_factory_test.cpp index e2a84257eb..679edc6e3f 100644 --- a/cpp/unittest/scheduler/resource_factory_test.cpp +++ b/cpp/unittest/scheduler/resource_factory_test.cpp @@ -19,15 +19,18 @@ #include "scheduler/ResourceFactory.h" #include +namespace { -using namespace zilliz::milvus::scheduler; +namespace ms = zilliz::milvus::scheduler; + +} // namespace TEST(ResourceFactoryTest, CREATE) { - auto disk = ResourceFactory::Create("ssd", "DISK", 0); - auto cpu = ResourceFactory::Create("cpu", "CPU", 0); - auto gpu = ResourceFactory::Create("gpu", "GPU", 0); + auto disk = ms::ResourceFactory::Create("ssd", "DISK", 0); + auto cpu = ms::ResourceFactory::Create("cpu", "CPU", 0); + auto gpu = ms::ResourceFactory::Create("gpu", "GPU", 0); - ASSERT_TRUE(std::dynamic_pointer_cast(disk)); - ASSERT_TRUE(std::dynamic_pointer_cast(cpu)); - ASSERT_TRUE(std::dynamic_pointer_cast(gpu)); + ASSERT_TRUE(std::dynamic_pointer_cast(disk)); + ASSERT_TRUE(std::dynamic_pointer_cast(cpu)); + ASSERT_TRUE(std::dynamic_pointer_cast(gpu)); } diff --git a/cpp/unittest/scheduler/resource_mgr_test.cpp b/cpp/unittest/scheduler/resource_mgr_test.cpp index c2be785a00..bf8ade9f64 100644 --- a/cpp/unittest/scheduler/resource_mgr_test.cpp +++ b/cpp/unittest/scheduler/resource_mgr_test.cpp @@ -24,15 +24,13 @@ #include "scheduler/ResourceMgr.h" #include - namespace zilliz { namespace milvus { namespace scheduler { - /************ ResourceMgrBaseTest ************/ class ResourceMgrBaseTest : public testing::Test { -protected: + protected: void SetUp() override { empty_mgr_ = std::make_shared(); @@ -77,7 +75,6 @@ TEST_F(ResourceMgrBaseTest, CONNECT) { ASSERT_TRUE(empty_mgr_->Connect("resource1", "resource2", io)); } - TEST_F(ResourceMgrBaseTest, INVALID_CONNECT) { auto resource1 = std::make_shared("resource1", 0, true, true); auto resource2 = std::make_shared("resource2", 2, true, true); @@ -87,7 +84,6 @@ TEST_F(ResourceMgrBaseTest, INVALID_CONNECT) { ASSERT_FALSE(empty_mgr_->Connect("xx", "yy", io)); } - TEST_F(ResourceMgrBaseTest, CLEAR) { ASSERT_EQ(mgr1_->GetNumOfResource(), 3); mgr1_->Clear(); @@ -163,7 +159,7 @@ TEST_F(ResourceMgrBaseTest, DUMP_TASKTABLES) { /************ ResourceMgrAdvanceTest ************/ class ResourceMgrAdvanceTest : public testing::Test { - protected: + protected: void SetUp() override { mgr1_ = std::make_shared(); @@ -193,7 +189,6 @@ TEST_F(ResourceMgrAdvanceTest, REGISTER_SUBSCRIBER) { ASSERT_TRUE(flag); } - -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/unittest/scheduler/resource_test.cpp b/cpp/unittest/scheduler/resource_test.cpp index b335a601db..b6c522b8b5 100644 --- a/cpp/unittest/scheduler/resource_test.cpp +++ b/cpp/unittest/scheduler/resource_test.cpp @@ -26,20 +26,19 @@ #include "scheduler/ResourceFactory.h" #include - namespace zilliz { namespace milvus { namespace scheduler { /************ ResourceBaseTest ************/ class ResourceBaseTest : public testing::Test { -protected: + protected: void SetUp() override { - only_loader_ = std::make_shared(name1, id1, true, false); - only_executor_ = std::make_shared(name2, id2, false, true); - both_enable_ = std::make_shared(name3, id3, true, true); - both_disable_ = std::make_shared(name4, id4, false, false); + only_loader_ = std::make_shared(name1, id1, true, false); + only_executor_ = std::make_shared(name2, id2, false, true); + both_enable_ = std::make_shared(name3, id3, true, true); + both_disable_ = std::make_shared(name4, id4, false, false); } const std::string name1 = "only_loader_"; @@ -104,7 +103,7 @@ TEST_F(ResourceBaseTest, DUMP) { /************ ResourceAdvanceTest ************/ class ResourceAdvanceTest : public testing::Test { -protected: + protected: void SetUp() override { disk_resource_ = ResourceFactory::Create("ssd", "DISK", 0); @@ -156,13 +155,17 @@ protected: void WaitLoader(uint64_t count) { std::unique_lock lock(load_mutex_); - cv_.wait(lock, [&] { return load_count_ == count; }); + cv_.wait(lock, [&] { + return load_count_ == count; + }); } void WaitExecutor(uint64_t count) { std::unique_lock lock(exec_mutex_); - cv_.wait(lock, [&] { return exec_count_ == count; }); + cv_.wait(lock, [&] { + return exec_count_ == count; + }); } ResourcePtr disk_resource_; @@ -277,6 +280,6 @@ TEST_F(ResourceAdvanceTest, TEST_RESOURCE_TEST) { } } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/unittest/scheduler/schedinst_test.cpp b/cpp/unittest/scheduler/schedinst_test.cpp index cf1003f5ce..4f3364c8fb 100644 --- a/cpp/unittest/scheduler/schedinst_test.cpp +++ b/cpp/unittest/scheduler/schedinst_test.cpp @@ -25,10 +25,8 @@ namespace zilliz { namespace milvus { namespace scheduler { - class SchedInstTest : public testing::Test { - -protected: + protected: void SetUp() override { boost::filesystem::create_directory(TMP_DIR); @@ -83,6 +81,8 @@ TEST_F(SchedInstTest, SIMPLE_GPU) { StartSchedulerService(); } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz + + diff --git a/cpp/unittest/scheduler/scheduler_test.cpp b/cpp/unittest/scheduler/scheduler_test.cpp index 8fc65f7d3b..4acfda315a 100644 --- a/cpp/unittest/scheduler/scheduler_test.cpp +++ b/cpp/unittest/scheduler/scheduler_test.cpp @@ -28,19 +28,18 @@ #include "utils/Error.h" #include "wrapper/VecIndex.h" - namespace zilliz { namespace milvus { namespace scheduler { class MockVecIndex : public engine::VecIndex { -public: - virtual Status BuildAll(const long &nb, - const float *xb, - const long *ids, - const engine::Config &cfg, - const long &nt = 0, - const float *xt = nullptr) { + public: + virtual Status BuildAll(const int64_t &nb, + const float *xb, + const int64_t *ids, + const engine::Config &cfg, + const int64_t &nt = 0, + const float *xt = nullptr) { } engine::VecIndexPtr Clone() override { @@ -55,27 +54,23 @@ public: return engine::IndexType::INVALID; } - virtual Status Add(const long &nb, - const float *xb, - const long *ids, - const engine::Config &cfg = engine::Config()) { - + virtual Status Add(const int64_t &nb, + const float *xb, + const int64_t *ids, + const engine::Config &cfg = engine::Config()) { } - virtual Status Search(const long &nq, - const float *xq, - float *dist, - long *ids, - const engine::Config &cfg = engine::Config()) { - + virtual Status Search(const int64_t &nq, + const float *xq, + float *dist, + int64_t *ids, + const engine::Config &cfg = engine::Config()) { } engine::VecIndexPtr CopyToGpu(const int64_t &device_id, const engine::Config &cfg) override { - } engine::VecIndexPtr CopyToCpu(const engine::Config &cfg) override { - } virtual int64_t Dimension() { @@ -92,20 +87,18 @@ public: } virtual Status Load(const zilliz::knowhere::BinarySet &index_binary) { - } -public: + public: int64_t dimension_ = 512; int64_t ntotal_ = 0; }; - class SchedulerTest : public testing::Test { -protected: + protected: void SetUp() override { - constexpr int64_t cache_cap = 1024*1024*1024; + constexpr int64_t cache_cap = 1024 * 1024 * 1024; cache::GpuCacheMgr::GetInstance(0)->SetCapacity(cache_cap); cache::GpuCacheMgr::GetInstance(1)->SetCapacity(cache_cap); @@ -170,7 +163,6 @@ TEST_F(SchedulerTest, ON_LOAD_COMPLETED) { sleep(3); ASSERT_EQ(res_mgr_->GetResource(ResourceType::GPU, 1)->task_table().Size(), NUM); - } TEST_F(SchedulerTest, PUSH_TASK_TO_NEIGHBOUR_RANDOMLY_TEST) { @@ -193,7 +185,7 @@ TEST_F(SchedulerTest, PUSH_TASK_TO_NEIGHBOUR_RANDOMLY_TEST) { } class SchedulerTest2 : public testing::Test { -protected: + protected: void SetUp() override { ResourcePtr disk = ResourceFactory::Create("disk", "DISK", 0, true, false); @@ -243,7 +235,6 @@ protected: std::shared_ptr scheduler_; }; - TEST_F(SchedulerTest2, SPECIFIED_RESOURCE_TEST) { const uint64_t NUM = 10; std::vector> tasks; @@ -260,6 +251,6 @@ TEST_F(SchedulerTest2, SPECIFIED_RESOURCE_TEST) { // ASSERT_EQ(res_mgr_->GetResource(ResourceType::GPU, 1)->task_table().Size(), NUM); } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/unittest/scheduler/task_test.cpp b/cpp/unittest/scheduler/task_test.cpp index 17d1e28cd6..d4bb0318da 100644 --- a/cpp/unittest/scheduler/task_test.cpp +++ b/cpp/unittest/scheduler/task_test.cpp @@ -19,17 +19,16 @@ #include "scheduler/task/SearchTask.h" #include - namespace zilliz { namespace milvus { namespace scheduler { - TEST(TaskTest, INVALID_INDEX) { auto search_task = std::make_shared(nullptr); search_task->Load(LoadType::TEST, 10); } -} -} -} +} // namespace scheduler +} // namespace milvus +} // namespace zilliz + diff --git a/cpp/unittest/scheduler/tasktable_test.cpp b/cpp/unittest/scheduler/tasktable_test.cpp index 5c64a72351..0cb7018023 100644 --- a/cpp/unittest/scheduler/tasktable_test.cpp +++ b/cpp/unittest/scheduler/tasktable_test.cpp @@ -20,51 +20,53 @@ #include "scheduler/task/TestTask.h" #include +namespace { -using namespace zilliz::milvus::scheduler; +namespace ms = zilliz::milvus::scheduler; +} // namespace /************ TaskTableBaseTest ************/ class TaskTableItemTest : public ::testing::Test { -protected: + protected: void SetUp() override { - std::vector states{ - TaskTableItemState::INVALID, - TaskTableItemState::START, - TaskTableItemState::LOADING, - TaskTableItemState::LOADED, - TaskTableItemState::EXECUTING, - TaskTableItemState::EXECUTED, - TaskTableItemState::MOVING, - TaskTableItemState::MOVED}; + std::vector states{ + ms::TaskTableItemState::INVALID, + ms::TaskTableItemState::START, + ms::TaskTableItemState::LOADING, + ms::TaskTableItemState::LOADED, + ms::TaskTableItemState::EXECUTING, + ms::TaskTableItemState::EXECUTED, + ms::TaskTableItemState::MOVING, + ms::TaskTableItemState::MOVED}; for (auto &state : states) { - auto item = std::make_shared(); + auto item = std::make_shared(); item->state = state; items_.emplace_back(item); } } - TaskTableItem default_; - std::vector items_; + ms::TaskTableItem default_; + std::vector items_; }; TEST_F(TaskTableItemTest, CONSTRUCT) { ASSERT_EQ(default_.id, 0); ASSERT_EQ(default_.task, nullptr); - ASSERT_EQ(default_.state, TaskTableItemState::INVALID); + ASSERT_EQ(default_.state, ms::TaskTableItemState::INVALID); } TEST_F(TaskTableItemTest, DESTRUCT) { - auto p_item = new TaskTableItem(); + auto p_item = new ms::TaskTableItem(); delete p_item; } TEST_F(TaskTableItemTest, IS_FINISH) { for (auto &item : items_) { - if (item->state == TaskTableItemState::EXECUTED - || item->state == TaskTableItemState::MOVED) { + if (item->state == ms::TaskTableItemState::EXECUTED + || item->state == ms::TaskTableItemState::MOVED) { ASSERT_TRUE(item->IsFinish()); } else { ASSERT_FALSE(item->IsFinish()); @@ -82,9 +84,9 @@ TEST_F(TaskTableItemTest, LOAD) { for (auto &item : items_) { auto before_state = item->state; auto ret = item->Load(); - if (before_state == TaskTableItemState::START) { + if (before_state == ms::TaskTableItemState::START) { ASSERT_TRUE(ret); - ASSERT_EQ(item->state, TaskTableItemState::LOADING); + ASSERT_EQ(item->state, ms::TaskTableItemState::LOADING); } else { ASSERT_FALSE(ret); ASSERT_EQ(item->state, before_state); @@ -96,9 +98,9 @@ TEST_F(TaskTableItemTest, LOADED) { for (auto &item : items_) { auto before_state = item->state; auto ret = item->Loaded(); - if (before_state == TaskTableItemState::LOADING) { + if (before_state == ms::TaskTableItemState::LOADING) { ASSERT_TRUE(ret); - ASSERT_EQ(item->state, TaskTableItemState::LOADED); + ASSERT_EQ(item->state, ms::TaskTableItemState::LOADED); } else { ASSERT_FALSE(ret); ASSERT_EQ(item->state, before_state); @@ -110,9 +112,9 @@ TEST_F(TaskTableItemTest, EXECUTE) { for (auto &item : items_) { auto before_state = item->state; auto ret = item->Execute(); - if (before_state == TaskTableItemState::LOADED) { + if (before_state == ms::TaskTableItemState::LOADED) { ASSERT_TRUE(ret); - ASSERT_EQ(item->state, TaskTableItemState::EXECUTING); + ASSERT_EQ(item->state, ms::TaskTableItemState::EXECUTING); } else { ASSERT_FALSE(ret); ASSERT_EQ(item->state, before_state); @@ -120,14 +122,13 @@ TEST_F(TaskTableItemTest, EXECUTE) { } } - TEST_F(TaskTableItemTest, EXECUTED) { for (auto &item : items_) { auto before_state = item->state; auto ret = item->Executed(); - if (before_state == TaskTableItemState::EXECUTING) { + if (before_state == ms::TaskTableItemState::EXECUTING) { ASSERT_TRUE(ret); - ASSERT_EQ(item->state, TaskTableItemState::EXECUTED); + ASSERT_EQ(item->state, ms::TaskTableItemState::EXECUTED); } else { ASSERT_FALSE(ret); ASSERT_EQ(item->state, before_state); @@ -139,9 +140,9 @@ TEST_F(TaskTableItemTest, MOVE) { for (auto &item : items_) { auto before_state = item->state; auto ret = item->Move(); - if (before_state == TaskTableItemState::LOADED) { + if (before_state == ms::TaskTableItemState::LOADED) { ASSERT_TRUE(ret); - ASSERT_EQ(item->state, TaskTableItemState::MOVING); + ASSERT_EQ(item->state, ms::TaskTableItemState::MOVING); } else { ASSERT_FALSE(ret); ASSERT_EQ(item->state, before_state); @@ -153,9 +154,9 @@ TEST_F(TaskTableItemTest, MOVED) { for (auto &item : items_) { auto before_state = item->state; auto ret = item->Moved(); - if (before_state == TaskTableItemState::MOVING) { + if (before_state == ms::TaskTableItemState::MOVING) { ASSERT_TRUE(ret); - ASSERT_EQ(item->state, TaskTableItemState::MOVED); + ASSERT_EQ(item->state, ms::TaskTableItemState::MOVED); } else { ASSERT_FALSE(ret); ASSERT_EQ(item->state, before_state); @@ -166,19 +167,19 @@ TEST_F(TaskTableItemTest, MOVED) { /************ TaskTableBaseTest ************/ class TaskTableBaseTest : public ::testing::Test { -protected: + protected: void SetUp() override { - TableFileSchemaPtr dummy = nullptr; + ms::TableFileSchemaPtr dummy = nullptr; invalid_task_ = nullptr; - task1_ = std::make_shared(dummy); - task2_ = std::make_shared(dummy); + task1_ = std::make_shared(dummy); + task2_ = std::make_shared(dummy); } - TaskPtr invalid_task_; - TaskPtr task1_; - TaskPtr task2_; - TaskTable empty_table_; + ms::TaskPtr invalid_task_; + ms::TaskPtr task1_; + ms::TaskPtr task2_; + ms::TaskTable empty_table_; }; TEST_F(TaskTableBaseTest, SUBSCRIBER) { @@ -191,7 +192,6 @@ TEST_F(TaskTableBaseTest, SUBSCRIBER) { ASSERT_TRUE(flag); } - TEST_F(TaskTableBaseTest, PUT_TASK) { empty_table_.Put(task1_); ASSERT_EQ(empty_table_.Get(0)->task, task1_); @@ -203,14 +203,14 @@ TEST_F(TaskTableBaseTest, PUT_INVALID_TEST) { } TEST_F(TaskTableBaseTest, PUT_BATCH) { - std::vector tasks{task1_, task2_}; + std::vector tasks{task1_, task2_}; empty_table_.Put(tasks); ASSERT_EQ(empty_table_.Get(0)->task, task1_); ASSERT_EQ(empty_table_.Get(1)->task, task2_); } TEST_F(TaskTableBaseTest, PUT_EMPTY_BATCH) { - std::vector tasks{}; + std::vector tasks{}; empty_table_.Put(tasks); } @@ -236,8 +236,8 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD) { for (size_t i = 0; i < NUM_TASKS; ++i) { empty_table_.Put(task1_); } - empty_table_[0]->state = TaskTableItemState::MOVED; - empty_table_[1]->state = TaskTableItemState::EXECUTED; + empty_table_[0]->state = ms::TaskTableItemState::MOVED; + empty_table_[1]->state = ms::TaskTableItemState::EXECUTED; auto indexes = empty_table_.PickToLoad(1); ASSERT_EQ(indexes.size(), 1); @@ -249,8 +249,8 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD_LIMIT) { for (size_t i = 0; i < NUM_TASKS; ++i) { empty_table_.Put(task1_); } - empty_table_[0]->state = TaskTableItemState::MOVED; - empty_table_[1]->state = TaskTableItemState::EXECUTED; + empty_table_[0]->state = ms::TaskTableItemState::MOVED; + empty_table_[1]->state = ms::TaskTableItemState::EXECUTED; auto indexes = empty_table_.PickToLoad(3); ASSERT_EQ(indexes.size(), 3); @@ -264,8 +264,8 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD_CACHE) { for (size_t i = 0; i < NUM_TASKS; ++i) { empty_table_.Put(task1_); } - empty_table_[0]->state = TaskTableItemState::MOVED; - empty_table_[1]->state = TaskTableItemState::EXECUTED; + empty_table_[0]->state = ms::TaskTableItemState::MOVED; + empty_table_[1]->state = ms::TaskTableItemState::EXECUTED; // first pick, non-cache auto indexes = empty_table_.PickToLoad(1); @@ -274,7 +274,7 @@ TEST_F(TaskTableBaseTest, PICK_TO_LOAD_CACHE) { // second pick, iterate from 2 // invalid state change - empty_table_[1]->state = TaskTableItemState::START; + empty_table_[1]->state = ms::TaskTableItemState::START; indexes = empty_table_.PickToLoad(1); ASSERT_EQ(indexes.size(), 1); ASSERT_EQ(indexes[0], 2); @@ -285,9 +285,9 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE) { for (size_t i = 0; i < NUM_TASKS; ++i) { empty_table_.Put(task1_); } - empty_table_[0]->state = TaskTableItemState::MOVED; - empty_table_[1]->state = TaskTableItemState::EXECUTED; - empty_table_[2]->state = TaskTableItemState::LOADED; + empty_table_[0]->state = ms::TaskTableItemState::MOVED; + empty_table_[1]->state = ms::TaskTableItemState::EXECUTED; + empty_table_[2]->state = ms::TaskTableItemState::LOADED; auto indexes = empty_table_.PickToExecute(1); ASSERT_EQ(indexes.size(), 1); @@ -299,10 +299,10 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE_LIMIT) { for (size_t i = 0; i < NUM_TASKS; ++i) { empty_table_.Put(task1_); } - empty_table_[0]->state = TaskTableItemState::MOVED; - empty_table_[1]->state = TaskTableItemState::EXECUTED; - empty_table_[2]->state = TaskTableItemState::LOADED; - empty_table_[3]->state = TaskTableItemState::LOADED; + empty_table_[0]->state = ms::TaskTableItemState::MOVED; + empty_table_[1]->state = ms::TaskTableItemState::EXECUTED; + empty_table_[2]->state = ms::TaskTableItemState::LOADED; + empty_table_[3]->state = ms::TaskTableItemState::LOADED; auto indexes = empty_table_.PickToExecute(3); ASSERT_EQ(indexes.size(), 2); @@ -315,9 +315,9 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE_CACHE) { for (size_t i = 0; i < NUM_TASKS; ++i) { empty_table_.Put(task1_); } - empty_table_[0]->state = TaskTableItemState::MOVED; - empty_table_[1]->state = TaskTableItemState::EXECUTED; - empty_table_[2]->state = TaskTableItemState::LOADED; + empty_table_[0]->state = ms::TaskTableItemState::MOVED; + empty_table_[1]->state = ms::TaskTableItemState::EXECUTED; + empty_table_[2]->state = ms::TaskTableItemState::LOADED; // first pick, non-cache auto indexes = empty_table_.PickToExecute(1); @@ -326,40 +326,39 @@ TEST_F(TaskTableBaseTest, PICK_TO_EXECUTE_CACHE) { // second pick, iterate from 2 // invalid state change - empty_table_[1]->state = TaskTableItemState::START; + empty_table_[1]->state = ms::TaskTableItemState::START; indexes = empty_table_.PickToExecute(1); ASSERT_EQ(indexes.size(), 1); ASSERT_EQ(indexes[0], 2); } - /************ TaskTableAdvanceTest ************/ class TaskTableAdvanceTest : public ::testing::Test { -protected: + protected: void SetUp() override { - TableFileSchemaPtr dummy = nullptr; + ms::TableFileSchemaPtr dummy = nullptr; for (uint64_t i = 0; i < 8; ++i) { - auto task = std::make_shared(dummy); + auto task = std::make_shared(dummy); table1_.Put(task); } - table1_.Get(0)->state = TaskTableItemState::INVALID; - table1_.Get(1)->state = TaskTableItemState::START; - table1_.Get(2)->state = TaskTableItemState::LOADING; - table1_.Get(3)->state = TaskTableItemState::LOADED; - table1_.Get(4)->state = TaskTableItemState::EXECUTING; - table1_.Get(5)->state = TaskTableItemState::EXECUTED; - table1_.Get(6)->state = TaskTableItemState::MOVING; - table1_.Get(7)->state = TaskTableItemState::MOVED; + table1_.Get(0)->state = ms::TaskTableItemState::INVALID; + table1_.Get(1)->state = ms::TaskTableItemState::START; + table1_.Get(2)->state = ms::TaskTableItemState::LOADING; + table1_.Get(3)->state = ms::TaskTableItemState::LOADED; + table1_.Get(4)->state = ms::TaskTableItemState::EXECUTING; + table1_.Get(5)->state = ms::TaskTableItemState::EXECUTED; + table1_.Get(6)->state = ms::TaskTableItemState::MOVING; + table1_.Get(7)->state = ms::TaskTableItemState::MOVED; } - TaskTable table1_; + ms::TaskTable table1_; }; TEST_F(TaskTableAdvanceTest, LOAD) { - std::vector before_state; + std::vector before_state; for (auto &task : table1_) { before_state.push_back(task->state); } @@ -369,8 +368,8 @@ TEST_F(TaskTableAdvanceTest, LOAD) { } for (size_t i = 0; i < table1_.Size(); ++i) { - if (before_state[i] == TaskTableItemState::START) { - ASSERT_EQ(table1_.Get(i)->state, TaskTableItemState::LOADING); + if (before_state[i] == ms::TaskTableItemState::START) { + ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::LOADING); } else { ASSERT_EQ(table1_.Get(i)->state, before_state[i]); } @@ -378,7 +377,7 @@ TEST_F(TaskTableAdvanceTest, LOAD) { } TEST_F(TaskTableAdvanceTest, LOADED) { - std::vector before_state; + std::vector before_state; for (auto &task : table1_) { before_state.push_back(task->state); } @@ -388,8 +387,8 @@ TEST_F(TaskTableAdvanceTest, LOADED) { } for (size_t i = 0; i < table1_.Size(); ++i) { - if (before_state[i] == TaskTableItemState::LOADING) { - ASSERT_EQ(table1_.Get(i)->state, TaskTableItemState::LOADED); + if (before_state[i] == ms::TaskTableItemState::LOADING) { + ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::LOADED); } else { ASSERT_EQ(table1_.Get(i)->state, before_state[i]); } @@ -397,7 +396,7 @@ TEST_F(TaskTableAdvanceTest, LOADED) { } TEST_F(TaskTableAdvanceTest, EXECUTE) { - std::vector before_state; + std::vector before_state; for (auto &task : table1_) { before_state.push_back(task->state); } @@ -407,8 +406,8 @@ TEST_F(TaskTableAdvanceTest, EXECUTE) { } for (size_t i = 0; i < table1_.Size(); ++i) { - if (before_state[i] == TaskTableItemState::LOADED) { - ASSERT_EQ(table1_.Get(i)->state, TaskTableItemState::EXECUTING); + if (before_state[i] == ms::TaskTableItemState::LOADED) { + ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::EXECUTING); } else { ASSERT_EQ(table1_.Get(i)->state, before_state[i]); } @@ -416,7 +415,7 @@ TEST_F(TaskTableAdvanceTest, EXECUTE) { } TEST_F(TaskTableAdvanceTest, EXECUTED) { - std::vector before_state; + std::vector before_state; for (auto &task : table1_) { before_state.push_back(task->state); } @@ -426,8 +425,8 @@ TEST_F(TaskTableAdvanceTest, EXECUTED) { } for (size_t i = 0; i < table1_.Size(); ++i) { - if (before_state[i] == TaskTableItemState::EXECUTING) { - ASSERT_EQ(table1_.Get(i)->state, TaskTableItemState::EXECUTED); + if (before_state[i] == ms::TaskTableItemState::EXECUTING) { + ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::EXECUTED); } else { ASSERT_EQ(table1_.Get(i)->state, before_state[i]); } @@ -435,7 +434,7 @@ TEST_F(TaskTableAdvanceTest, EXECUTED) { } TEST_F(TaskTableAdvanceTest, MOVE) { - std::vector before_state; + std::vector before_state; for (auto &task : table1_) { before_state.push_back(task->state); } @@ -445,8 +444,8 @@ TEST_F(TaskTableAdvanceTest, MOVE) { } for (size_t i = 0; i < table1_.Size(); ++i) { - if (before_state[i] == TaskTableItemState::LOADED) { - ASSERT_EQ(table1_.Get(i)->state, TaskTableItemState::MOVING); + if (before_state[i] == ms::TaskTableItemState::LOADED) { + ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::MOVING); } else { ASSERT_EQ(table1_.Get(i)->state, before_state[i]); } @@ -454,7 +453,7 @@ TEST_F(TaskTableAdvanceTest, MOVE) { } TEST_F(TaskTableAdvanceTest, MOVED) { - std::vector before_state; + std::vector before_state; for (auto &task : table1_) { before_state.push_back(task->state); } @@ -464,8 +463,8 @@ TEST_F(TaskTableAdvanceTest, MOVED) { } for (size_t i = 0; i < table1_.Size(); ++i) { - if (before_state[i] == TaskTableItemState::MOVING) { - ASSERT_EQ(table1_.Get(i)->state, TaskTableItemState::MOVED); + if (before_state[i] == ms::TaskTableItemState::MOVING) { + ASSERT_EQ(table1_.Get(i)->state, ms::TaskTableItemState::MOVED); } else { ASSERT_EQ(table1_.Get(i)->state, before_state[i]); } diff --git a/cpp/unittest/server/cache_test.cpp b/cpp/unittest/server/cache_test.cpp index 290caf4a97..056d60185a 100644 --- a/cpp/unittest/server/cache_test.cpp +++ b/cpp/unittest/server/cache_test.cpp @@ -21,41 +21,40 @@ #include "utils/Error.h" #include "wrapper/VecIndex.h" -using namespace zilliz::milvus; - namespace { -class InvalidCacheMgr : public cache::CacheMgr { -public: +namespace ms = zilliz::milvus; + +class InvalidCacheMgr : public ms::cache::CacheMgr { + public: InvalidCacheMgr() { } }; -class LessItemCacheMgr : public cache::CacheMgr { -public: +class LessItemCacheMgr : public ms::cache::CacheMgr { + public: LessItemCacheMgr() { - cache_ = std::make_shared>(1UL << 12, 10); + cache_ = std::make_shared>(1UL << 12, 10); } }; -class MockVecIndex : public engine::VecIndex { -public: +class MockVecIndex : public ms::engine::VecIndex { + public: MockVecIndex(int64_t dim, int64_t total) : dimension_(dim), - ntotal_(total){ - + ntotal_(total) { } - virtual Status BuildAll(const long &nb, - const float *xb, - const long *ids, - const engine::Config &cfg, - const long &nt = 0, - const float *xt = nullptr) { - return Status(); + virtual ms::Status BuildAll(const int64_t &nb, + const float *xb, + const int64_t *ids, + const ms::engine::Config &cfg, + const int64_t &nt = 0, + const float *xt = nullptr) { + return ms::Status(); } - engine::VecIndexPtr Clone() override { + ms::engine::VecIndexPtr Clone() override { return zilliz::milvus::engine::VecIndexPtr(); } @@ -63,31 +62,31 @@ public: return 0; } - engine::IndexType GetType() override { - return engine::IndexType::INVALID; + ms::engine::IndexType GetType() override { + return ms::engine::IndexType::INVALID; } - virtual Status Add(const long &nb, - const float *xb, - const long *ids, - const engine::Config &cfg = engine::Config()) { - return Status(); + virtual ms::Status Add(const int64_t &nb, + const float *xb, + const int64_t *ids, + const ms::engine::Config &cfg = ms::engine::Config()) { + return ms::Status(); } - virtual Status Search(const long &nq, - const float *xq, - float *dist, - long *ids, - const engine::Config &cfg = engine::Config()) { - return Status(); + virtual ms::Status Search(const int64_t &nq, + const float *xq, + float *dist, + int64_t *ids, + const ms::engine::Config &cfg = ms::engine::Config()) { + return ms::Status(); } - engine::VecIndexPtr CopyToGpu(const int64_t &device_id, - const engine::Config &cfg) override { + ms::engine::VecIndexPtr CopyToGpu(const int64_t &device_id, + const ms::engine::Config &cfg) override { return nullptr; } - engine::VecIndexPtr CopyToCpu(const engine::Config &cfg) override { + ms::engine::VecIndexPtr CopyToCpu(const ms::engine::Config &cfg) override { return nullptr; } @@ -104,19 +103,19 @@ public: return binset; } - virtual Status Load(const zilliz::knowhere::BinarySet &index_binary) { - return Status(); + virtual ms::Status Load(const zilliz::knowhere::BinarySet &index_binary) { + return ms::Status(); } -public: + public: int64_t dimension_ = 256; int64_t ntotal_ = 0; }; -} +} // namespace TEST(CacheTest, DUMMY_TEST) { - engine::Config cfg; + ms::engine::Config cfg; MockVecIndex mock_index(256, 1000); mock_index.Dimension(); mock_index.Count(); @@ -134,9 +133,9 @@ TEST(CacheTest, DUMMY_TEST) { } TEST(CacheTest, CPU_CACHE_TEST) { - auto cpu_mgr = cache::CpuCacheMgr::GetInstance(); + auto cpu_mgr = ms::cache::CpuCacheMgr::GetInstance(); - const int64_t gbyte = 1024*1024*1024; + const int64_t gbyte = 1024 * 1024 * 1024; int64_t g_num = 16; int64_t cap = g_num * gbyte; cpu_mgr->SetCapacity(cap); @@ -145,8 +144,8 @@ TEST(CacheTest, CPU_CACHE_TEST) { uint64_t item_count = 20; for (uint64_t i = 0; i < item_count; i++) { //each vector is 1k byte, total size less than 1G - engine::VecIndexPtr mock_index = std::make_shared(256, 1000000); - cache::DataObjPtr data_obj = std::make_shared(mock_index); + ms::engine::VecIndexPtr mock_index = std::make_shared(256, 1000000); + ms::cache::DataObjPtr data_obj = std::make_shared(mock_index); cpu_mgr->InsertItem("index_" + std::to_string(i), data_obj); } ASSERT_LT(cpu_mgr->ItemCount(), g_num); @@ -169,8 +168,8 @@ TEST(CacheTest, CPU_CACHE_TEST) { cpu_mgr->SetCapacity(g_num * gbyte); //each vector is 1k byte, total size less than 6G - engine::VecIndexPtr mock_index = std::make_shared(256, 6000000); - cache::DataObjPtr data_obj = std::make_shared(mock_index); + ms::engine::VecIndexPtr mock_index = std::make_shared(256, 6000000); + ms::cache::DataObjPtr data_obj = std::make_shared(mock_index); cpu_mgr->InsertItem("index_6g", data_obj); ASSERT_TRUE(cpu_mgr->ItemExists("index_6g")); } @@ -179,12 +178,12 @@ TEST(CacheTest, CPU_CACHE_TEST) { } TEST(CacheTest, GPU_CACHE_TEST) { - auto gpu_mgr = cache::GpuCacheMgr::GetInstance(0); + auto gpu_mgr = ms::cache::GpuCacheMgr::GetInstance(0); - for(int i = 0; i < 20; i++) { + for (int i = 0; i < 20; i++) { //each vector is 1k byte - engine::VecIndexPtr mock_index = std::make_shared(256, 1000); - cache::DataObjPtr data_obj = std::make_shared(mock_index); + ms::engine::VecIndexPtr mock_index = std::make_shared(256, 1000); + ms::cache::DataObjPtr data_obj = std::make_shared(mock_index); gpu_mgr->InsertItem("index_" + std::to_string(i), data_obj); } @@ -196,15 +195,14 @@ TEST(CacheTest, GPU_CACHE_TEST) { for (auto i = 0; i < 3; i++) { // TODO: use gpu index to mock //each vector is 1k byte, total size less than 2G - engine::VecIndexPtr mock_index = std::make_shared(256, 2000000); - cache::DataObjPtr data_obj = std::make_shared(mock_index); - std::cout << data_obj->size() <(256, 2000000); + ms::cache::DataObjPtr data_obj = std::make_shared(mock_index); + std::cout << data_obj->size() << std::endl; gpu_mgr->InsertItem("index_" + std::to_string(i), data_obj); } gpu_mgr->ClearCache(); ASSERT_EQ(gpu_mgr->ItemCount(), 0); - } TEST(CacheTest, INVALID_TEST) { @@ -214,7 +212,7 @@ TEST(CacheTest, INVALID_TEST) { ASSERT_FALSE(mgr.ItemExists("test")); ASSERT_EQ(mgr.GetItem("test"), nullptr); - mgr.InsertItem("test", cache::DataObjPtr()); + mgr.InsertItem("test", ms::cache::DataObjPtr()); mgr.InsertItem("test", nullptr); mgr.EraseItem("test"); mgr.PrintInfo(); @@ -226,12 +224,12 @@ TEST(CacheTest, INVALID_TEST) { { LessItemCacheMgr mgr; - for(int i = 0; i < 20; i++) { + for (int i = 0; i < 20; i++) { //each vector is 1k byte - engine::VecIndexPtr mock_index = std::make_shared(256, 2); - cache::DataObjPtr data_obj = std::make_shared(mock_index); + ms::engine::VecIndexPtr mock_index = std::make_shared(256, 2); + ms::cache::DataObjPtr data_obj = std::make_shared(mock_index); mgr.InsertItem("index_" + std::to_string(i), data_obj); } ASSERT_EQ(mgr.GetItem("index_0"), nullptr); } -} \ No newline at end of file +} diff --git a/cpp/unittest/server/config_test.cpp b/cpp/unittest/server/config_test.cpp index b27d1e60c8..0c613d8954 100644 --- a/cpp/unittest/server/config_test.cpp +++ b/cpp/unittest/server/config_test.cpp @@ -23,39 +23,39 @@ #include "utils/ValidationUtil.h" #include "server/Config.h" -using namespace zilliz::milvus; - namespace { -static const char* CONFIG_FILE_PATH = "./milvus/conf/server_config.yaml"; -static const char* LOG_FILE_PATH = "./milvus/conf/log_config.conf"; +namespace ms = zilliz::milvus; + +static const char *CONFIG_FILE_PATH = "./milvus/conf/server_config.yaml"; +static const char *LOG_FILE_PATH = "./milvus/conf/log_config.conf"; static constexpr uint64_t KB = 1024; -static constexpr uint64_t MB = KB*1024; -static constexpr uint64_t GB = MB*1024; +static constexpr uint64_t MB = KB * 1024; +static constexpr uint64_t GB = MB * 1024; -} +} // namespace TEST(ConfigTest, CONFIG_TEST) { - server::ConfigMgr* config_mgr = server::ConfigMgr::GetInstance(); + ms::server::ConfigMgr *config_mgr = ms::server::ConfigMgr::GetInstance(); - ErrorCode err = config_mgr->LoadConfigFile(""); - ASSERT_EQ(err, SERVER_UNEXPECTED_ERROR); + ms::ErrorCode err = config_mgr->LoadConfigFile(""); + ASSERT_EQ(err, ms::SERVER_UNEXPECTED_ERROR); err = config_mgr->LoadConfigFile(LOG_FILE_PATH); - ASSERT_EQ(err, SERVER_UNEXPECTED_ERROR); + ASSERT_EQ(err, ms::SERVER_UNEXPECTED_ERROR); err = config_mgr->LoadConfigFile(CONFIG_FILE_PATH); - ASSERT_EQ(err, SERVER_SUCCESS); + ASSERT_EQ(err, ms::SERVER_SUCCESS); config_mgr->Print(); - server::ConfigNode& root_config = config_mgr->GetRootNode(); - server::ConfigNode& server_config = root_config.GetChild("server_config"); - server::ConfigNode& db_config = root_config.GetChild("db_config"); - server::ConfigNode& metric_config = root_config.GetChild("metric_config"); - server::ConfigNode& cache_config = root_config.GetChild("cache_config"); - server::ConfigNode invalid_config = root_config.GetChild("invalid_config"); + ms::server::ConfigNode &root_config = config_mgr->GetRootNode(); + ms::server::ConfigNode &server_config = root_config.GetChild("server_config"); + ms::server::ConfigNode &db_config = root_config.GetChild("db_config"); + ms::server::ConfigNode &metric_config = root_config.GetChild("metric_config"); + ms::server::ConfigNode &cache_config = root_config.GetChild("cache_config"); + ms::server::ConfigNode invalid_config = root_config.GetChild("invalid_config"); auto valus = invalid_config.GetSequence("not_exist"); float ff = invalid_config.GetFloatValue("not_exist", 3.0); ASSERT_EQ(ff, 3.0); @@ -63,16 +63,16 @@ TEST(ConfigTest, CONFIG_TEST) { std::string address = server_config.GetValue("address"); ASSERT_TRUE(!address.empty()); int64_t port = server_config.GetInt64Value("port"); - ASSERT_TRUE(port != 0); + ASSERT_NE(port, 0); server_config.SetValue("test", "2.5"); double test = server_config.GetDoubleValue("test"); ASSERT_EQ(test, 2.5); - server::ConfigNode fake; + ms::server::ConfigNode fake; server_config.AddChild("fake", fake); fake = server_config.GetChild("fake"); - server::ConfigNodeArr arr; + ms::server::ConfigNodeArr arr; server_config.GetChildren(arr); ASSERT_EQ(arr.size(), 1UL); @@ -89,7 +89,7 @@ TEST(ConfigTest, CONFIG_TEST) { auto seq = server_config.GetSequence("seq"); ASSERT_EQ(seq.size(), 2UL); - server::ConfigNode combine; + ms::server::ConfigNode combine; combine.Combine(server_config); combine.PrintAll(); @@ -102,8 +102,8 @@ TEST(ConfigTest, CONFIG_TEST) { } TEST(ConfigTest, SERVER_CONFIG_TEST) { - server::Config& config = server::Config::GetInstance(); - Status s = config.LoadConfigFile(CONFIG_FILE_PATH); + ms::server::Config &config = ms::server::Config::GetInstance(); + ms::Status s = config.LoadConfigFile(CONFIG_FILE_PATH); ASSERT_TRUE(s.ok()); s = config.ValidateConfig(); @@ -113,4 +113,4 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) { s = config.ResetDefaultConfig(); ASSERT_TRUE(s.ok()); -} \ No newline at end of file +} diff --git a/cpp/unittest/server/rpc_test.cpp b/cpp/unittest/server/rpc_test.cpp index 0eaadf6146..3a36e3ee50 100644 --- a/cpp/unittest/server/rpc_test.cpp +++ b/cpp/unittest/server/rpc_test.cpp @@ -23,7 +23,7 @@ #include "server/grpc_impl/GrpcRequestHandler.h" #include "server/grpc_impl/GrpcRequestScheduler.h" #include "server/grpc_impl/GrpcRequestTask.h" -#include "version.h" +#include "../version.h" #include "grpc/gen-milvus/milvus.grpc.pb.h" #include "grpc/gen-status/status.pb.h" @@ -34,10 +34,10 @@ #include "scheduler/ResourceFactory.h" #include "utils/CommonUtil.h" -using namespace zilliz::milvus; - namespace { +namespace ms = zilliz::milvus; + static const char *TABLE_NAME = "test_grpc"; static constexpr int64_t TABLE_DIM = 256; static constexpr int64_t INDEX_FILE_SIZE = 1024; @@ -49,30 +49,29 @@ class RpcHandlerTest : public testing::Test { protected: void SetUp() override { - - auto res_mgr = scheduler::ResMgrInst::GetInstance(); + auto res_mgr = ms::scheduler::ResMgrInst::GetInstance(); res_mgr->Clear(); - res_mgr->Add(scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false)); - res_mgr->Add(scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, true)); - res_mgr->Add(scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true)); + res_mgr->Add(ms::scheduler::ResourceFactory::Create("disk", "DISK", 0, true, false)); + res_mgr->Add(ms::scheduler::ResourceFactory::Create("cpu", "CPU", 0, true, true)); + res_mgr->Add(ms::scheduler::ResourceFactory::Create("gtx1660", "GPU", 0, true, true)); - auto default_conn = scheduler::Connection("IO", 500.0); - auto PCIE = scheduler::Connection("IO", 11000.0); + auto default_conn = ms::scheduler::Connection("IO", 500.0); + auto PCIE = ms::scheduler::Connection("IO", 11000.0); res_mgr->Connect("disk", "cpu", default_conn); res_mgr->Connect("cpu", "gtx1660", PCIE); res_mgr->Start(); - scheduler::SchedInst::GetInstance()->Start(); - scheduler::JobMgrInst::GetInstance()->Start(); + ms::scheduler::SchedInst::GetInstance()->Start(); + ms::scheduler::JobMgrInst::GetInstance()->Start(); - engine::DBOptions opt; + ms::engine::DBOptions opt; - server::Config::GetInstance().SetDBConfigBackendUrl("sqlite://:@:/"); - server::Config::GetInstance().SetDBConfigPrimaryPath("/tmp/milvus_test"); - server::Config::GetInstance().SetDBConfigSecondaryPath(""); - server::Config::GetInstance().SetDBConfigArchiveDiskThreshold(""); - server::Config::GetInstance().SetDBConfigArchiveDaysThreshold(""); - server::Config::GetInstance().SetCacheConfigCacheInsertData(""); - server::Config::GetInstance().SetEngineConfigOmpThreadNum(""); + ms::server::Config::GetInstance().SetDBConfigBackendUrl("sqlite://:@:/"); + ms::server::Config::GetInstance().SetDBConfigPrimaryPath("/tmp/milvus_test"); + ms::server::Config::GetInstance().SetDBConfigSecondaryPath(""); + ms::server::Config::GetInstance().SetDBConfigArchiveDiskThreshold(""); + ms::server::Config::GetInstance().SetDBConfigArchiveDaysThreshold(""); + ms::server::Config::GetInstance().SetCacheConfigCacheInsertData(""); + ms::server::Config::GetInstance().SetEngineConfigOmpThreadNum(""); // serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "cluster"); // DBWrapper::GetInstance().GetInstance().StartService(); @@ -82,11 +81,11 @@ class RpcHandlerTest : public testing::Test { // DBWrapper::GetInstance().GetInstance().StartService(); // DBWrapper::GetInstance().GetInstance().StopService(); - server::Config::GetInstance().SetResourceConfigMode("single"); - server::DBWrapper::GetInstance().StartService(); + ms::server::Config::GetInstance().SetResourceConfigMode("single"); + ms::server::DBWrapper::GetInstance().StartService(); //initialize handler, create table - handler = std::make_shared(); + handler = std::make_shared(); ::grpc::ServerContext context; ::milvus::grpc::TableSchema request; ::milvus::grpc::Status status; @@ -99,18 +98,20 @@ class RpcHandlerTest : public testing::Test { void TearDown() override { - server::DBWrapper::GetInstance().StopService(); - scheduler::JobMgrInst::GetInstance()->Stop(); - scheduler::ResMgrInst::GetInstance()->Stop(); - scheduler::SchedInst::GetInstance()->Stop(); + ms::server::DBWrapper::GetInstance().StopService(); + ms::scheduler::JobMgrInst::GetInstance()->Stop(); + ms::scheduler::ResMgrInst::GetInstance()->Stop(); + ms::scheduler::SchedInst::GetInstance()->Stop(); boost::filesystem::remove_all("/tmp/milvus_test"); } + protected: - std::shared_ptr handler; + std::shared_ptr handler; }; -void BuildVectors(int64_t from, int64_t to, - std::vector> &vector_record_array) { +void +BuildVectors(int64_t from, int64_t to, + std::vector> &vector_record_array) { if (to <= from) { return; } @@ -127,20 +128,22 @@ void BuildVectors(int64_t from, int64_t to, } } -std::string CurrentTmDate(int64_t offset_day = 0) { +std::string +CurrentTmDate(int64_t offset_day = 0) { time_t tt; time(&tt); tt = tt + 8 * SECONDS_EACH_HOUR; tt = tt + 24 * SECONDS_EACH_HOUR * offset_day; - tm *t = gmtime(&tt); + tm t; + gmtime_r(&tt, &t); - std::string str = std::to_string(t->tm_year + 1900) + "-" + std::to_string(t->tm_mon + 1) - + "-" + std::to_string(t->tm_mday); + std::string str = std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1) + + "-" + std::to_string(t.tm_mday); return str; } -} +} // namespace TEST_F(RpcHandlerTest, HAS_TABLE_TEST) { ::grpc::ServerContext context; @@ -311,7 +314,6 @@ TEST_F(RpcHandlerTest, TABLES_TEST) { ::grpc::Status status = handler->DescribeTable(&context, &table_name, &table_schema); ASSERT_EQ(status.error_code(), ::grpc::Status::OK.error_code()); - ::milvus::grpc::InsertParam request; std::vector> record_array; BuildVectors(0, VECTOR_COUNT, record_array); @@ -419,31 +421,29 @@ TEST_F(RpcHandlerTest, DELETE_BY_RANGE_TEST) { grpc_status = handler->DeleteByRange(&context, &request, &status); request.mutable_range()->set_end_value(CurrentTmDate(-2)); grpc_status = handler->DeleteByRange(&context, &request, &status); - } ////////////////////////////////////////////////////////////////////// namespace { -class DummyTask : public server::grpc::GrpcBaseTask { -public: - Status +class DummyTask : public ms::server::grpc::GrpcBaseTask { + public: + ms::Status OnExecute() override { - return Status::OK(); + return ms::Status::OK(); } - static server::grpc::BaseTaskPtr + static ms::server::grpc::BaseTaskPtr Create(std::string &dummy) { - return std::shared_ptr(new DummyTask(dummy)); + return std::shared_ptr(new DummyTask(dummy)); } -public: + public: explicit DummyTask(std::string &dummy) : GrpcBaseTask(dummy) { - } }; class RpcSchedulerTest : public testing::Test { -protected: + protected: void SetUp() override { std::string dummy = "dql"; @@ -453,22 +453,22 @@ protected: std::shared_ptr task_ptr; }; -} +} // namespace -TEST_F(RpcSchedulerTest, BASE_TASK_TEST){ +TEST_F(RpcSchedulerTest, BASE_TASK_TEST) { auto status = task_ptr->Execute(); ASSERT_TRUE(status.ok()); - server::grpc::GrpcRequestScheduler::GetInstance().Start(); + ms::server::grpc::GrpcRequestScheduler::GetInstance().Start(); ::milvus::grpc::Status grpc_status; std::string dummy = "dql"; - server::grpc::BaseTaskPtr base_task_ptr = DummyTask::Create(dummy); - server::grpc::GrpcRequestScheduler::GetInstance().ExecTask(base_task_ptr, &grpc_status); + ms::server::grpc::BaseTaskPtr base_task_ptr = DummyTask::Create(dummy); + ms::server::grpc::GrpcRequestScheduler::GetInstance().ExecTask(base_task_ptr, &grpc_status); - server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); + ms::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); task_ptr = nullptr; - server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); + ms::server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); - server::grpc::GrpcRequestScheduler::GetInstance().Stop(); + ms::server::grpc::GrpcRequestScheduler::GetInstance().Stop(); } diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index 1757a7ddfe..89095193fb 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -31,75 +31,76 @@ #include #include -using namespace zilliz::milvus; - namespace { -static const char* LOG_FILE_PATH = "./milvus/conf/log_config.conf"; +namespace ms = zilliz::milvus; -void CopyStatus(Status& st1, Status& st2) { +static const char *LOG_FILE_PATH = "./milvus/conf/log_config.conf"; + +void +CopyStatus(ms::Status &st1, ms::Status &st2) { st1 = st2; } -} +} // namespace TEST(UtilTest, EXCEPTION_TEST) { std::string err_msg = "failed"; - server::ServerException ex(SERVER_UNEXPECTED_ERROR, err_msg); - ASSERT_EQ(ex.error_code(), SERVER_UNEXPECTED_ERROR); + ms::server::ServerException ex(ms::SERVER_UNEXPECTED_ERROR, err_msg); + ASSERT_EQ(ex.error_code(), ms::SERVER_UNEXPECTED_ERROR); std::string msg = ex.what(); ASSERT_EQ(msg, err_msg); } TEST(UtilTest, SIGNAL_TEST) { - server::SignalUtil::PrintStacktrace(); + ms::server::SignalUtil::PrintStacktrace(); } TEST(UtilTest, COMMON_TEST) { - unsigned long total_mem = 0, free_mem = 0; - server::CommonUtil::GetSystemMemInfo(total_mem, free_mem); + uint64_t total_mem = 0, free_mem = 0; + ms::server::CommonUtil::GetSystemMemInfo(total_mem, free_mem); ASSERT_GT(total_mem, 0); ASSERT_GT(free_mem, 0); - unsigned int thread_cnt = 0; - server::CommonUtil::GetSystemAvailableThreads(thread_cnt); + uint32_t thread_cnt = 0; + ms::server::CommonUtil::GetSystemAvailableThreads(thread_cnt); ASSERT_GT(thread_cnt, 0); std::string path1 = "/tmp/milvus_test/"; std::string path2 = path1 + "common_test_12345/"; std::string path3 = path2 + "abcdef"; - Status status = server::CommonUtil::CreateDirectory(path3); + ms::Status status = ms::server::CommonUtil::CreateDirectory(path3); ASSERT_TRUE(status.ok()); //test again - status = server::CommonUtil::CreateDirectory(path3); + status = ms::server::CommonUtil::CreateDirectory(path3); ASSERT_TRUE(status.ok()); - ASSERT_TRUE(server::CommonUtil::IsDirectoryExist(path3)); + ASSERT_TRUE(ms::server::CommonUtil::IsDirectoryExist(path3)); - status = server::CommonUtil::DeleteDirectory(path1); + status = ms::server::CommonUtil::DeleteDirectory(path1); ASSERT_TRUE(status.ok()); //test again - status = server::CommonUtil::DeleteDirectory(path1); + status = ms::server::CommonUtil::DeleteDirectory(path1); ASSERT_TRUE(status.ok()); - ASSERT_FALSE(server::CommonUtil::IsDirectoryExist(path1)); - ASSERT_FALSE(server::CommonUtil::IsFileExist(path1)); + ASSERT_FALSE(ms::server::CommonUtil::IsDirectoryExist(path1)); + ASSERT_FALSE(ms::server::CommonUtil::IsFileExist(path1)); - std::string exe_path = server::CommonUtil::GetExePath(); + std::string exe_path = ms::server::CommonUtil::GetExePath(); ASSERT_FALSE(exe_path.empty()); time_t tt; - time( &tt ); + time(&tt); tm time_struct; memset(&time_struct, 0, sizeof(tm)); - server::CommonUtil::ConvertTime(tt, time_struct); + ms::server::CommonUtil::ConvertTime(tt, time_struct); ASSERT_GT(time_struct.tm_year, 0); ASSERT_GT(time_struct.tm_mon, 0); ASSERT_GT(time_struct.tm_mday, 0); - server::CommonUtil::ConvertTime(time_struct, tt); + ms::server::CommonUtil::ConvertTime(time_struct, tt); ASSERT_GT(tt, 0); - bool res = server::CommonUtil::TimeStrToTime("2019-03-23", tt, time_struct); + bool res = ms::server::CommonUtil::TimeStrToTime("2019-03-23", tt, time_struct); ASSERT_EQ(time_struct.tm_year, 119); ASSERT_EQ(time_struct.tm_mon, 2); ASSERT_EQ(time_struct.tm_mday, 23); @@ -109,45 +110,43 @@ TEST(UtilTest, COMMON_TEST) { TEST(UtilTest, STRINGFUNCTIONS_TEST) { std::string str = " test zilliz"; - server::StringHelpFunctions::TrimStringBlank(str); + ms::server::StringHelpFunctions::TrimStringBlank(str); ASSERT_EQ(str, "test zilliz"); str = "\"test zilliz\""; - server::StringHelpFunctions::TrimStringQuote(str, "\""); + ms::server::StringHelpFunctions::TrimStringQuote(str, "\""); ASSERT_EQ(str, "test zilliz"); str = "a,b,c"; std::vector result; - auto status = server::StringHelpFunctions::SplitStringByDelimeter(str , ",", result); + auto status = ms::server::StringHelpFunctions::SplitStringByDelimeter(str, ",", result); ASSERT_TRUE(status.ok()); ASSERT_EQ(result.size(), 3UL); result.clear(); - status = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result); + status = ms::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result); ASSERT_TRUE(status.ok()); ASSERT_EQ(result.size(), 3UL); result.clear(); - status = server::StringHelpFunctions::SplitStringByQuote(str , ",", "", result); + status = ms::server::StringHelpFunctions::SplitStringByQuote(str, ",", "", result); ASSERT_TRUE(status.ok()); ASSERT_EQ(result.size(), 3UL); str = "55,\"aa,gg,yy\",b"; result.clear(); - status = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result); + status = ms::server::StringHelpFunctions::SplitStringByQuote(str, ",", "\"", result); ASSERT_TRUE(status.ok()); ASSERT_EQ(result.size(), 3UL); - - } TEST(UtilTest, BLOCKINGQUEUE_TEST) { - server::BlockingQueue bq; + ms::server::BlockingQueue bq; static const size_t count = 10; bq.SetCapacity(count); - for(size_t i = 1; i <= count; i++) { + for (size_t i = 1; i <= count; i++) { std::string id = "No." + std::to_string(i); bq.Put(id); } @@ -161,7 +160,7 @@ TEST(UtilTest, BLOCKINGQUEUE_TEST) { str = bq.Back(); ASSERT_EQ(str, "No." + std::to_string(count)); - for(size_t i = 1; i <= count; i++) { + for (size_t i = 1; i <= count; i++) { std::string id = "No." + std::to_string(i); str = bq.Take(); ASSERT_EQ(id, str); @@ -171,54 +170,54 @@ TEST(UtilTest, BLOCKINGQUEUE_TEST) { } TEST(UtilTest, LOG_TEST) { - auto status = server::InitLog(LOG_FILE_PATH); + auto status = ms::server::InitLog(LOG_FILE_PATH); ASSERT_TRUE(status.ok()); EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::NewLineForContainer)); EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::LogDetailedCrashReason)); - std::string fname = server::CommonUtil::GetFileName(LOG_FILE_PATH); + std::string fname = ms::server::CommonUtil::GetFileName(LOG_FILE_PATH); ASSERT_EQ(fname, "log_config.conf"); } TEST(UtilTest, TIMERECORDER_TEST) { - for(int64_t log_level = 0; log_level <= 6; log_level++) { - if(log_level == 5) { + for (int64_t log_level = 0; log_level <= 6; log_level++) { + if (log_level == 5) { continue; //skip fatal } - TimeRecorder rc("time", log_level); + ms::TimeRecorder rc("time", log_level); rc.RecordSection("end"); } } TEST(UtilTest, STATUS_TEST) { - auto status = Status::OK(); + auto status = ms::Status::OK(); std::string str = status.ToString(); ASSERT_FALSE(str.empty()); - status = Status(DB_ERROR, "mistake"); - ASSERT_EQ(status.code(), DB_ERROR); + status = ms::Status(ms::DB_ERROR, "mistake"); + ASSERT_EQ(status.code(), ms::DB_ERROR); str = status.ToString(); ASSERT_FALSE(str.empty()); - status = Status(DB_NOT_FOUND, "mistake"); - ASSERT_EQ(status.code(), DB_NOT_FOUND); + status = ms::Status(ms::DB_NOT_FOUND, "mistake"); + ASSERT_EQ(status.code(), ms::DB_NOT_FOUND); str = status.ToString(); ASSERT_FALSE(str.empty()); - status = Status(DB_ALREADY_EXIST, "mistake"); - ASSERT_EQ(status.code(), DB_ALREADY_EXIST); + status = ms::Status(ms::DB_ALREADY_EXIST, "mistake"); + ASSERT_EQ(status.code(), ms::DB_ALREADY_EXIST); str = status.ToString(); ASSERT_FALSE(str.empty()); - status = Status(DB_META_TRANSACTION_FAILED, "mistake"); - ASSERT_EQ(status.code(), DB_META_TRANSACTION_FAILED); + status = ms::Status(ms::DB_META_TRANSACTION_FAILED, "mistake"); + ASSERT_EQ(status.code(), ms::DB_META_TRANSACTION_FAILED); str = status.ToString(); ASSERT_FALSE(str.empty()); - auto status_copy = Status::OK(); + auto status_copy = ms::Status::OK(); CopyStatus(status_copy, status); - ASSERT_EQ(status.code(), DB_META_TRANSACTION_FAILED); + ASSERT_EQ(status.code(), ms::DB_META_TRANSACTION_FAILED); auto status_ref(status); ASSERT_EQ(status_ref.code(), status.code()); @@ -231,123 +230,140 @@ TEST(UtilTest, STATUS_TEST) { TEST(ValidationUtilTest, VALIDATE_TABLENAME_TEST) { std::string table_name = "Normal123_"; - auto status = server::ValidationUtil::ValidateTableName(table_name); + auto status = ms::server::ValidationUtil::ValidateTableName(table_name); ASSERT_TRUE(status.ok()); table_name = "12sds"; - status = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(status.code(), SERVER_INVALID_TABLE_NAME); + status = ms::server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME); table_name = ""; - status = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(status.code(), SERVER_INVALID_TABLE_NAME); + status = ms::server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME); table_name = "_asdasd"; - status = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(status.code(), SERVER_SUCCESS); + status = ms::server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), ms::SERVER_SUCCESS); table_name = "!@#!@"; - status = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(status.code(), SERVER_INVALID_TABLE_NAME); + status = ms::server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME); table_name = "_!@#!@"; - status = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(status.code(), SERVER_INVALID_TABLE_NAME); + status = ms::server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME); table_name = "中文"; - status = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(status.code(), SERVER_INVALID_TABLE_NAME); + status = ms::server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME); table_name = std::string(10000, 'a'); - status = server::ValidationUtil::ValidateTableName(table_name); - ASSERT_EQ(status.code(), SERVER_INVALID_TABLE_NAME); + status = ms::server::ValidationUtil::ValidateTableName(table_name); + ASSERT_EQ(status.code(), ms::SERVER_INVALID_TABLE_NAME); } TEST(ValidationUtilTest, VALIDATE_DIMENSION_TEST) { - ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(-1).code(), SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(0).code(), SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16385).code(), SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16384).code(), SERVER_SUCCESS); - ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(1).code(), SERVER_SUCCESS); + ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(-1).code(), ms::SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(0).code(), ms::SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(16385).code(), ms::SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(16384).code(), ms::SERVER_SUCCESS); + ASSERT_EQ(ms::server::ValidationUtil::ValidateTableDimension(1).code(), ms::SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_INDEX_TEST) { - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::INVALID).code(), SERVER_INVALID_INDEX_TYPE); - for(int i = 1; i <= (int)engine::EngineType::MAX_VALUE; i++) { - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType(i).code(), SERVER_SUCCESS); + ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexType((int) ms::engine::EngineType::INVALID).code(), + ms::SERVER_INVALID_INDEX_TYPE); + for (int i = 1; i <= (int) ms::engine::EngineType::MAX_VALUE; i++) { + ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexType(i).code(), ms::SERVER_SUCCESS); } - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1).code(), SERVER_INVALID_INDEX_TYPE); + ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexType((int) ms::engine::EngineType::MAX_VALUE + 1).code(), + ms::SERVER_INVALID_INDEX_TYPE); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexNlist(0).code(), SERVER_INVALID_INDEX_NLIST); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexNlist(100).code(), SERVER_SUCCESS); + ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexNlist(0).code(), ms::SERVER_INVALID_INDEX_NLIST); + ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexNlist(100).code(), ms::SERVER_SUCCESS); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexFileSize(0).code(), SERVER_INVALID_INDEX_FILE_SIZE); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexFileSize(100).code(), SERVER_SUCCESS); + ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexFileSize(0).code(), ms::SERVER_INVALID_INDEX_FILE_SIZE); + ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexFileSize(100).code(), ms::SERVER_SUCCESS); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(0).code(), SERVER_INVALID_INDEX_METRIC_TYPE); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(1).code(), SERVER_SUCCESS); - ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(2).code(), SERVER_SUCCESS); + ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexMetricType(0).code(), ms::SERVER_INVALID_INDEX_METRIC_TYPE); + ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexMetricType(1).code(), ms::SERVER_SUCCESS); + ASSERT_EQ(ms::server::ValidationUtil::ValidateTableIndexMetricType(2).code(), ms::SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_TOPK_TEST) { - engine::meta::TableSchema schema; - ASSERT_EQ(server::ValidationUtil::ValidateSearchTopk(10, schema).code(), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(65536, schema).code(), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(0, schema).code(), SERVER_SUCCESS); + ms::engine::meta::TableSchema schema; + ASSERT_EQ(ms::server::ValidationUtil::ValidateSearchTopk(10, schema).code(), ms::SERVER_SUCCESS); + ASSERT_NE(ms::server::ValidationUtil::ValidateSearchTopk(65536, schema).code(), ms::SERVER_SUCCESS); + ASSERT_NE(ms::server::ValidationUtil::ValidateSearchTopk(0, schema).code(), ms::SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_NPROBE_TEST) { - engine::meta::TableSchema schema; + ms::engine::meta::TableSchema schema; schema.nlist_ = 100; - ASSERT_EQ(server::ValidationUtil::ValidateSearchNprobe(10, schema).code(), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateSearchNprobe(0, schema).code(), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateSearchNprobe(101, schema).code(), SERVER_SUCCESS); + ASSERT_EQ(ms::server::ValidationUtil::ValidateSearchNprobe(10, schema).code(), ms::SERVER_SUCCESS); + ASSERT_NE(ms::server::ValidationUtil::ValidateSearchNprobe(0, schema).code(), ms::SERVER_SUCCESS); + ASSERT_NE(ms::server::ValidationUtil::ValidateSearchNprobe(101, schema).code(), ms::SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_GPU_TEST) { - ASSERT_EQ(server::ValidationUtil::ValidateGpuIndex(0).code(), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateGpuIndex(100).code(), SERVER_SUCCESS); + ASSERT_EQ(ms::server::ValidationUtil::ValidateGpuIndex(0).code(), ms::SERVER_SUCCESS); + ASSERT_NE(ms::server::ValidationUtil::ValidateGpuIndex(100).code(), ms::SERVER_SUCCESS); size_t memory = 0; - ASSERT_EQ(server::ValidationUtil::GetGpuMemory(0, memory).code(), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::GetGpuMemory(100, memory).code(), SERVER_SUCCESS); + ASSERT_EQ(ms::server::ValidationUtil::GetGpuMemory(0, memory).code(), ms::SERVER_SUCCESS); + ASSERT_NE(ms::server::ValidationUtil::GetGpuMemory(100, memory).code(), ms::SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_IPADDRESS_TEST) { - ASSERT_EQ(server::ValidationUtil::ValidateIpAddress("127.0.0.1").code(), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateIpAddress("not ip").code(), SERVER_SUCCESS); + ASSERT_EQ(ms::server::ValidationUtil::ValidateIpAddress("127.0.0.1").code(), ms::SERVER_SUCCESS); + ASSERT_NE(ms::server::ValidationUtil::ValidateIpAddress("not ip").code(), ms::SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_NUMBER_TEST) { - ASSERT_EQ(server::ValidationUtil::ValidateStringIsNumber("1234").code(), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateStringIsNumber("not number").code(), SERVER_SUCCESS); + ASSERT_EQ(ms::server::ValidationUtil::ValidateStringIsNumber("1234").code(), ms::SERVER_SUCCESS); + ASSERT_NE(ms::server::ValidationUtil::ValidateStringIsNumber("not number").code(), ms::SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_BOOL_TEST) { std::string str = "true"; - ASSERT_EQ(server::ValidationUtil::ValidateStringIsBool(str).code(), SERVER_SUCCESS); + ASSERT_EQ(ms::server::ValidationUtil::ValidateStringIsBool(str).code(), ms::SERVER_SUCCESS); str = "not bool"; - ASSERT_NE(server::ValidationUtil::ValidateStringIsBool(str).code(), SERVER_SUCCESS); + ASSERT_NE(ms::server::ValidationUtil::ValidateStringIsBool(str).code(), ms::SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_DOUBLE_TEST) { - ASSERT_EQ(server::ValidationUtil::ValidateStringIsFloat("2.5").code(), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateStringIsFloat("not double").code(), SERVER_SUCCESS); + ASSERT_EQ(ms::server::ValidationUtil::ValidateStringIsFloat("2.5").code(), ms::SERVER_SUCCESS); + ASSERT_NE(ms::server::ValidationUtil::ValidateStringIsFloat("not double").code(), ms::SERVER_SUCCESS); } TEST(ValidationUtilTest, VALIDATE_DBURI_TEST) { - ASSERT_EQ(server::ValidationUtil::ValidateDbURI("sqlite://:@:/").code(), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateDbURI("xxx://:@:/").code(), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateDbURI("not uri").code(), SERVER_SUCCESS); - ASSERT_EQ(server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:3303/milvus").code(), SERVER_SUCCESS); - ASSERT_NE(server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:port/milvus").code(), SERVER_SUCCESS); + ASSERT_EQ(ms::server::ValidationUtil::ValidateDbURI("sqlite://:@:/").code(), ms::SERVER_SUCCESS); + ASSERT_NE(ms::server::ValidationUtil::ValidateDbURI("xxx://:@:/").code(), ms::SERVER_SUCCESS); + ASSERT_NE(ms::server::ValidationUtil::ValidateDbURI("not uri").code(), ms::SERVER_SUCCESS); + ASSERT_EQ(ms::server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:3303/milvus").code(), + ms::SERVER_SUCCESS); + ASSERT_NE(ms::server::ValidationUtil::ValidateDbURI("mysql://root:123456@127.0.0.1:port/milvus").code(), + ms::SERVER_SUCCESS); } -TEST(UtilTest, ROLLOUTHANDLER_TEST){ +TEST(UtilTest, ROLLOUTHANDLER_TEST) { std::string dir1 = "/tmp/milvus_test"; std::string dir2 = "/tmp/milvus_test/log_test"; - std::string filename[6] = {"log_global.log", "log_debug.log", "log_warning.log", "log_trace.log", "log_error.log", "log_fatal.log"}; - el::Level list[6] = {el::Level::Global, el::Level::Debug, el::Level::Warning, el::Level::Trace, el::Level::Error, el::Level::Fatal}; + std::string filename[6] = { + "log_global.log", + "log_debug.log", + "log_warning.log", + "log_trace.log", + "log_error.log", + "log_fatal.log"}; + + el::Level list[6] = { + el::Level::Global, + el::Level::Debug, + el::Level::Warning, + el::Level::Trace, + el::Level::Error, + el::Level::Fatal}; mkdir(dir1.c_str(), S_IRWXU); mkdir(dir2.c_str(), S_IRWXU); @@ -358,7 +374,7 @@ TEST(UtilTest, ROLLOUTHANDLER_TEST){ file.open(tmp.c_str()); file << "zilliz" << std::endl; - server::RolloutHandler(tmp.c_str(), 0, list[i]); + ms::server::RolloutHandler(tmp.c_str(), 0, list[i]); tmp.append(".1"); std::ifstream file2; @@ -369,4 +385,4 @@ TEST(UtilTest, ROLLOUTHANDLER_TEST){ ASSERT_EQ(tmp2, "zilliz"); } boost::filesystem::remove_all(dir2); -} \ No newline at end of file +} diff --git a/cpp/unittest/wrapper/utils.cpp b/cpp/unittest/wrapper/utils.cpp index f404f000e9..f9fe8a50fc 100644 --- a/cpp/unittest/wrapper/utils.cpp +++ b/cpp/unittest/wrapper/utils.cpp @@ -20,10 +20,10 @@ #include "utils.h" - -void DataGenBase::GenData(const int &dim, const int &nb, const int &nq, - float *xb, float *xq, long *ids, - const int &k, long *gt_ids, float *gt_dis) { +void +DataGenBase::GenData(const int &dim, const int &nb, const int &nq, + float *xb, float *xq, long *ids, + const int &k, long *gt_ids, float *gt_dis) { for (auto i = 0; i < nb; ++i) { for (auto j = 0; j < dim; ++j) { //p_data[i * d + j] = float(base + i); @@ -42,15 +42,16 @@ void DataGenBase::GenData(const int &dim, const int &nb, const int &nq, index.search(nq, xq, k, gt_dis, gt_ids); } -void DataGenBase::GenData(const int &dim, - const int &nb, - const int &nq, - std::vector &xb, - std::vector &xq, - std::vector &ids, - const int &k, - std::vector >_ids, - std::vector >_dis) { +void +DataGenBase::GenData(const int &dim, + const int &nb, + const int &nq, + std::vector &xb, + std::vector &xq, + std::vector &ids, + const int &k, + std::vector >_ids, + std::vector >_dis) { xb.resize(nb * dim); xq.resize(nq * dim); ids.resize(nb); diff --git a/cpp/unittest/wrapper/utils.h b/cpp/unittest/wrapper/utils.h index 7a73d84713..431aa30306 100644 --- a/cpp/unittest/wrapper/utils.h +++ b/cpp/unittest/wrapper/utils.h @@ -24,12 +24,10 @@ #include #include - class DataGenBase; using DataGenPtr = std::shared_ptr; - class DataGenBase { public: virtual void GenData(const int &dim, const int &nb, const int &nq, float *xb, float *xq, long *ids, diff --git a/cpp/unittest/wrapper/wrapper_test.cpp b/cpp/unittest/wrapper/wrapper_test.cpp index 4c50201fb3..19b84d315f 100644 --- a/cpp/unittest/wrapper/wrapper_test.cpp +++ b/cpp/unittest/wrapper/wrapper_test.cpp @@ -18,7 +18,7 @@ #include "utils/easylogging++.h" #include "src/wrapper/VecIndex.h" #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" -#include "utils.h" +#include "wrapper/utils.h" #include diff --git a/cpp/version.h.macro b/cpp/version.h.macro index 2463407a0d..454d8a990a 100644 --- a/cpp/version.h.macro +++ b/cpp/version.h.macro @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + #pragma once #define MILVUS_VERSION "@MILVUS_VERSION@" From 29e93e00b48e955c2732b11993e64b64fa6d40c8 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 27 Sep 2019 20:18:07 +0800 Subject: [PATCH 139/217] code stype xml Former-commit-id: 811cc247f82d0aba97e6dd491cb5d5003e545f1d --- .clang-format | 5 ++++ cpp/build-support/code_style_clion.xml | 39 ++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 cpp/build-support/code_style_clion.xml diff --git a/.clang-format b/.clang-format index 819c2d361e..1e124f7be8 100644 --- a/.clang-format +++ b/.clang-format @@ -18,3 +18,8 @@ BasedOnStyle: Google DerivePointerAlignment: false ColumnLimit: 120 +IndentWidth: 4 +AccessModifierOffset: -3 +AlwaysBreakAfterReturnType: All +AllowShortBlocksOnASingleLine: false +AllowShortFunctionsOnASingleLine: false diff --git a/cpp/build-support/code_style_clion.xml b/cpp/build-support/code_style_clion.xml new file mode 100644 index 0000000000..4c7dc4718c --- /dev/null +++ b/cpp/build-support/code_style_clion.xml @@ -0,0 +1,39 @@ + + + + + + \ No newline at end of file From c6501ec0499b1cf04aa3313780ebd453753a516d Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Fri, 27 Sep 2019 20:27:59 +0800 Subject: [PATCH 140/217] MS-596 enable knowhere and fix compile error Former-commit-id: 75c435ef22ceec6f07e92e42f99b369657e3911a --- cpp/src/core/CMakeLists.txt | 6 ++- .../knowhere/index/vector_index/nsg/NSG.cpp | 6 ++- cpp/src/core/test/CMakeLists.txt | 14 ++--- cpp/src/core/test/test_nsg/CMakeLists.txt | 26 +++------- cpp/src/core/test/test_nsg/test_nsg.cpp | 52 +++++++++++-------- cpp/src/wrapper/ConfAdapter.cpp | 4 ++ cpp/src/wrapper/ConfAdapter.h | 3 ++ cpp/src/wrapper/VecIndex.cpp | 2 +- cpp/unittest/wrapper/wrapper_test.cpp | 2 +- 9 files changed, 56 insertions(+), 59 deletions(-) diff --git a/cpp/src/core/CMakeLists.txt b/cpp/src/core/CMakeLists.txt index bef3f5e2f6..f1a13b4520 100644 --- a/cpp/src/core/CMakeLists.txt +++ b/cpp/src/core/CMakeLists.txt @@ -46,9 +46,11 @@ if(NOT CMAKE_BUILD_TYPE) endif(NOT CMAKE_BUILD_TYPE) if(CMAKE_BUILD_TYPE STREQUAL "Release") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -fopenmp") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp") + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3") else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -fopenmp") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp") + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") endif() MESSAGE(STATUS "CMAKE_CXX_FLAGS" ${CMAKE_CXX_FLAGS}) diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp index c2b5122d65..0f3bb639c2 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp @@ -24,6 +24,7 @@ #include "NSG.h" #include "knowhere/common/Exception.h" +#include "knowhere/common/Log.h" #include "knowhere/common/Timer.h" #include "NSGHelper.h" @@ -83,8 +84,9 @@ void NsgIndex::Build_with_ids(size_t nb, const float *data, const long *ids, con for (int i = 0; i < ntotal; ++i) { total_degree += nsg[i].size(); } - std::cout << "graph physical size: " << total_degree * sizeof(node_t) / 1024 / 1024; - std::cout << "average degree: " << total_degree / ntotal; + + KNOWHERE_LOG_DEBUG << "Graph physical size: " << total_degree * sizeof(node_t) / 1024 / 1024 << "m"; + KNOWHERE_LOG_DEBUG << "Average degree: " << total_degree / ntotal; ///// is_trained = true; diff --git a/cpp/src/core/test/CMakeLists.txt b/cpp/src/core/test/CMakeLists.txt index cc6ea5bdcc..5ad1c9e00e 100644 --- a/cpp/src/core/test/CMakeLists.txt +++ b/cpp/src/core/test/CMakeLists.txt @@ -29,7 +29,7 @@ set(util_srcs ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/ArrowAdapter.cpp ${CORE_SOURCE_DIR}/knowhere/knowhere/common/Exception.cpp ${CORE_SOURCE_DIR}/knowhere/knowhere/common/Timer.cpp - utils.cpp + ${CORE_SOURCE_DIR}/test/utils.cpp ) # @@ -52,18 +52,10 @@ target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs}) # set(idmap_srcs - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVF.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp - ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp ) if(NOT TARGET test_idmap) - add_executable(test_idmap test_idmap.cpp ${idmap_srcs} ${util_srcs}) + add_executable(test_idmap test_idmap.cpp ${idmap_srcs} ${ivf_srcs} ${util_srcs}) endif() target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs}) @@ -86,5 +78,5 @@ install(TARGETS test_idmap DESTINATION unittest) install(TARGETS test_kdt DESTINATION unittest) #add_subdirectory(faiss_ori) -#add_subdirectory(test_nsg) +add_subdirectory(test_nsg) diff --git a/cpp/src/core/test/test_nsg/CMakeLists.txt b/cpp/src/core/test/test_nsg/CMakeLists.txt index 17b62cd40d..393cc188b0 100644 --- a/cpp/src/core/test/test_nsg/CMakeLists.txt +++ b/cpp/src/core/test/test_nsg/CMakeLists.txt @@ -1,6 +1,6 @@ ############################## -include_directories(/usr/local/include/gperftools) -link_directories(/usr/local/lib) +#include_directories(/usr/local/include/gperftools) +#link_directories(/usr/local/lib) add_definitions(-std=c++11 -O3 -lboost -march=native -Wall -DINFO) @@ -13,29 +13,15 @@ else () endif () message(${OpenMP_CXX_FLAGS}) -include_directories(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg) - -aux_source_directory(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src) +include_directories(${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/nsg) +aux_source_directory(${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/nsg nsg_src) set(interface_src - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp - ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg_index.cpp - ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp - ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp - ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp - ../utils.cpp + ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNSG.cpp ) if(NOT TARGET test_nsg) - add_executable(test_nsg - test_nsg.cpp - ${interface_src} - ${nsg_src} - ${util_srcs} - ) + add_executable(test_nsg test_nsg.cpp ${interface_src} ${nsg_src} ${util_srcs} ${ivf_srcs}) endif() target_link_libraries(test_nsg ${depend_libs} ${unittest_libs} ${basic_libs}) diff --git a/cpp/src/core/test/test_nsg/test_nsg.cpp b/cpp/src/core/test/test_nsg/test_nsg.cpp index b1b1edef46..8f0187e0a3 100644 --- a/cpp/src/core/test/test_nsg/test_nsg.cpp +++ b/cpp/src/core/test/test_nsg/test_nsg.cpp @@ -19,10 +19,11 @@ #include #include -#include "knowhere/common/exception.h" -#include "knowhere/index/vector_index/gpu_ivf.h" -#include "knowhere/index/vector_index/nsg_index.h" -#include "knowhere/index/vector_index/nsg/nsg_io.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/FaissBaseIndex.h" +#include "knowhere/index/vector_index/IndexNSG.h" +#include "knowhere/index/vector_index/nsg/NSGIO.h" +#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #include "../utils.h" @@ -32,16 +33,31 @@ using ::testing::TestWithParam; using ::testing::Values; using ::testing::Combine; -constexpr int64_t DEVICE_ID = 0; +constexpr int64_t DEVICE_ID = 1; -class NSGInterfaceTest : public DataGen, public TestWithParam<::std::tuple> { + class NSGInterfaceTest : public DataGen, public ::testing::Test { protected: void SetUp() override { //Init_with_default(); FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID, 1024*1024*200, 1024*1024*600, 2); - Generate(256, 10000, 1); + Generate(256, 1000000, 1); index_ = std::make_shared(); - std::tie(train_cfg, search_cfg) = GetParam(); + + auto tmp_conf = std::make_shared(); + tmp_conf->gpu_id = DEVICE_ID; + tmp_conf->knng = 100; + tmp_conf->nprobe = 32; + tmp_conf->nlist = 16384; + tmp_conf->search_length = 60; + tmp_conf->out_degree = 70; + tmp_conf->candidate_pool_size = 500; + tmp_conf->metric_type = METRICTYPE::L2; + train_conf = tmp_conf; + + auto tmp2_conf = std::make_shared(); + tmp2_conf->k = k; + tmp2_conf->search_length = 30; + search_conf = tmp2_conf; } void TearDown() override { @@ -50,18 +66,10 @@ class NSGInterfaceTest : public DataGen, public TestWithParam<::std::tuple index_; - Config train_cfg; - Config search_cfg; + Config train_conf; + Config search_conf; }; -INSTANTIATE_TEST_CASE_P(NSGparameters, NSGInterfaceTest, - Values(std::make_tuple( - // search length > out_degree - Config::object{{"nlist", 128}, {"nprobe", 50}, {"knng", 100}, {"metric_type", "L2"}, - {"search_length", 60}, {"out_degree", 70}, {"candidate_pool_size", 500}}, - Config::object{{"k", 20}, {"search_length", 30}})) -); - void AssertAnns(const DatasetPtr &result, const int &nq, const int &k) { @@ -71,17 +79,17 @@ void AssertAnns(const DatasetPtr &result, } } -TEST_P(NSGInterfaceTest, basic_test) { +TEST_F(NSGInterfaceTest, basic_test) { assert(!xb.empty()); - auto model = index_->Train(base_dataset, train_cfg); - auto result = index_->Search(query_dataset, search_cfg); + auto model = index_->Train(base_dataset, train_conf); + auto result = index_->Search(query_dataset, search_conf); AssertAnns(result, nq, k); auto binaryset = index_->Serialize(); auto new_index = std::make_shared(); new_index->Load(binaryset); - auto new_result = new_index->Search(query_dataset, Config::object{{"k", k}}); + auto new_result = new_index->Search(query_dataset, search_conf); AssertAnns(result, nq, k); ASSERT_EQ(index_->Count(), nb); diff --git a/cpp/src/wrapper/ConfAdapter.cpp b/cpp/src/wrapper/ConfAdapter.cpp index ea6be1acf3..1ccbaee977 100644 --- a/cpp/src/wrapper/ConfAdapter.cpp +++ b/cpp/src/wrapper/ConfAdapter.cpp @@ -145,6 +145,10 @@ NSGConfAdapter::Match(const TempMetaConf &metaconf) { conf->out_degree = 50 + 5 * scale_factor; conf->candidate_pool_size = 200 + 100 * scale_factor; MatchBase(conf); + +// WRAPPER_LOG_DEBUG << "nlist: " << conf->nlist +// << ", gpu_id: " << conf->gpu_id << ", d: " << conf->d +// << ", nprobe: " << conf->nprobe << ", knng: " << conf->knng; return conf; } diff --git a/cpp/src/wrapper/ConfAdapter.h b/cpp/src/wrapper/ConfAdapter.h index d0f032cb61..d8281d8887 100644 --- a/cpp/src/wrapper/ConfAdapter.h +++ b/cpp/src/wrapper/ConfAdapter.h @@ -47,6 +47,9 @@ class ConfAdapter { virtual knowhere::Config MatchSearch(const TempMetaConf &metaconf, const IndexType &type); +// virtual void +// Dump(){} + protected: static void MatchBase(knowhere::Config conf); }; diff --git a/cpp/src/wrapper/VecIndex.cpp b/cpp/src/wrapper/VecIndex.cpp index 734f66338f..ffc14ebd87 100644 --- a/cpp/src/wrapper/VecIndex.cpp +++ b/cpp/src/wrapper/VecIndex.cpp @@ -137,7 +137,7 @@ GetVecIndexFactory(const IndexType &type, const Config &cfg) { index = std::make_shared(gpu_device); break; } - case IndexType::NSG_MIX: { // TODO(linxj): bug. + case IndexType::NSG_MIX: { index = std::make_shared(gpu_device); break; } diff --git a/cpp/unittest/wrapper/wrapper_test.cpp b/cpp/unittest/wrapper/wrapper_test.cpp index d19f98076c..43c4103a10 100644 --- a/cpp/unittest/wrapper/wrapper_test.cpp +++ b/cpp/unittest/wrapper/wrapper_test.cpp @@ -173,7 +173,7 @@ INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest, std::make_tuple(IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10), std::make_tuple(IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10), std::make_tuple(IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10), -// std::make_tuple(IndexType::NSG_MIX, "Default", 128, 250000, 10, 10), + std::make_tuple(IndexType::NSG_MIX, "Default", 128, 250000, 10, 10), // std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 250000, 10, 10), std::make_tuple(IndexType::FAISS_IDMAP, "Default", 64, 100000, 10, 10) ) From 58b82398cc15282e3ce456d0575e921106f9d117 Mon Sep 17 00:00:00 2001 From: starlord Date: Sat, 28 Sep 2019 10:28:51 +0800 Subject: [PATCH 141/217] format code by clang-format Former-commit-id: c817c67a409054a49da2be878872e885a0b3e041 --- cpp/build.sh | 14 +- cpp/src/cache/Cache.h | 56 +- cpp/src/cache/CacheMgr.h | 43 +- cpp/src/cache/CpuCacheMgr.cpp | 17 +- cpp/src/cache/CpuCacheMgr.h | 16 +- cpp/src/cache/DataObj.h | 23 +- cpp/src/cache/GpuCacheMgr.cpp | 19 +- cpp/src/cache/GpuCacheMgr.h | 15 +- cpp/src/cache/LRU.h | 44 +- cpp/src/config/ConfigMgr.cpp | 8 +- cpp/src/config/ConfigMgr.h | 26 +- cpp/src/config/ConfigNode.cpp | 80 +- cpp/src/config/ConfigNode.h | 82 +- cpp/src/config/YamlConfigMgr.cpp | 29 +- cpp/src/config/YamlConfigMgr.h | 42 +- cpp/src/db/Constants.h | 6 +- cpp/src/db/DB.h | 81 +- cpp/src/db/DBFactory.cpp | 15 +- cpp/src/db/DBFactory.h | 14 +- cpp/src/db/DBImpl.cpp | 347 ++++---- cpp/src/db/DBImpl.h | 150 ++-- cpp/src/db/IDGenerator.cpp | 18 +- cpp/src/db/IDGenerator.h | 25 +- cpp/src/db/Options.cpp | 26 +- cpp/src/db/Options.h | 42 +- cpp/src/db/Types.h | 14 +- cpp/src/db/Utils.cpp | 67 +- cpp/src/db/Utils.h | 28 +- cpp/src/db/engine/EngineFactory.cpp | 13 +- cpp/src/db/engine/EngineFactory.h | 16 +- cpp/src/db/engine/ExecutionEngine.h | 70 +- cpp/src/db/engine/ExecutionEngineImpl.cpp | 88 +- cpp/src/db/engine/ExecutionEngineImpl.h | 84 +- cpp/src/db/insert/MemManager.h | 32 +- cpp/src/db/insert/MemManagerImpl.cpp | 42 +- cpp/src/db/insert/MemManagerImpl.h | 51 +- cpp/src/db/insert/MemMenagerFactory.cpp | 16 +- cpp/src/db/insert/MemMenagerFactory.h | 9 +- cpp/src/db/insert/MemTable.cpp | 23 +- cpp/src/db/insert/MemTable.h | 36 +- cpp/src/db/insert/MemTableFile.cpp | 38 +- cpp/src/db/insert/MemTableFile.h | 33 +- cpp/src/db/insert/VectorSource.cpp | 32 +- cpp/src/db/insert/VectorSource.h | 32 +- cpp/src/db/meta/Meta.h | 101 ++- cpp/src/db/meta/MetaConsts.h | 8 +- cpp/src/db/meta/MetaFactory.cpp | 24 +- cpp/src/db/meta/MetaFactory.h | 12 +- cpp/src/db/meta/MetaTypes.h | 30 +- cpp/src/db/meta/MySQLConnectionPool.cpp | 31 +- cpp/src/db/meta/MySQLConnectionPool.h | 57 +- cpp/src/db/meta/MySQLMetaImpl.cpp | 836 +++++++++--------- cpp/src/db/meta/MySQLMetaImpl.h | 126 +-- cpp/src/db/meta/SqliteMetaImpl.h | 118 ++- cpp/src/main.cpp | 28 +- cpp/src/metrics/MetricBase.h | 140 ++- cpp/src/metrics/Metrics.cpp | 16 +- cpp/src/metrics/Metrics.h | 33 +- cpp/src/metrics/PrometheusMetrics.cpp | 48 +- cpp/src/metrics/PrometheusMetrics.h | 713 ++++++++------- cpp/src/metrics/SystemInfo.cpp | 52 +- cpp/src/metrics/SystemInfo.h | 87 +- cpp/src/scheduler/Algorithm.cpp | 19 +- cpp/src/scheduler/Algorithm.h | 17 +- cpp/src/scheduler/Definition.h | 24 +- cpp/src/scheduler/JobMgr.cpp | 21 +- cpp/src/scheduler/JobMgr.h | 28 +- cpp/src/scheduler/ResourceFactory.cpp | 12 +- cpp/src/scheduler/ResourceFactory.h | 17 +- cpp/src/scheduler/ResourceMgr.cpp | 38 +- cpp/src/scheduler/ResourceMgr.h | 22 +- cpp/src/scheduler/SchedInst.cpp | 153 ++-- cpp/src/scheduler/SchedInst.h | 10 +- cpp/src/scheduler/Scheduler.cpp | 34 +- cpp/src/scheduler/Scheduler.h | 28 +- cpp/src/scheduler/TaskCreator.cpp | 14 +- cpp/src/scheduler/TaskCreator.h | 34 +- cpp/src/scheduler/TaskTable.cpp | 53 +- cpp/src/scheduler/TaskTable.h | 61 +- cpp/src/scheduler/Utils.cpp | 9 +- cpp/src/scheduler/Utils.h | 7 +- cpp/src/scheduler/action/Action.h | 17 +- .../scheduler/action/PushTaskToNeighbour.cpp | 48 +- cpp/src/scheduler/event/Event.h | 23 +- cpp/src/scheduler/event/EventDump.cpp | 31 +- cpp/src/scheduler/event/FinishTaskEvent.h | 15 +- cpp/src/scheduler/event/LoadCompletedEvent.h | 16 +- cpp/src/scheduler/event/StartUpEvent.h | 14 +- .../scheduler/event/TaskTableUpdatedEvent.h | 11 +- cpp/src/scheduler/job/DeleteJob.cpp | 15 +- cpp/src/scheduler/job/DeleteJob.h | 27 +- cpp/src/scheduler/job/Job.h | 22 +- cpp/src/scheduler/job/SearchJob.cpp | 29 +- cpp/src/scheduler/job/SearchJob.h | 36 +- cpp/src/scheduler/resource/Connection.h | 13 +- cpp/src/scheduler/resource/CpuResource.cpp | 11 +- cpp/src/scheduler/resource/CpuResource.h | 12 +- cpp/src/scheduler/resource/DiskResource.cpp | 10 +- cpp/src/scheduler/resource/DiskResource.h | 12 +- cpp/src/scheduler/resource/GpuResource.cpp | 11 +- cpp/src/scheduler/resource/GpuResource.h | 12 +- cpp/src/scheduler/resource/Node.cpp | 12 +- cpp/src/scheduler/resource/Node.h | 17 +- cpp/src/scheduler/resource/Resource.cpp | 32 +- cpp/src/scheduler/resource/Resource.h | 39 +- cpp/src/scheduler/resource/TestResource.cpp | 10 +- cpp/src/scheduler/resource/TestResource.h | 14 +- cpp/src/scheduler/task/DeleteTask.cpp | 9 +- cpp/src/scheduler/task/DeleteTask.h | 10 +- cpp/src/scheduler/task/Path.h | 21 +- cpp/src/scheduler/task/SearchTask.cpp | 142 ++- cpp/src/scheduler/task/SearchTask.h | 29 +- cpp/src/scheduler/task/Task.h | 20 +- cpp/src/scheduler/task/TestTask.cpp | 12 +- cpp/src/scheduler/task/TestTask.h | 8 +- cpp/src/scheduler/tasklabel/BroadcastLabel.h | 7 +- cpp/src/scheduler/tasklabel/DefaultLabel.h | 6 +- cpp/src/scheduler/tasklabel/SpecResLabel.h | 14 +- cpp/src/scheduler/tasklabel/TaskLabel.h | 12 +- cpp/src/sdk/examples/grpcsimple/main.cpp | 14 +- .../examples/grpcsimple/src/ClientTest.cpp | 113 ++- .../sdk/examples/grpcsimple/src/ClientTest.h | 3 +- cpp/src/sdk/grpc/ClientProxy.cpp | 136 ++- cpp/src/sdk/grpc/ClientProxy.h | 49 +- cpp/src/sdk/grpc/GrpcClient.cpp | 53 +- cpp/src/sdk/grpc/GrpcClient.h | 39 +- cpp/src/sdk/include/MilvusApi.h | 73 +- cpp/src/sdk/include/Status.h | 27 +- cpp/src/sdk/interface/ConnectionImpl.cpp | 49 +- cpp/src/sdk/interface/ConnectionImpl.h | 45 +- cpp/src/sdk/interface/Status.cpp | 39 +- cpp/src/server/Config.cpp | 253 +++--- cpp/src/server/Config.h | 422 +++++---- cpp/src/server/DBWrapper.cpp | 36 +- cpp/src/server/DBWrapper.h | 23 +- cpp/src/server/Server.cpp | 38 +- cpp/src/server/Server.h | 34 +- .../server/grpc_impl/GrpcRequestHandler.cpp | 86 +- cpp/src/server/grpc_impl/GrpcRequestHandler.h | 80 +- .../server/grpc_impl/GrpcRequestScheduler.cpp | 48 +- .../server/grpc_impl/GrpcRequestScheduler.h | 63 +- cpp/src/server/grpc_impl/GrpcRequestTask.cpp | 336 ++++--- cpp/src/server/grpc_impl/GrpcRequestTask.h | 104 +-- cpp/src/server/grpc_impl/GrpcServer.cpp | 24 +- cpp/src/server/grpc_impl/GrpcServer.h | 27 +- cpp/src/utils/BlockingQueue.h | 34 +- cpp/src/utils/CommonUtil.cpp | 78 +- cpp/src/utils/CommonUtil.h | 49 +- cpp/src/utils/Error.h | 17 +- cpp/src/utils/Exception.h | 20 +- cpp/src/utils/Log.h | 4 +- cpp/src/utils/LogUtil.cpp | 24 +- cpp/src/utils/LogUtil.h | 12 +- cpp/src/utils/SignalUtil.cpp | 18 +- cpp/src/utils/SignalUtil.h | 12 +- cpp/src/utils/Status.cpp | 60 +- cpp/src/utils/Status.h | 29 +- cpp/src/utils/StringHelpFunctions.cpp | 21 +- cpp/src/utils/StringHelpFunctions.h | 30 +- cpp/src/utils/ThreadPool.h | 80 +- cpp/src/utils/TimeRecorder.cpp | 15 +- cpp/src/utils/TimeRecorder.h | 23 +- cpp/src/utils/ValidationUtil.cpp | 77 +- cpp/src/utils/ValidationUtil.h | 25 +- 164 files changed, 4353 insertions(+), 4311 deletions(-) mode change 100755 => 100644 cpp/src/utils/CommonUtil.h diff --git a/cpp/build.sh b/cpp/build.sh index 8b2f0b142b..69fd4f0f29 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -104,13 +104,13 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then fi echo "cpplint check passed!" -# # clang-format check -# make check-clang-format -# if [ $? -ne 0 ]; then -# echo "ERROR! clang-format check failed" -# exit 1 -# fi -# echo "clang-format check passed!" + # clang-format check + make check-clang-format + if [ $? -ne 0 ]; then + echo "ERROR! clang-format check failed" + exit 1 + fi + echo "clang-format check passed!" # # # clang-tidy check # make check-clang-tidy diff --git a/cpp/src/cache/Cache.h b/cpp/src/cache/Cache.h index b526e9339f..7818a2e6b7 100644 --- a/cpp/src/cache/Cache.h +++ b/cpp/src/cache/Cache.h @@ -15,55 +15,67 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "LRU.h" #include "utils/Log.h" -#include -#include #include +#include #include +#include namespace zilliz { namespace milvus { namespace cache { -template +template class Cache { public: - //mem_capacity, units:GB + // mem_capacity, units:GB Cache(int64_t capacity_gb, uint64_t cache_max_count); ~Cache() = default; - int64_t usage() const { + int64_t + usage() const { return usage_; } - int64_t capacity() const { + int64_t + capacity() const { return capacity_; - } //unit: BYTE - void set_capacity(int64_t capacity); //unit: BYTE + } // unit: BYTE + void + set_capacity(int64_t capacity); // unit: BYTE - double freemem_percent() const { + double + freemem_percent() const { return freemem_percent_; } - void set_freemem_percent(double percent) { + void + set_freemem_percent(double percent) { freemem_percent_ = percent; } - size_t size() const; - bool exists(const std::string &key); - ItemObj get(const std::string &key); - void insert(const std::string &key, const ItemObj &item); - void erase(const std::string &key); - void print(); - void clear(); + size_t + size() const; + bool + exists(const std::string& key); + ItemObj + get(const std::string& key); + void + insert(const std::string& key, const ItemObj& item); + void + erase(const std::string& key); + void + print(); + void + clear(); private: - void free_memory(); + void + free_memory(); private: int64_t usage_; @@ -74,8 +86,8 @@ class Cache { mutable std::mutex mutex_; }; -} // namespace cache -} // namespace milvus -} // namespace zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz #include "cache/Cache.inl" diff --git a/cpp/src/cache/CacheMgr.h b/cpp/src/cache/CacheMgr.h index 53004d10b2..11cd742a6a 100644 --- a/cpp/src/cache/CacheMgr.h +++ b/cpp/src/cache/CacheMgr.h @@ -15,40 +15,49 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "Cache.h" -#include "utils/Log.h" #include "metrics/Metrics.h" +#include "utils/Log.h" -#include #include +#include namespace zilliz { namespace milvus { namespace cache { -template +template class CacheMgr { public: - virtual uint64_t ItemCount() const; + virtual uint64_t + ItemCount() const; - virtual bool ItemExists(const std::string &key); + virtual bool + ItemExists(const std::string& key); - virtual ItemObj GetItem(const std::string &key); + virtual ItemObj + GetItem(const std::string& key); - virtual void InsertItem(const std::string &key, const ItemObj &data); + virtual void + InsertItem(const std::string& key, const ItemObj& data); - virtual void EraseItem(const std::string &key); + virtual void + EraseItem(const std::string& key); - virtual void PrintInfo(); + virtual void + PrintInfo(); - virtual void ClearCache(); + virtual void + ClearCache(); - int64_t CacheUsage() const; - int64_t CacheCapacity() const; - void SetCapacity(int64_t capacity); + int64_t + CacheUsage() const; + int64_t + CacheCapacity() const; + void + SetCapacity(int64_t capacity); protected: CacheMgr(); @@ -59,8 +68,8 @@ class CacheMgr { CachePtr cache_; }; -} // namespace cache -} // namespace milvus -} // namespace zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz #include "cache/CacheMgr.inl" diff --git a/cpp/src/cache/CpuCacheMgr.cpp b/cpp/src/cache/CpuCacheMgr.cpp index 7f53493017..99a284f074 100644 --- a/cpp/src/cache/CpuCacheMgr.cpp +++ b/cpp/src/cache/CpuCacheMgr.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include "cache/CpuCacheMgr.h" #include "server/Config.h" #include "utils/Log.h" @@ -31,7 +30,7 @@ constexpr int64_t unit = 1024 * 1024 * 1024; } CpuCacheMgr::CpuCacheMgr() { - server::Config &config = server::Config::GetInstance(); + server::Config& config = server::Config::GetInstance(); Status s; int32_t cpu_cache_cap; @@ -50,19 +49,19 @@ CpuCacheMgr::CpuCacheMgr() { if (cpu_cache_threshold > 0.0 && cpu_cache_threshold <= 1.0) { cache_->set_freemem_percent(cpu_cache_threshold); } else { - SERVER_LOG_ERROR << "Invalid cpu_cache_threshold: " << cpu_cache_threshold - << ", by default set to " << cache_->freemem_percent(); + SERVER_LOG_ERROR << "Invalid cpu_cache_threshold: " << cpu_cache_threshold << ", by default set to " + << cache_->freemem_percent(); } } -CpuCacheMgr * +CpuCacheMgr* CpuCacheMgr::GetInstance() { static CpuCacheMgr s_mgr; return &s_mgr; } engine::VecIndexPtr -CpuCacheMgr::GetIndex(const std::string &key) { +CpuCacheMgr::GetIndex(const std::string& key) { DataObjPtr obj = GetItem(key); if (obj != nullptr) { return obj->data(); @@ -71,6 +70,6 @@ CpuCacheMgr::GetIndex(const std::string &key) { return nullptr; } -} // namespace cache -} // namespace milvus -} // namespace zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/CpuCacheMgr.h b/cpp/src/cache/CpuCacheMgr.h index 32a83c4cc0..fbc4f6dfb3 100644 --- a/cpp/src/cache/CpuCacheMgr.h +++ b/cpp/src/cache/CpuCacheMgr.h @@ -20,8 +20,8 @@ #include "CacheMgr.h" #include "DataObj.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -32,12 +32,14 @@ class CpuCacheMgr : public CacheMgr { CpuCacheMgr(); public: - //TODO: use smart pointer instead - static CpuCacheMgr *GetInstance(); + // TODO: use smart pointer instead + static CpuCacheMgr* + GetInstance(); - engine::VecIndexPtr GetIndex(const std::string &key); + engine::VecIndexPtr + GetIndex(const std::string& key); }; -} // namespace cache -} // namespace milvus -} // namespace zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/DataObj.h b/cpp/src/cache/DataObj.h index 6ed3256eee..80cc560e7c 100644 --- a/cpp/src/cache/DataObj.h +++ b/cpp/src/cache/DataObj.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "src/wrapper/VecIndex.h" @@ -28,24 +27,24 @@ namespace cache { class DataObj { public: - explicit DataObj(const engine::VecIndexPtr &index) - : index_(index) { + explicit DataObj(const engine::VecIndexPtr& index) : index_(index) { } - DataObj(const engine::VecIndexPtr &index, int64_t size) - : index_(index), - size_(size) { + DataObj(const engine::VecIndexPtr& index, int64_t size) : index_(index), size_(size) { } - engine::VecIndexPtr data() { + engine::VecIndexPtr + data() { return index_; } - const engine::VecIndexPtr &data() const { + const engine::VecIndexPtr& + data() const { return index_; } - int64_t size() const { + int64_t + size() const { if (index_ == nullptr) { return 0; } @@ -64,6 +63,6 @@ class DataObj { using DataObjPtr = std::shared_ptr; -} // namespace cache -} // namespace milvus -} // namespace zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/GpuCacheMgr.cpp b/cpp/src/cache/GpuCacheMgr.cpp index f19e70de6b..9c9e94eda9 100644 --- a/cpp/src/cache/GpuCacheMgr.cpp +++ b/cpp/src/cache/GpuCacheMgr.cpp @@ -15,10 +15,9 @@ // specific language governing permissions and limitations // under the License. - #include "cache/GpuCacheMgr.h" -#include "utils/Log.h" #include "server/Config.h" +#include "utils/Log.h" #include #include @@ -35,7 +34,7 @@ constexpr int64_t G_BYTE = 1024 * 1024 * 1024; } GpuCacheMgr::GpuCacheMgr() { - server::Config &config = server::Config::GetInstance(); + server::Config& config = server::Config::GetInstance(); Status s; int32_t gpu_cache_cap; @@ -54,12 +53,12 @@ GpuCacheMgr::GpuCacheMgr() { if (gpu_mem_threshold > 0.0 && gpu_mem_threshold <= 1.0) { cache_->set_freemem_percent(gpu_mem_threshold); } else { - SERVER_LOG_ERROR << "Invalid gpu_mem_threshold: " << gpu_mem_threshold - << ", by default set to " << cache_->freemem_percent(); + SERVER_LOG_ERROR << "Invalid gpu_mem_threshold: " << gpu_mem_threshold << ", by default set to " + << cache_->freemem_percent(); } } -GpuCacheMgr * +GpuCacheMgr* GpuCacheMgr::GetInstance(uint64_t gpu_id) { if (instance_.find(gpu_id) == instance_.end()) { std::lock_guard lock(mutex_); @@ -74,7 +73,7 @@ GpuCacheMgr::GetInstance(uint64_t gpu_id) { } engine::VecIndexPtr -GpuCacheMgr::GetIndex(const std::string &key) { +GpuCacheMgr::GetIndex(const std::string& key) { DataObjPtr obj = GetItem(key); if (obj != nullptr) { return obj->data(); @@ -83,6 +82,6 @@ GpuCacheMgr::GetIndex(const std::string &key) { return nullptr; } -} // namespace cache -} // namespace milvus -} // namespace zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/GpuCacheMgr.h b/cpp/src/cache/GpuCacheMgr.h index 843a5ff67d..15cdcae28f 100644 --- a/cpp/src/cache/GpuCacheMgr.h +++ b/cpp/src/cache/GpuCacheMgr.h @@ -15,13 +15,12 @@ // specific language governing permissions and limitations // under the License. - #include "CacheMgr.h" #include "DataObj.h" -#include #include #include +#include namespace zilliz { namespace milvus { @@ -34,15 +33,17 @@ class GpuCacheMgr : public CacheMgr { public: GpuCacheMgr(); - static GpuCacheMgr *GetInstance(uint64_t gpu_id); + static GpuCacheMgr* + GetInstance(uint64_t gpu_id); - engine::VecIndexPtr GetIndex(const std::string &key); + engine::VecIndexPtr + GetIndex(const std::string& key); private: static std::mutex mutex_; static std::unordered_map instance_; }; -} // namespace cache -} // namespace milvus -} // namespace zilliz +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/cache/LRU.h b/cpp/src/cache/LRU.h index 5446dd0f14..d6a5b9ace7 100644 --- a/cpp/src/cache/LRU.h +++ b/cpp/src/cache/LRU.h @@ -15,20 +15,19 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include -#include #include +#include #include +#include #include namespace zilliz { namespace milvus { namespace cache { -template +template class LRU { public: typedef typename std::pair key_value_pair_t; @@ -38,7 +37,8 @@ class LRU { explicit LRU(size_t max_size) : max_size_(max_size) { } - void put(const key_t &key, const value_t &value) { + void + put(const key_t& key, const value_t& value) { auto it = cache_items_map_.find(key); cache_items_list_.push_front(key_value_pair_t(key, value)); if (it != cache_items_map_.end()) { @@ -55,7 +55,8 @@ class LRU { } } - const value_t &get(const key_t &key) { + const value_t& + get(const key_t& key) { auto it = cache_items_map_.find(key); if (it == cache_items_map_.end()) { throw std::range_error("There is no such key in cache"); @@ -65,7 +66,8 @@ class LRU { } } - void erase(const key_t &key) { + void + erase(const key_t& key) { auto it = cache_items_map_.find(key); if (it != cache_items_map_.end()) { cache_items_list_.erase(it->second); @@ -73,32 +75,39 @@ class LRU { } } - bool exists(const key_t &key) const { + bool + exists(const key_t& key) const { return cache_items_map_.find(key) != cache_items_map_.end(); } - size_t size() const { + size_t + size() const { return cache_items_map_.size(); } - list_iterator_t begin() { + list_iterator_t + begin() { iter_ = cache_items_list_.begin(); return iter_; } - list_iterator_t end() { + list_iterator_t + end() { return cache_items_list_.end(); } - reverse_list_iterator_t rbegin() { + reverse_list_iterator_t + rbegin() { return cache_items_list_.rbegin(); } - reverse_list_iterator_t rend() { + reverse_list_iterator_t + rend() { return cache_items_list_.rend(); } - void clear() { + void + clear() { cache_items_list_.clear(); cache_items_map_.clear(); } @@ -110,7 +119,6 @@ class LRU { list_iterator_t iter_; }; -} // namespace cache -} // namespace milvus -} // namespace zilliz - +} // namespace cache +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/ConfigMgr.cpp b/cpp/src/config/ConfigMgr.cpp index a7cfdc9fda..e006217fd2 100644 --- a/cpp/src/config/ConfigMgr.cpp +++ b/cpp/src/config/ConfigMgr.cpp @@ -22,12 +22,12 @@ namespace zilliz { namespace milvus { namespace server { -ConfigMgr * +ConfigMgr* ConfigMgr::GetInstance() { static YamlConfigMgr mgr; return &mgr; } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/ConfigMgr.h b/cpp/src/config/ConfigMgr.h index 96a63aa2c8..e5336ee34d 100644 --- a/cpp/src/config/ConfigMgr.h +++ b/cpp/src/config/ConfigMgr.h @@ -17,8 +17,8 @@ #pragma once -#include "utils/Error.h" #include "ConfigNode.h" +#include "utils/Error.h" #include @@ -42,16 +42,22 @@ namespace server { class ConfigMgr { public: - static ConfigMgr *GetInstance(); + static ConfigMgr* + GetInstance(); - virtual ErrorCode LoadConfigFile(const std::string &filename) = 0; - virtual void Print() const = 0;//will be deleted - virtual std::string DumpString() const = 0; + virtual ErrorCode + LoadConfigFile(const std::string& filename) = 0; + virtual void + Print() const = 0; // will be deleted + virtual std::string + DumpString() const = 0; - virtual const ConfigNode &GetRootNode() const = 0; - virtual ConfigNode &GetRootNode() = 0; + virtual const ConfigNode& + GetRootNode() const = 0; + virtual ConfigNode& + GetRootNode() = 0; }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/ConfigNode.cpp b/cpp/src/config/ConfigNode.cpp index 87b080d572..ff27a0af75 100644 --- a/cpp/src/config/ConfigNode.cpp +++ b/cpp/src/config/ConfigNode.cpp @@ -19,51 +19,51 @@ #include "utils/Error.h" #include "utils/Log.h" +#include #include #include -#include namespace zilliz { namespace milvus { namespace server { void -ConfigNode::Combine(const ConfigNode &target) { - const std::map &kv = target.GetConfig(); +ConfigNode::Combine(const ConfigNode& target) { + const std::map& kv = target.GetConfig(); for (auto itr = kv.begin(); itr != kv.end(); ++itr) { config_[itr->first] = itr->second; } - const std::map > &sequences = target.GetSequences(); + const std::map >& sequences = target.GetSequences(); for (auto itr = sequences.begin(); itr != sequences.end(); ++itr) { sequences_[itr->first] = itr->second; } - const std::map &children = target.GetChildren(); + const std::map& children = target.GetChildren(); for (auto itr = children.begin(); itr != children.end(); ++itr) { children_[itr->first] = itr->second; } } -//key/value pair config +// key/value pair config void -ConfigNode::SetValue(const std::string &key, const std::string &value) { +ConfigNode::SetValue(const std::string& key, const std::string& value) { config_[key] = value; } std::string -ConfigNode::GetValue(const std::string ¶m_key, const std::string &default_val) const { +ConfigNode::GetValue(const std::string& param_key, const std::string& default_val) const { auto ref = config_.find(param_key); if (ref != config_.end()) { return ref->second; } - //THROW_UNEXPECTED_ERROR("Can't find parameter key: " + param_key); + // THROW_UNEXPECTED_ERROR("Can't find parameter key: " + param_key); return default_val; } bool -ConfigNode::GetBoolValue(const std::string ¶m_key, bool default_val) const { +ConfigNode::GetBoolValue(const std::string& param_key, bool default_val) const { std::string val = GetValue(param_key); if (!val.empty()) { std::transform(val.begin(), val.end(), val.begin(), ::tolower); @@ -74,17 +74,17 @@ ConfigNode::GetBoolValue(const std::string ¶m_key, bool default_val) const { } int32_t -ConfigNode::GetInt32Value(const std::string ¶m_key, int32_t default_val) const { +ConfigNode::GetInt32Value(const std::string& param_key, int32_t default_val) const { std::string val = GetValue(param_key); if (!val.empty()) { - return (int32_t) std::strtol(val.c_str(), nullptr, 10); + return (int32_t)std::strtol(val.c_str(), nullptr, 10); } else { return default_val; } } int64_t -ConfigNode::GetInt64Value(const std::string ¶m_key, int64_t default_val) const { +ConfigNode::GetInt64Value(const std::string& param_key, int64_t default_val) const { std::string val = GetValue(param_key); if (!val.empty()) { return std::strtol(val.c_str(), nullptr, 10); @@ -94,7 +94,7 @@ ConfigNode::GetInt64Value(const std::string ¶m_key, int64_t default_val) con } float -ConfigNode::GetFloatValue(const std::string ¶m_key, float default_val) const { +ConfigNode::GetFloatValue(const std::string& param_key, float default_val) const { std::string val = GetValue(param_key); if (!val.empty()) { return std::strtof(val.c_str(), nullptr); @@ -104,7 +104,7 @@ ConfigNode::GetFloatValue(const std::string ¶m_key, float default_val) const } double -ConfigNode::GetDoubleValue(const std::string ¶m_key, double default_val) const { +ConfigNode::GetDoubleValue(const std::string& param_key, double default_val) const { std::string val = GetValue(param_key); if (!val.empty()) { return std::strtod(val.c_str(), nullptr); @@ -113,7 +113,7 @@ ConfigNode::GetDoubleValue(const std::string ¶m_key, double default_val) con } } -const std::map & +const std::map& ConfigNode::GetConfig() const { return config_; } @@ -123,14 +123,14 @@ ConfigNode::ClearConfig() { config_.clear(); } -//key/object config +// key/object config void -ConfigNode::AddChild(const std::string &type_name, const ConfigNode &config) { +ConfigNode::AddChild(const std::string& type_name, const ConfigNode& config) { children_[type_name] = config; } ConfigNode -ConfigNode::GetChild(const std::string &type_name) const { +ConfigNode::GetChild(const std::string& type_name) const { auto ref = children_.find(type_name); if (ref != children_.end()) { return ref->second; @@ -140,20 +140,20 @@ ConfigNode::GetChild(const std::string &type_name) const { return nc; } -ConfigNode & -ConfigNode::GetChild(const std::string &type_name) { +ConfigNode& +ConfigNode::GetChild(const std::string& type_name) { return children_[type_name]; } void -ConfigNode::GetChildren(ConfigNodeArr &arr) const { +ConfigNode::GetChildren(ConfigNodeArr& arr) const { arr.clear(); for (auto ref : children_) { arr.push_back(ref.second); } } -const std::map & +const std::map& ConfigNode::GetChildren() const { return children_; } @@ -163,14 +163,14 @@ ConfigNode::ClearChildren() { children_.clear(); } -//key/sequence config +// key/sequence config void -ConfigNode::AddSequenceItem(const std::string &key, const std::string &item) { +ConfigNode::AddSequenceItem(const std::string& key, const std::string& item) { sequences_[key].push_back(item); } std::vector -ConfigNode::GetSequence(const std::string &key) const { +ConfigNode::GetSequence(const std::string& key) const { auto itr = sequences_.find(key); if (itr != sequences_.end()) { return itr->second; @@ -180,7 +180,7 @@ ConfigNode::GetSequence(const std::string &key) const { } } -const std::map > & +const std::map >& ConfigNode::GetSequences() const { return sequences_; } @@ -191,40 +191,40 @@ ConfigNode::ClearSequences() { } void -ConfigNode::PrintAll(const std::string &prefix) const { - for (auto &elem : config_) { +ConfigNode::PrintAll(const std::string& prefix) const { + for (auto& elem : config_) { SERVER_LOG_INFO << prefix << elem.first + ": " << elem.second; } - for (auto &elem : sequences_) { + for (auto& elem : sequences_) { SERVER_LOG_INFO << prefix << elem.first << ": "; - for (auto &str : elem.second) { + for (auto& str : elem.second) { SERVER_LOG_INFO << prefix << " - " << str; } } - for (auto &elem : children_) { + for (auto& elem : children_) { SERVER_LOG_INFO << prefix << elem.first << ": "; elem.second.PrintAll(prefix + " "); } } std::string -ConfigNode::DumpString(const std::string &prefix) const { +ConfigNode::DumpString(const std::string& prefix) const { std::stringstream str_buffer; const std::string endl = "\n"; - for (auto &elem : config_) { + for (auto& elem : config_) { str_buffer << prefix << elem.first << ": " << elem.second << endl; } - for (auto &elem : sequences_) { + for (auto& elem : sequences_) { str_buffer << prefix << elem.first << ": " << endl; - for (auto &str : elem.second) { + for (auto& str : elem.second) { str_buffer << prefix + " - " << str << endl; } } - for (auto &elem : children_) { + for (auto& elem : children_) { str_buffer << prefix << elem.first << ": " << endl; str_buffer << elem.second.DumpString(prefix + " ") << endl; } @@ -232,6 +232,6 @@ ConfigNode::DumpString(const std::string &prefix) const { return str_buffer.str(); } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/ConfigNode.h b/cpp/src/config/ConfigNode.h index d3fabc0655..937066dbf5 100644 --- a/cpp/src/config/ConfigNode.h +++ b/cpp/src/config/ConfigNode.h @@ -17,9 +17,9 @@ #pragma once -#include -#include #include +#include +#include namespace zilliz { namespace milvus { @@ -30,39 +30,61 @@ typedef std::vector ConfigNodeArr; class ConfigNode { public: - void Combine(const ConfigNode &target); + void + Combine(const ConfigNode& target); - //key/value pair config - void SetValue(const std::string &key, const std::string &value); + // key/value pair config + void + SetValue(const std::string& key, const std::string& value); - std::string GetValue(const std::string ¶m_key, const std::string &default_val = "") const; - bool GetBoolValue(const std::string ¶m_key, bool default_val = false) const; - int32_t GetInt32Value(const std::string ¶m_key, int32_t default_val = 0) const; - int64_t GetInt64Value(const std::string ¶m_key, int64_t default_val = 0) const; - float GetFloatValue(const std::string ¶m_key, float default_val = 0.0) const; - double GetDoubleValue(const std::string ¶m_key, double default_val = 0.0) const; + std::string + GetValue(const std::string& param_key, const std::string& default_val = "") const; + bool + GetBoolValue(const std::string& param_key, bool default_val = false) const; + int32_t + GetInt32Value(const std::string& param_key, int32_t default_val = 0) const; + int64_t + GetInt64Value(const std::string& param_key, int64_t default_val = 0) const; + float + GetFloatValue(const std::string& param_key, float default_val = 0.0) const; + double + GetDoubleValue(const std::string& param_key, double default_val = 0.0) const; - const std::map &GetConfig() const; - void ClearConfig(); + const std::map& + GetConfig() const; + void + ClearConfig(); - //key/object config - void AddChild(const std::string &type_name, const ConfigNode &config); - ConfigNode GetChild(const std::string &type_name) const; - ConfigNode &GetChild(const std::string &type_name); - void GetChildren(ConfigNodeArr &arr) const; + // key/object config + void + AddChild(const std::string& type_name, const ConfigNode& config); + ConfigNode + GetChild(const std::string& type_name) const; + ConfigNode& + GetChild(const std::string& type_name); + void + GetChildren(ConfigNodeArr& arr) const; - const std::map &GetChildren() const; - void ClearChildren(); + const std::map& + GetChildren() const; + void + ClearChildren(); - //key/sequence config - void AddSequenceItem(const std::string &key, const std::string &item); - std::vector GetSequence(const std::string &key) const; + // key/sequence config + void + AddSequenceItem(const std::string& key, const std::string& item); + std::vector + GetSequence(const std::string& key) const; - const std::map > &GetSequences() const; - void ClearSequences(); + const std::map >& + GetSequences() const; + void + ClearSequences(); - void PrintAll(const std::string &prefix = "") const; - std::string DumpString(const std::string &prefix = "") const; + void + PrintAll(const std::string& prefix = "") const; + std::string + DumpString(const std::string& prefix = "") const; private: std::map config_; @@ -70,6 +92,6 @@ class ConfigNode { std::map > sequences_; }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/YamlConfigMgr.cpp b/cpp/src/config/YamlConfigMgr.cpp index 1dd8a3d943..02653d6fad 100644 --- a/cpp/src/config/YamlConfigMgr.cpp +++ b/cpp/src/config/YamlConfigMgr.cpp @@ -25,7 +25,7 @@ namespace milvus { namespace server { ErrorCode -YamlConfigMgr::LoadConfigFile(const std::string &filename) { +YamlConfigMgr::LoadConfigFile(const std::string& filename) { struct stat directoryStat; int statOK = stat(filename.c_str(), &directoryStat); if (statOK != 0) { @@ -36,8 +36,7 @@ YamlConfigMgr::LoadConfigFile(const std::string &filename) { try { node_ = YAML::LoadFile(filename); LoadConfigNode(node_, config_); - } - catch (YAML::Exception &e) { + } catch (YAML::Exception& e) { SERVER_LOG_ERROR << "Failed to load config file: " << std::string(e.what()); return SERVER_UNEXPECTED_ERROR; } @@ -56,20 +55,18 @@ YamlConfigMgr::DumpString() const { return config_.DumpString(""); } -const ConfigNode & +const ConfigNode& YamlConfigMgr::GetRootNode() const { return config_; } -ConfigNode & +ConfigNode& YamlConfigMgr::GetRootNode() { return config_; } bool -YamlConfigMgr::SetConfigValue(const YAML::Node &node, - const std::string &key, - ConfigNode &config) { +YamlConfigMgr::SetConfigValue(const YAML::Node& node, const std::string& key, ConfigNode& config) { if (node[key].IsDefined()) { config.SetValue(key, node[key].as()); return true; @@ -78,9 +75,7 @@ YamlConfigMgr::SetConfigValue(const YAML::Node &node, } bool -YamlConfigMgr::SetChildConfig(const YAML::Node &node, - const std::string &child_name, - ConfigNode &config) { +YamlConfigMgr::SetChildConfig(const YAML::Node& node, const std::string& child_name, ConfigNode& config) { if (node[child_name].IsDefined()) { ConfigNode sub_config; LoadConfigNode(node[child_name], sub_config); @@ -91,9 +86,7 @@ YamlConfigMgr::SetChildConfig(const YAML::Node &node, } bool -YamlConfigMgr::SetSequence(const YAML::Node &node, - const std::string &child_name, - ConfigNode &config) { +YamlConfigMgr::SetSequence(const YAML::Node& node, const std::string& child_name, ConfigNode& config) { if (node[child_name].IsDefined()) { size_t cnt = node[child_name].size(); for (size_t i = 0; i < cnt; i++) { @@ -105,7 +98,7 @@ YamlConfigMgr::SetSequence(const YAML::Node &node, } void -YamlConfigMgr::LoadConfigNode(const YAML::Node &node, ConfigNode &config) { +YamlConfigMgr::LoadConfigNode(const YAML::Node& node, ConfigNode& config) { std::string key; for (YAML::const_iterator it = node.begin(); it != node.end(); ++it) { if (!it->first.IsNull()) { @@ -121,6 +114,6 @@ YamlConfigMgr::LoadConfigNode(const YAML::Node &node, ConfigNode &config) { } } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/config/YamlConfigMgr.h b/cpp/src/config/YamlConfigMgr.h index 6b84943bf8..9697ce3cae 100644 --- a/cpp/src/config/YamlConfigMgr.h +++ b/cpp/src/config/YamlConfigMgr.h @@ -21,8 +21,8 @@ #include "ConfigNode.h" #include "utils/Error.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -30,34 +30,36 @@ namespace server { class YamlConfigMgr : public ConfigMgr { public: - virtual ErrorCode LoadConfigFile(const std::string &filename); - virtual void Print() const; - virtual std::string DumpString() const; + virtual ErrorCode + LoadConfigFile(const std::string& filename); + virtual void + Print() const; + virtual std::string + DumpString() const; - virtual const ConfigNode &GetRootNode() const; - virtual ConfigNode &GetRootNode(); + virtual const ConfigNode& + GetRootNode() const; + virtual ConfigNode& + GetRootNode(); private: - bool SetConfigValue(const YAML::Node &node, - const std::string &key, - ConfigNode &config); - - bool SetChildConfig(const YAML::Node &node, - const std::string &name, - ConfigNode &config); + bool + SetConfigValue(const YAML::Node& node, const std::string& key, ConfigNode& config); bool - SetSequence(const YAML::Node &node, - const std::string &child_name, - ConfigNode &config); + SetChildConfig(const YAML::Node& node, const std::string& name, ConfigNode& config); - void LoadConfigNode(const YAML::Node &node, ConfigNode &config); + bool + SetSequence(const YAML::Node& node, const std::string& child_name, ConfigNode& config); + + void + LoadConfigNode(const YAML::Node& node, ConfigNode& config); private: YAML::Node node_; ConfigNode config_; }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/Constants.h b/cpp/src/db/Constants.h index 2beb3c3a97..0feffadaf4 100644 --- a/cpp/src/db/Constants.h +++ b/cpp/src/db/Constants.h @@ -36,6 +36,6 @@ static constexpr uint64_t ONE_KB = K; static constexpr uint64_t ONE_MB = ONE_KB * ONE_KB; static constexpr uint64_t ONE_GB = ONE_KB * ONE_MB; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index 127812fc30..9a6a2e74a6 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "Options.h" @@ -23,8 +22,8 @@ #include "meta/Meta.h" #include "utils/Status.h" -#include #include +#include #include namespace zilliz { @@ -36,47 +35,65 @@ class Env; class DB { public: DB() = default; - DB(const DB &) = delete; - DB &operator=(const DB &) = delete; + DB(const DB&) = delete; + DB& + operator=(const DB&) = delete; virtual ~DB() = default; - virtual Status Start() = 0; - virtual Status Stop() = 0; + virtual Status + Start() = 0; + virtual Status + Stop() = 0; - virtual Status CreateTable(meta::TableSchema &table_schema_) = 0; - virtual Status DeleteTable(const std::string &table_id, const meta::DatesT &dates) = 0; - virtual Status DescribeTable(meta::TableSchema &table_schema_) = 0; - virtual Status HasTable(const std::string &table_id, bool &has_or_not_) = 0; - virtual Status AllTables(std::vector &table_schema_array) = 0; - virtual Status GetTableRowCount(const std::string &table_id, uint64_t &row_count) = 0; - virtual Status PreloadTable(const std::string &table_id) = 0; - virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0; + virtual Status + CreateTable(meta::TableSchema& table_schema_) = 0; + virtual Status + DeleteTable(const std::string& table_id, const meta::DatesT& dates) = 0; + virtual Status + DescribeTable(meta::TableSchema& table_schema_) = 0; + virtual Status + HasTable(const std::string& table_id, bool& has_or_not_) = 0; + virtual Status + AllTables(std::vector& table_schema_array) = 0; + virtual Status + GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0; + virtual Status + PreloadTable(const std::string& table_id) = 0; + virtual Status + UpdateTableFlag(const std::string& table_id, int64_t flag) = 0; - virtual Status InsertVectors(const std::string &table_id_, - uint64_t n, const float *vectors, IDNumbers &vector_ids_) = 0; + virtual Status + InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0; - virtual Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, - const float *vectors, QueryResults &results) = 0; + virtual Status + Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, + QueryResults& results) = 0; - virtual Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, - const float *vectors, const meta::DatesT &dates, QueryResults &results) = 0; + virtual Status + Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, + const meta::DatesT& dates, QueryResults& results) = 0; - virtual Status Query(const std::string &table_id, const std::vector &file_ids, - uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors, - const meta::DatesT &dates, QueryResults &results) = 0; + virtual Status + Query(const std::string& table_id, const std::vector& file_ids, uint64_t k, uint64_t nq, + uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) = 0; - virtual Status Size(uint64_t &result) = 0; + virtual Status + Size(uint64_t& result) = 0; - virtual Status CreateIndex(const std::string &table_id, const TableIndex &index) = 0; - virtual Status DescribeIndex(const std::string &table_id, TableIndex &index) = 0; - virtual Status DropIndex(const std::string &table_id) = 0; + virtual Status + CreateIndex(const std::string& table_id, const TableIndex& index) = 0; + virtual Status + DescribeIndex(const std::string& table_id, TableIndex& index) = 0; + virtual Status + DropIndex(const std::string& table_id) = 0; - virtual Status DropAll() = 0; -}; // DB + virtual Status + DropAll() = 0; +}; // DB using DBPtr = std::shared_ptr; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/DBFactory.cpp b/cpp/src/db/DBFactory.cpp index edf70cccf8..89d04b3632 100644 --- a/cpp/src/db/DBFactory.cpp +++ b/cpp/src/db/DBFactory.cpp @@ -15,18 +15,17 @@ // specific language governing permissions and limitations // under the License. - #include "db/DBFactory.h" #include "DBImpl.h" -#include "utils/Exception.h" #include "meta/MetaFactory.h" -#include "meta/SqliteMetaImpl.h" #include "meta/MySQLMetaImpl.h" +#include "meta/SqliteMetaImpl.h" +#include "utils/Exception.h" #include #include -#include #include +#include #include namespace zilliz { @@ -42,10 +41,10 @@ DBFactory::BuildOption() { } DBPtr -DBFactory::Build(const DBOptions &options) { +DBFactory::Build(const DBOptions& options) { return std::make_shared(options); } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/DBFactory.h b/cpp/src/db/DBFactory.h index 1223a11c29..c5c73a0649 100644 --- a/cpp/src/db/DBFactory.h +++ b/cpp/src/db/DBFactory.h @@ -20,8 +20,8 @@ #include "DB.h" #include "Options.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -29,11 +29,13 @@ namespace engine { class DBFactory { public: - static DBOptions BuildOption(); + static DBOptions + BuildOption(); - static DBPtr Build(const DBOptions &options); + static DBPtr + Build(const DBOptions& options); }; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index ab3f3104af..2780d0f763 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -16,28 +16,28 @@ // under the License. #include "db/DBImpl.h" +#include "Utils.h" #include "cache/CpuCacheMgr.h" #include "cache/GpuCacheMgr.h" #include "engine/EngineFactory.h" #include "insert/MemMenagerFactory.h" -#include "meta/SqliteMetaImpl.h" -#include "meta/MetaFactory.h" #include "meta/MetaConsts.h" +#include "meta/MetaFactory.h" +#include "meta/SqliteMetaImpl.h" #include "metrics/Metrics.h" -#include "scheduler/job/SearchJob.h" -#include "scheduler/job/DeleteJob.h" #include "scheduler/SchedInst.h" -#include "utils/TimeRecorder.h" +#include "scheduler/job/DeleteJob.h" +#include "scheduler/job/SearchJob.h" #include "utils/Log.h" -#include "Utils.h" +#include "utils/TimeRecorder.h" #include -#include -#include -#include -#include #include #include +#include +#include +#include +#include namespace zilliz { namespace milvus { @@ -49,13 +49,10 @@ constexpr uint64_t METRIC_ACTION_INTERVAL = 1; constexpr uint64_t COMPACT_ACTION_INTERVAL = 1; constexpr uint64_t INDEX_ACTION_INTERVAL = 1; -} // namespace +} // namespace -DBImpl::DBImpl(const DBOptions &options) - : options_(options), - shutting_down_(true), - compact_thread_pool_(1, 1), - index_thread_pool_(1, 1) { +DBImpl::DBImpl(const DBOptions& options) + : options_(options), shutting_down_(true), compact_thread_pool_(1, 1), index_thread_pool_(1, 1) { meta_ptr_ = MetaFactory::Build(options.meta_, options.mode_); mem_mgr_ = MemManagerFactory::Build(meta_ptr_, options_); Start(); @@ -66,7 +63,7 @@ DBImpl::~DBImpl() { } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -//external api +// external api /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// Status DBImpl::Start() { @@ -77,7 +74,7 @@ DBImpl::Start() { ENGINE_LOG_TRACE << "DB service start"; shutting_down_.store(false, std::memory_order_release); - //for distribute version, some nodes are read only + // for distribute version, some nodes are read only if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) { ENGINE_LOG_TRACE << "StartTimerTasks"; bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this); @@ -94,10 +91,10 @@ DBImpl::Stop() { shutting_down_.store(true, std::memory_order_release); - //makesure all memory data serialized + // makesure all memory data serialized MemSerialize(); - //wait compaction/buildindex finish + // wait compaction/buildindex finish bg_timer_thread_.join(); if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) { @@ -114,30 +111,30 @@ DBImpl::DropAll() { } Status -DBImpl::CreateTable(meta::TableSchema &table_schema) { +DBImpl::CreateTable(meta::TableSchema& table_schema) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } meta::TableSchema temp_schema = table_schema; - temp_schema.index_file_size_ *= ONE_MB; //store as MB + temp_schema.index_file_size_ *= ONE_MB; // store as MB return meta_ptr_->CreateTable(temp_schema); } Status -DBImpl::DeleteTable(const std::string &table_id, const meta::DatesT &dates) { +DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } - //dates partly delete files of the table but currently we don't support + // dates partly delete files of the table but currently we don't support ENGINE_LOG_DEBUG << "Prepare to delete table " << table_id; if (dates.empty()) { - mem_mgr_->EraseMemVector(table_id); //not allow insert - meta_ptr_->DeleteTable(table_id); //soft delete table + mem_mgr_->EraseMemVector(table_id); // not allow insert + meta_ptr_->DeleteTable(table_id); // soft delete table - //scheduler will determine when to delete table files + // scheduler will determine when to delete table files auto nres = scheduler::ResMgrInst::GetInstance()->GetNumOfComputeResource(); scheduler::DeleteJobPtr job = std::make_shared(0, table_id, meta_ptr_, nres); scheduler::JobMgrInst::GetInstance()->Put(job); @@ -150,18 +147,18 @@ DBImpl::DeleteTable(const std::string &table_id, const meta::DatesT &dates) { } Status -DBImpl::DescribeTable(meta::TableSchema &table_schema) { +DBImpl::DescribeTable(meta::TableSchema& table_schema) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } auto stat = meta_ptr_->DescribeTable(table_schema); - table_schema.index_file_size_ /= ONE_MB; //return as MB + table_schema.index_file_size_ /= ONE_MB; // return as MB return stat; } Status -DBImpl::HasTable(const std::string &table_id, bool &has_or_not) { +DBImpl::HasTable(const std::string& table_id, bool& has_or_not) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -170,7 +167,7 @@ DBImpl::HasTable(const std::string &table_id, bool &has_or_not) { } Status -DBImpl::AllTables(std::vector &table_schema_array) { +DBImpl::AllTables(std::vector& table_schema_array) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -179,7 +176,7 @@ DBImpl::AllTables(std::vector &table_schema_array) { } Status -DBImpl::PreloadTable(const std::string &table_id) { +DBImpl::PreloadTable(const std::string& table_id) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -198,13 +195,11 @@ DBImpl::PreloadTable(const std::string &table_id) { int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); int64_t available_size = cache_total - cache_usage; - for (auto &day_files : files) { - for (auto &file : day_files.second) { - ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, - file.location_, - (EngineType) file.engine_type_, - (MetricType) file.metric_type_, - file.nlist_); + for (auto& day_files : files) { + for (auto& file : day_files.second) { + ExecutionEnginePtr engine = + EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, + (MetricType)file.metric_type_, file.nlist_); if (engine == nullptr) { ENGINE_LOG_ERROR << "Invalid engine type"; return Status(DB_ERROR, "Invalid engine type"); @@ -215,9 +210,9 @@ DBImpl::PreloadTable(const std::string &table_id) { break; } else { try { - //step 1: load index + // step 1: load index engine->Load(true); - } catch (std::exception &ex) { + } catch (std::exception& ex) { std::string msg = "Pre-load table encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; return Status(DB_ERROR, msg); @@ -229,7 +224,7 @@ DBImpl::PreloadTable(const std::string &table_id) { } Status -DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { +DBImpl::UpdateTableFlag(const std::string& table_id, int64_t flag) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -238,7 +233,7 @@ DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { } Status -DBImpl::GetTableRowCount(const std::string &table_id, uint64_t &row_count) { +DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -247,9 +242,8 @@ DBImpl::GetTableRowCount(const std::string &table_id, uint64_t &row_count) { } Status -DBImpl::InsertVectors(const std::string &table_id_, - uint64_t n, const float *vectors, IDNumbers &vector_ids_) { -// ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache"; +DBImpl::InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) { + // ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache"; if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -257,21 +251,21 @@ DBImpl::InsertVectors(const std::string &table_id_, Status status; zilliz::milvus::server::CollectInsertMetrics metrics(n, status); status = mem_mgr_->InsertVectors(table_id_, n, vectors, vector_ids_); -// std::chrono::microseconds time_span = -// std::chrono::duration_cast(end_time - start_time); -// double average_time = double(time_span.count()) / n; + // std::chrono::microseconds time_span = + // std::chrono::duration_cast(end_time - start_time); + // double average_time = double(time_span.count()) / n; -// ENGINE_LOG_DEBUG << "Insert vectors to cache finished"; + // ENGINE_LOG_DEBUG << "Insert vectors to cache finished"; return status; } Status -DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) { +DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) { { std::unique_lock lock(build_index_mutex_); - //step 1: check index difference + // step 1: check index difference TableIndex old_index; auto status = DescribeIndex(table_id, old_index); if (!status.ok()) { @@ -279,9 +273,9 @@ DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) { return status; } - //step 2: update index info + // step 2: update index info TableIndex new_index = index; - new_index.metric_type_ = old_index.metric_type_;//dont change metric type, it was defined by CreateTable + new_index.metric_type_ = old_index.metric_type_; // dont change metric type, it was defined by CreateTable if (!utils::IsSameIndex(old_index, new_index)) { DropIndex(table_id); @@ -293,26 +287,23 @@ DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) { } } - //step 3: let merge file thread finish - //to avoid duplicate data bug + // step 3: let merge file thread finish + // to avoid duplicate data bug WaitMergeFileFinish(); - //step 4: wait and build index - //for IDMAP type, only wait all NEW file converted to RAW file - //for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files + // step 4: wait and build index + // for IDMAP type, only wait all NEW file converted to RAW file + // for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files std::vector file_types; - if (index.engine_type_ == (int) EngineType::FAISS_IDMAP) { + if (index.engine_type_ == (int)EngineType::FAISS_IDMAP) { file_types = { - (int) meta::TableFileSchema::NEW, - (int) meta::TableFileSchema::NEW_MERGE, + (int)meta::TableFileSchema::NEW, (int)meta::TableFileSchema::NEW_MERGE, }; } else { file_types = { - (int) meta::TableFileSchema::RAW, - (int) meta::TableFileSchema::NEW, - (int) meta::TableFileSchema::NEW_MERGE, - (int) meta::TableFileSchema::NEW_INDEX, - (int) meta::TableFileSchema::TO_INDEX, + (int)meta::TableFileSchema::RAW, (int)meta::TableFileSchema::NEW, + (int)meta::TableFileSchema::NEW_MERGE, (int)meta::TableFileSchema::NEW_INDEX, + (int)meta::TableFileSchema::TO_INDEX, }; } @@ -322,7 +313,7 @@ DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) { while (!file_ids.empty()) { ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times; - if (index.engine_type_ != (int) EngineType::FAISS_IDMAP) { + if (index.engine_type_ != (int)EngineType::FAISS_IDMAP) { status = meta_ptr_->UpdateTableFilesToIndex(table_id); } @@ -335,19 +326,19 @@ DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) { } Status -DBImpl::DescribeIndex(const std::string &table_id, TableIndex &index) { +DBImpl::DescribeIndex(const std::string& table_id, TableIndex& index) { return meta_ptr_->DescribeTableIndex(table_id, index); } Status -DBImpl::DropIndex(const std::string &table_id) { +DBImpl::DropIndex(const std::string& table_id) { ENGINE_LOG_DEBUG << "Drop index for table: " << table_id; return meta_ptr_->DropTableIndex(table_id); } Status -DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, - const float *vectors, QueryResults &results) { +DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, + QueryResults& results) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -359,46 +350,47 @@ DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t npr } Status -DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, - const float *vectors, const meta::DatesT &dates, QueryResults &results) { +DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, + const meta::DatesT& dates, QueryResults& results) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } ENGINE_LOG_DEBUG << "Query by dates for table: " << table_id; - //get all table files from table + // get all table files from table meta::DatePartionedTableFilesSchema files; std::vector ids; auto status = meta_ptr_->FilesToSearch(table_id, ids, dates, files); - if (!status.ok()) { return status; } + if (!status.ok()) { + return status; + } meta::TableFilesSchema file_id_array; - for (auto &day_files : files) { - for (auto &file : day_files.second) { + for (auto& day_files : files) { + for (auto& file : day_files.second) { file_id_array.push_back(file); } } - cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info before query + cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info before query status = QueryAsync(table_id, file_id_array, k, nq, nprobe, vectors, dates, results); - cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info after query + cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info after query return status; } Status -DBImpl::Query(const std::string &table_id, const std::vector &file_ids, - uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors, - const meta::DatesT &dates, QueryResults &results) { +DBImpl::Query(const std::string& table_id, const std::vector& file_ids, uint64_t k, uint64_t nq, + uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } ENGINE_LOG_DEBUG << "Query by file ids for table: " << table_id; - //get specified files + // get specified files std::vector ids; - for (auto &id : file_ids) { + for (auto& id : file_ids) { meta::TableFileSchema table_file; table_file.table_id_ = table_id; std::string::size_type sz; @@ -412,8 +404,8 @@ DBImpl::Query(const std::string &table_id, const std::vector &file_ } meta::TableFilesSchema file_id_array; - for (auto &day_files : files_array) { - for (auto &file : day_files.second) { + for (auto& day_files : files_array) { + for (auto& file : day_files.second) { file_id_array.push_back(file); } } @@ -422,14 +414,14 @@ DBImpl::Query(const std::string &table_id, const std::vector &file_ return Status(DB_ERROR, "Invalid file id"); } - cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info before query + cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info before query status = QueryAsync(table_id, file_id_array, k, nq, nprobe, vectors, dates, results); - cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info after query + cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info after query return status; } Status -DBImpl::Size(uint64_t &result) { +DBImpl::Size(uint64_t& result) { if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); } @@ -438,58 +430,57 @@ DBImpl::Size(uint64_t &result) { } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -//internal methods +// internal methods /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// Status -DBImpl::QueryAsync(const std::string &table_id, const meta::TableFilesSchema &files, - uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors, - const meta::DatesT &dates, QueryResults &results) { +DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, uint64_t k, uint64_t nq, + uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) { server::CollectQueryMetrics metrics(nq); TimeRecorder rc(""); - //step 1: get files to search - ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size() << " date range count: " - << dates.size(); + // step 1: get files to search + ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size() + << " date range count: " << dates.size(); scheduler::SearchJobPtr job = std::make_shared(0, k, nq, nprobe, vectors); - for (auto &file : files) { + for (auto& file : files) { scheduler::TableFileSchemaPtr file_ptr = std::make_shared(file); job->AddIndexFile(file_ptr); } - //step 2: put search task to scheduler + // step 2: put search task to scheduler scheduler::JobMgrInst::GetInstance()->Put(job); job->WaitResult(); if (!job->GetStatus().ok()) { return job->GetStatus(); } - //step 3: print time cost information -// double load_cost = context->LoadCost(); -// double search_cost = context->SearchCost(); -// double reduce_cost = context->ReduceCost(); -// std::string load_info = TimeRecorder::GetTimeSpanStr(load_cost); -// std::string search_info = TimeRecorder::GetTimeSpanStr(search_cost); -// std::string reduce_info = TimeRecorder::GetTimeSpanStr(reduce_cost); -// if(search_cost > 0.0 || reduce_cost > 0.0) { -// double total_cost = load_cost + search_cost + reduce_cost; -// double load_percent = load_cost/total_cost; -// double search_percent = search_cost/total_cost; -// double reduce_percent = reduce_cost/total_cost; -// -// ENGINE_LOG_DEBUG << "Engine load index totally cost: " << load_info -// << " percent: " << load_percent*100 << "%"; -// ENGINE_LOG_DEBUG << "Engine search index totally cost: " << search_info -// << " percent: " << search_percent*100 << "%"; -// ENGINE_LOG_DEBUG << "Engine reduce topk totally cost: " << reduce_info -// << " percent: " << reduce_percent*100 << "%"; -// } else { -// ENGINE_LOG_DEBUG << "Engine load cost: " << load_info -// << " search cost: " << search_info -// << " reduce cost: " << reduce_info; -// } + // step 3: print time cost information + // double load_cost = context->LoadCost(); + // double search_cost = context->SearchCost(); + // double reduce_cost = context->ReduceCost(); + // std::string load_info = TimeRecorder::GetTimeSpanStr(load_cost); + // std::string search_info = TimeRecorder::GetTimeSpanStr(search_cost); + // std::string reduce_info = TimeRecorder::GetTimeSpanStr(reduce_cost); + // if(search_cost > 0.0 || reduce_cost > 0.0) { + // double total_cost = load_cost + search_cost + reduce_cost; + // double load_percent = load_cost/total_cost; + // double search_percent = search_cost/total_cost; + // double reduce_percent = reduce_cost/total_cost; + // + // ENGINE_LOG_DEBUG << "Engine load index totally cost: " << load_info + // << " percent: " << load_percent*100 << "%"; + // ENGINE_LOG_DEBUG << "Engine search index totally cost: " << search_info + // << " percent: " << search_percent*100 << "%"; + // ENGINE_LOG_DEBUG << "Engine reduce topk totally cost: " << reduce_info + // << " percent: " << reduce_percent*100 << "%"; + // } else { + // ENGINE_LOG_DEBUG << "Engine load cost: " << load_info + // << " search cost: " << search_info + // << " reduce cost: " << reduce_info; + // } - //step 4: construct results + // step 4: construct results results = job->GetResult(); rc.ElapseFromBegin("Engine query totally cost"); @@ -520,7 +511,7 @@ DBImpl::BackgroundTimerTask() { void DBImpl::WaitMergeFileFinish() { std::lock_guard lck(compact_result_mutex_); - for (auto &iter : compact_thread_results_) { + for (auto& iter : compact_thread_results_) { iter.wait(); } } @@ -528,7 +519,7 @@ DBImpl::WaitMergeFileFinish() { void DBImpl::WaitBuildIndexFinish() { std::lock_guard lck(index_result_mutex_); - for (auto &iter : index_thread_results_) { + for (auto& iter : index_thread_results_) { iter.wait(); } } @@ -569,7 +560,7 @@ DBImpl::MemSerialize() { std::lock_guard lck(mem_serialize_mutex_); std::set temp_table_ids; mem_mgr_->Serialize(temp_table_ids); - for (auto &id : temp_table_ids) { + for (auto& id : temp_table_ids) { compact_table_ids_.insert(id); } @@ -588,10 +579,10 @@ DBImpl::StartCompactionTask() { return; } - //serialize memory data + // serialize memory data MemSerialize(); - //compactiong has been finished? + // compactiong has been finished? { std::lock_guard lck(compact_result_mutex_); if (!compact_thread_results_.empty()) { @@ -602,7 +593,7 @@ DBImpl::StartCompactionTask() { } } - //add new compaction task + // add new compaction task { std::lock_guard lck(compact_result_mutex_); if (compact_thread_results_.empty()) { @@ -614,11 +605,10 @@ DBImpl::StartCompactionTask() { } Status -DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date, - const meta::TableFilesSchema &files) { +DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, const meta::TableFilesSchema& files) { ENGINE_LOG_DEBUG << "Merge files for table: " << table_id; - //step 1: create table file + // step 1: create table file meta::TableFileSchema table_file; table_file.table_id_ = table_id; table_file.date_ = date; @@ -630,15 +620,15 @@ DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date, return status; } - //step 2: merge files + // step 2: merge files ExecutionEnginePtr index = - EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType) table_file.engine_type_, - (MetricType) table_file.metric_type_, table_file.nlist_); + EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType)table_file.engine_type_, + (MetricType)table_file.metric_type_, table_file.nlist_); meta::TableFilesSchema updated; int64_t index_size = 0; - for (auto &file : files) { + for (auto& file : files) { server::CollectMergeFilesMetrics metrics; index->Merge(file.location_); @@ -651,11 +641,11 @@ DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date, if (index_size >= file_schema.index_file_size_) break; } - //step 3: serialize to disk + // step 3: serialize to disk try { index->Serialize(); - } catch (std::exception &ex) { - //typical error: out of disk space or permition denied + } catch (std::exception& ex) { + // typical error: out of disk space or permition denied std::string msg = "Serialize merged index encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; @@ -669,12 +659,12 @@ DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date, return Status(DB_ERROR, msg); } - //step 4: update table files state - //if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size - //else set file type to RAW, no need to build index - if (table_file.engine_type_ != (int) EngineType::FAISS_IDMAP) { - table_file.file_type_ = (index->PhysicalSize() >= table_file.index_file_size_) ? - meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; + // step 4: update table files state + // if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size + // else set file type to RAW, no need to build index + if (table_file.engine_type_ != (int)EngineType::FAISS_IDMAP) { + table_file.file_type_ = (index->PhysicalSize() >= table_file.index_file_size_) ? meta::TableFileSchema::TO_INDEX + : meta::TableFileSchema::RAW; } else { table_file.file_type_ = meta::TableFileSchema::RAW; } @@ -682,8 +672,7 @@ DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date, table_file.row_count_ = index->Count(); updated.push_back(table_file); status = meta_ptr_->UpdateTableFiles(updated); - ENGINE_LOG_DEBUG << "New merged file " << table_file.file_id_ << - " of size " << index->PhysicalSize() << " bytes"; + ENGINE_LOG_DEBUG << "New merged file " << table_file.file_id_ << " of size " << index->PhysicalSize() << " bytes"; if (options_.insert_cache_immediately_) { index->Cache(); @@ -693,7 +682,7 @@ DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date, } Status -DBImpl::BackgroundMergeFiles(const std::string &table_id) { +DBImpl::BackgroundMergeFiles(const std::string& table_id) { meta::DatePartionedTableFilesSchema raw_files; auto status = meta_ptr_->FilesToMerge(table_id, raw_files); if (!status.ok()) { @@ -702,7 +691,7 @@ DBImpl::BackgroundMergeFiles(const std::string &table_id) { } bool has_merge = false; - for (auto &kv : raw_files) { + for (auto& kv : raw_files) { auto files = kv.second; if (files.size() < options_.merge_trigger_number_) { ENGINE_LOG_DEBUG << "Files number not greater equal than merge trigger number, skip merge action"; @@ -725,7 +714,7 @@ DBImpl::BackgroundCompaction(std::set table_ids) { ENGINE_LOG_TRACE << " Background compaction thread start"; Status status; - for (auto &table_id : table_ids) { + for (auto& table_id : table_ids) { status = BackgroundMergeFiles(table_id); if (!status.ok()) { ENGINE_LOG_ERROR << "Merge files for table " << table_id << " failed: " << status.ToString(); @@ -739,7 +728,7 @@ DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); - int ttl = 5 * meta::M_SEC;//default: file will be deleted after 5 minutes + int ttl = 5 * meta::M_SEC; // default: file will be deleted after 5 minutes if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { ttl = meta::D_SEC; } @@ -756,7 +745,7 @@ DBImpl::StartBuildIndexTask(bool force) { return; } - //build index has been finished? + // build index has been finished? { std::lock_guard lck(index_result_mutex_); if (!index_thread_results_.empty()) { @@ -767,52 +756,50 @@ DBImpl::StartBuildIndexTask(bool force) { } } - //add new build index task + // add new build index task { std::lock_guard lck(index_result_mutex_); if (index_thread_results_.empty()) { - index_thread_results_.push_back( - index_thread_pool_.enqueue(&DBImpl::BackgroundBuildIndex, this)); + index_thread_results_.push_back(index_thread_pool_.enqueue(&DBImpl::BackgroundBuildIndex, this)); } } } Status -DBImpl::BuildIndex(const meta::TableFileSchema &file) { - ExecutionEnginePtr to_index = - EngineFactory::Build(file.dimension_, file.location_, (EngineType) file.engine_type_, - (MetricType) file.metric_type_, file.nlist_); +DBImpl::BuildIndex(const meta::TableFileSchema& file) { + ExecutionEnginePtr to_index = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, + (MetricType)file.metric_type_, file.nlist_); if (to_index == nullptr) { ENGINE_LOG_ERROR << "Invalid engine type"; return Status(DB_ERROR, "Invalid engine type"); } try { - //step 1: load index + // step 1: load index Status status = to_index->Load(options_.insert_cache_immediately_); if (!status.ok()) { ENGINE_LOG_ERROR << "Failed to load index file: " << status.ToString(); return status; } - //step 2: create table file + // step 2: create table file meta::TableFileSchema table_file; table_file.table_id_ = file.table_id_; table_file.date_ = file.date_; table_file.file_type_ = - meta::TableFileSchema::NEW_INDEX; //for multi-db-path, distribute index file averagely to each path + meta::TableFileSchema::NEW_INDEX; // for multi-db-path, distribute index file averagely to each path status = meta_ptr_->CreateTableFile(table_file); if (!status.ok()) { ENGINE_LOG_ERROR << "Failed to create table file: " << status.ToString(); return status; } - //step 3: build index + // step 3: build index std::shared_ptr index; try { server::CollectBuildIndexMetrics metrics; - index = to_index->BuildIndex(table_file.location_, (EngineType) table_file.engine_type_); + index = to_index->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_); if (index == nullptr) { table_file.file_type_ = meta::TableFileSchema::TO_DELETE; status = meta_ptr_->UpdateTableFile(table_file); @@ -821,8 +808,8 @@ DBImpl::BuildIndex(const meta::TableFileSchema &file) { return status; } - } catch (std::exception &ex) { - //typical error: out of gpu memory + } catch (std::exception& ex) { + // typical error: out of gpu memory std::string msg = "BuildIndex encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; @@ -836,7 +823,7 @@ DBImpl::BuildIndex(const meta::TableFileSchema &file) { return Status(DB_ERROR, msg); } - //step 4: if table has been deleted, dont save index file + // step 4: if table has been deleted, dont save index file bool has_table = false; meta_ptr_->HasTable(file.table_id_, has_table); if (!has_table) { @@ -844,11 +831,11 @@ DBImpl::BuildIndex(const meta::TableFileSchema &file) { return Status::OK(); } - //step 5: save index file + // step 5: save index file try { index->Serialize(); - } catch (std::exception &ex) { - //typical error: out of disk space or permition denied + } catch (std::exception& ex) { + // typical error: out of disk space or permition denied std::string msg = "Serialize index encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; @@ -862,7 +849,7 @@ DBImpl::BuildIndex(const meta::TableFileSchema &file) { return Status(DB_ERROR, msg); } - //step 6: update meta + // step 6: update meta table_file.file_type_ = meta::TableFileSchema::INDEX; table_file.file_size_ = index->PhysicalSize(); table_file.row_count_ = index->Count(); @@ -873,15 +860,15 @@ DBImpl::BuildIndex(const meta::TableFileSchema &file) { meta::TableFilesSchema update_files = {table_file, origin_file}; status = meta_ptr_->UpdateTableFiles(update_files); if (status.ok()) { - ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size " - << index->PhysicalSize() << " bytes" + ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size " << index->PhysicalSize() + << " bytes" << " from file " << origin_file.file_id_; if (options_.insert_cache_immediately_) { index->Cache(); } } else { - //failed to update meta, mark the new file as to_delete, don't delete old file + // failed to update meta, mark the new file as to_delete, don't delete old file origin_file.file_type_ = meta::TableFileSchema::TO_INDEX; status = meta_ptr_->UpdateTableFile(origin_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << origin_file.file_id_ << " to to_index"; @@ -890,7 +877,7 @@ DBImpl::BuildIndex(const meta::TableFileSchema &file) { status = meta_ptr_->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; } - } catch (std::exception &ex) { + } catch (std::exception& ex) { std::string msg = "Build index encounter exception: " + std::string(ex.what()); ENGINE_LOG_ERROR << msg; return Status(DB_ERROR, msg); @@ -907,7 +894,7 @@ DBImpl::BackgroundBuildIndex() { meta::TableFilesSchema to_index_files; meta_ptr_->FilesToIndex(to_index_files); Status status; - for (auto &file : to_index_files) { + for (auto& file : to_index_files) { status = BuildIndex(file); if (!status.ok()) { ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString(); @@ -922,6 +909,6 @@ DBImpl::BackgroundBuildIndex() { ENGINE_LOG_TRACE << "Background build index thread exit"; } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 344e290445..366408d8b0 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -19,18 +19,18 @@ #include "DB.h" #include "Types.h" -#include "utils/ThreadPool.h" #include "src/db/insert/MemManager.h" +#include "utils/ThreadPool.h" -#include -#include -#include #include -#include +#include #include +#include +#include #include -#include #include +#include +#include namespace zilliz { namespace milvus { @@ -44,92 +44,101 @@ class Meta; class DBImpl : public DB { public: - explicit DBImpl(const DBOptions &options); + explicit DBImpl(const DBOptions& options); ~DBImpl(); - Status Start() override; - Status Stop() override; - Status DropAll() override; + Status + Start() override; + Status + Stop() override; + Status + DropAll() override; - Status CreateTable(meta::TableSchema &table_schema) override; + Status + CreateTable(meta::TableSchema& table_schema) override; - Status DeleteTable(const std::string &table_id, const meta::DatesT &dates) override; + Status + DeleteTable(const std::string& table_id, const meta::DatesT& dates) override; - Status DescribeTable(meta::TableSchema &table_schema) override; + Status + DescribeTable(meta::TableSchema& table_schema) override; - Status HasTable(const std::string &table_id, bool &has_or_not) override; + Status + HasTable(const std::string& table_id, bool& has_or_not) override; - Status AllTables(std::vector &table_schema_array) override; + Status + AllTables(std::vector& table_schema_array) override; - Status PreloadTable(const std::string &table_id) override; + Status + PreloadTable(const std::string& table_id) override; - Status UpdateTableFlag(const std::string &table_id, int64_t flag); + Status + UpdateTableFlag(const std::string& table_id, int64_t flag); - Status GetTableRowCount(const std::string &table_id, uint64_t &row_count) override; + Status + GetTableRowCount(const std::string& table_id, uint64_t& row_count) override; - Status InsertVectors(const std::string &table_id, uint64_t n, const float *vectors, IDNumbers &vector_ids) override; + Status + InsertVectors(const std::string& table_id, uint64_t n, const float* vectors, IDNumbers& vector_ids) override; - Status CreateIndex(const std::string &table_id, const TableIndex &index) override; + Status + CreateIndex(const std::string& table_id, const TableIndex& index) override; - Status DescribeIndex(const std::string &table_id, TableIndex &index) override; + Status + DescribeIndex(const std::string& table_id, TableIndex& index) override; - Status DropIndex(const std::string &table_id) override; + Status + DropIndex(const std::string& table_id) override; - Status Query(const std::string &table_id, - uint64_t k, - uint64_t nq, - uint64_t nprobe, - const float *vectors, - QueryResults &results) override; + Status + Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, + QueryResults& results) override; - Status Query(const std::string &table_id, - uint64_t k, - uint64_t nq, - uint64_t nprobe, - const float *vectors, - const meta::DatesT &dates, - QueryResults &results) override; + Status + Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, + const meta::DatesT& dates, QueryResults& results) override; - Status Query(const std::string &table_id, - const std::vector &file_ids, - uint64_t k, - uint64_t nq, - uint64_t nprobe, - const float *vectors, - const meta::DatesT &dates, - QueryResults &results) override; + Status + Query(const std::string& table_id, const std::vector& file_ids, uint64_t k, uint64_t nq, + uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) override; - Status Size(uint64_t &result) override; + Status + Size(uint64_t& result) override; private: - Status QueryAsync(const std::string &table_id, - const meta::TableFilesSchema &files, - uint64_t k, - uint64_t nq, - uint64_t nprobe, - const float *vectors, - const meta::DatesT &dates, - QueryResults &results); + Status + QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, uint64_t k, uint64_t nq, + uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results); - void BackgroundTimerTask(); - void WaitMergeFileFinish(); - void WaitBuildIndexFinish(); + void + BackgroundTimerTask(); + void + WaitMergeFileFinish(); + void + WaitBuildIndexFinish(); - void StartMetricTask(); + void + StartMetricTask(); - void StartCompactionTask(); - Status MergeFiles(const std::string &table_id, - const meta::DateT &date, - const meta::TableFilesSchema &files); - Status BackgroundMergeFiles(const std::string &table_id); - void BackgroundCompaction(std::set table_ids); + void + StartCompactionTask(); + Status + MergeFiles(const std::string& table_id, const meta::DateT& date, const meta::TableFilesSchema& files); + Status + BackgroundMergeFiles(const std::string& table_id); + void + BackgroundCompaction(std::set table_ids); - void StartBuildIndexTask(bool force = false); - void BackgroundBuildIndex(); + void + StartBuildIndexTask(bool force = false); + void + BackgroundBuildIndex(); - Status BuildIndex(const meta::TableFileSchema &); + Status + BuildIndex(const meta::TableFileSchema&); - Status MemSerialize(); + Status + MemSerialize(); private: const DBOptions options_; @@ -152,9 +161,8 @@ class DBImpl : public DB { std::list> index_thread_results_; std::mutex build_index_mutex_; -}; // DBImpl +}; // DBImpl - -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/IDGenerator.cpp b/cpp/src/db/IDGenerator.cpp index 78c88b2c03..b80e1ce354 100644 --- a/cpp/src/db/IDGenerator.cpp +++ b/cpp/src/db/IDGenerator.cpp @@ -17,8 +17,8 @@ #include "db/IDGenerator.h" -#include #include +#include #include namespace zilliz { @@ -32,13 +32,12 @@ constexpr size_t SimpleIDGenerator::MAX_IDS_PER_MICRO; IDNumber SimpleIDGenerator::GetNextIDNumber() { auto now = std::chrono::system_clock::now(); - auto micros = std::chrono::duration_cast( - now.time_since_epoch()).count(); + auto micros = std::chrono::duration_cast(now.time_since_epoch()).count(); return micros * MAX_IDS_PER_MICRO; } void -SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers &ids) { +SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers& ids) { if (n > MAX_IDS_PER_MICRO) { NextIDNumbers(n - MAX_IDS_PER_MICRO, ids); NextIDNumbers(MAX_IDS_PER_MICRO, ids); @@ -49,8 +48,7 @@ SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers &ids) { } auto now = std::chrono::system_clock::now(); - auto micros = std::chrono::duration_cast( - now.time_since_epoch()).count(); + auto micros = std::chrono::duration_cast(now.time_since_epoch()).count(); micros *= MAX_IDS_PER_MICRO; for (int pos = 0; pos < n; ++pos) { @@ -59,11 +57,11 @@ SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers &ids) { } void -SimpleIDGenerator::GetNextIDNumbers(size_t n, IDNumbers &ids) { +SimpleIDGenerator::GetNextIDNumbers(size_t n, IDNumbers& ids) { ids.clear(); NextIDNumbers(n, ids); } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/IDGenerator.h b/cpp/src/db/IDGenerator.h index 0785870c4c..f41f745864 100644 --- a/cpp/src/db/IDGenerator.h +++ b/cpp/src/db/IDGenerator.h @@ -28,16 +28,14 @@ namespace engine { class IDGenerator { public: - virtual - IDNumber GetNextIDNumber() = 0; + virtual IDNumber + GetNextIDNumber() = 0; virtual void - GetNextIDNumbers(size_t n, IDNumbers &ids) = 0; - - virtual - ~IDGenerator() = 0; -}; // IDGenerator + GetNextIDNumbers(size_t n, IDNumbers& ids) = 0; + virtual ~IDGenerator() = 0; +}; // IDGenerator class SimpleIDGenerator : public IDGenerator { public: @@ -47,16 +45,15 @@ class SimpleIDGenerator : public IDGenerator { GetNextIDNumber() override; void - GetNextIDNumbers(size_t n, IDNumbers &ids) override; + GetNextIDNumbers(size_t n, IDNumbers& ids) override; private: void - NextIDNumbers(size_t n, IDNumbers &ids); + NextIDNumbers(size_t n, IDNumbers& ids); static constexpr size_t MAX_IDS_PER_MICRO = 1000; -}; // SimpleIDGenerator +}; // SimpleIDGenerator - -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index 4c82f3f736..edaf69c9e6 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -19,28 +19,28 @@ #include "utils/Exception.h" #include "utils/Log.h" -#include #include +#include #include namespace zilliz { namespace milvus { namespace engine { -ArchiveConf::ArchiveConf(const std::string &type, const std::string &criterias) { +ArchiveConf::ArchiveConf(const std::string& type, const std::string& criterias) { ParseType(type); ParseCritirias(criterias); } void -ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT &criterial) { - for (auto &pair : criterial) { +ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT& criterial) { + for (auto& pair : criterial) { criterias_[pair.first] = pair.second; } } void -ArchiveConf::ParseCritirias(const std::string &criterias) { +ArchiveConf::ParseCritirias(const std::string& criterias) { std::stringstream ss(criterias); std::vector tokens; @@ -50,7 +50,7 @@ ArchiveConf::ParseCritirias(const std::string &criterias) { return; } - for (auto &token : tokens) { + for (auto& token : tokens) { if (token.empty()) { continue; } @@ -68,13 +68,11 @@ ArchiveConf::ParseCritirias(const std::string &criterias) { try { auto value = std::stoi(kv[1]); criterias_[kv[0]] = value; - } - catch (std::out_of_range &) { + } catch (std::out_of_range&) { std::string msg = "Out of range: '" + kv[1] + "'"; ENGINE_LOG_ERROR << msg; throw InvalidArgumentException(msg); - } - catch (...) { + } catch (...) { std::string msg = "Invalid argument: '" + kv[1] + "'"; ENGINE_LOG_ERROR << msg; throw InvalidArgumentException(msg); @@ -83,7 +81,7 @@ ArchiveConf::ParseCritirias(const std::string &criterias) { } void -ArchiveConf::ParseType(const std::string &type) { +ArchiveConf::ParseType(const std::string& type) { if (type != "delete" && type != "swap") { std::string msg = "Invalid argument: type='" + type + "'"; throw InvalidArgumentException(msg); @@ -91,6 +89,6 @@ ArchiveConf::ParseType(const std::string &type) { type_ = type; } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index d8be2767df..df3b1fc0c5 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -19,9 +19,9 @@ #include "Constants.h" -#include -#include #include +#include +#include #include namespace zilliz { @@ -30,27 +30,32 @@ namespace engine { class Env; -static const char *ARCHIVE_CONF_DISK = "disk"; -static const char *ARCHIVE_CONF_DAYS = "days"; +static const char* ARCHIVE_CONF_DISK = "disk"; +static const char* ARCHIVE_CONF_DAYS = "days"; struct ArchiveConf { using CriteriaT = std::map; - explicit ArchiveConf(const std::string &type, const std::string &criterias = std::string()); + explicit ArchiveConf(const std::string& type, const std::string& criterias = std::string()); - const std::string &GetType() const { + const std::string& + GetType() const { return type_; } - const CriteriaT GetCriterias() const { + const CriteriaT + GetCriterias() const { return criterias_; } - void SetCriterias(const ArchiveConf::CriteriaT &criterial); + void + SetCriterias(const ArchiveConf::CriteriaT& criterial); private: - void ParseCritirias(const std::string &type); - void ParseType(const std::string &criterias); + void + ParseCritirias(const std::string& type); + void + ParseType(const std::string& criterias); std::string type_; CriteriaT criterias_; @@ -61,14 +66,10 @@ struct DBMetaOptions { std::vector slave_paths_; std::string backend_uri_; ArchiveConf archive_conf_ = ArchiveConf("delete"); -}; // DBMetaOptions +}; // DBMetaOptions struct DBOptions { - typedef enum { - SINGLE = 0, - CLUSTER_READONLY, - CLUSTER_WRITABLE - } MODE; + typedef enum { SINGLE = 0, CLUSTER_READONLY, CLUSTER_WRITABLE } MODE; uint16_t merge_trigger_number_ = 2; DBMetaOptions meta_; @@ -76,9 +77,8 @@ struct DBOptions { size_t insert_buffer_size_ = 4 * ONE_GB; bool insert_cache_immediately_ = false; -}; // Options +}; // Options - -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/Types.h b/cpp/src/db/Types.h index 04bf680937..c37434a405 100644 --- a/cpp/src/db/Types.h +++ b/cpp/src/db/Types.h @@ -19,27 +19,27 @@ #include "db/engine/ExecutionEngine.h" -#include #include #include +#include namespace zilliz { namespace milvus { namespace engine { typedef int64_t IDNumber; -typedef IDNumber *IDNumberPtr; +typedef IDNumber* IDNumberPtr; typedef std::vector IDNumbers; typedef std::vector> QueryResult; typedef std::vector QueryResults; struct TableIndex { - int32_t engine_type_ = (int) EngineType::FAISS_IDMAP; + int32_t engine_type_ = (int)EngineType::FAISS_IDMAP; int32_t nlist_ = 16384; - int32_t metric_type_ = (int) MetricType::L2; + int32_t metric_type_ = (int)MetricType::L2; }; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index b3d72202cc..84e099b579 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -19,11 +19,11 @@ #include "utils/CommonUtil.h" #include "utils/Log.h" -#include +#include #include +#include #include #include -#include namespace zilliz { namespace milvus { @@ -32,20 +32,20 @@ namespace utils { namespace { -const char *TABLES_FOLDER = "/tables/"; +const char* TABLES_FOLDER = "/tables/"; uint64_t index_file_counter = 0; std::mutex index_file_counter_mutex; std::string -ConstructParentFolder(const std::string &db_path, const meta::TableFileSchema &table_file) { +ConstructParentFolder(const std::string& db_path, const meta::TableFileSchema& table_file) { std::string table_path = db_path + TABLES_FOLDER + table_file.table_id_; std::string partition_path = table_path + "/" + std::to_string(table_file.date_); return partition_path; } std::string -GetTableFileParentFolder(const DBMetaOptions &options, const meta::TableFileSchema &table_file) { +GetTableFileParentFolder(const DBMetaOptions& options, const meta::TableFileSchema& table_file) { uint64_t path_count = options.slave_paths_.size() + 1; std::string target_path = options.path_; uint64_t index = 0; @@ -70,19 +70,18 @@ GetTableFileParentFolder(const DBMetaOptions &options, const meta::TableFileSche return ConstructParentFolder(target_path, table_file); } -} // namespace +} // namespace int64_t GetMicroSecTimeStamp() { auto now = std::chrono::system_clock::now(); - auto micros = std::chrono::duration_cast( - now.time_since_epoch()).count(); + auto micros = std::chrono::duration_cast(now.time_since_epoch()).count(); return micros; } Status -CreateTablePath(const DBMetaOptions &options, const std::string &table_id) { +CreateTablePath(const DBMetaOptions& options, const std::string& table_id) { std::string db_path = options.path_; std::string table_path = db_path + TABLES_FOLDER + table_id; auto status = server::CommonUtil::CreateDirectory(table_path); @@ -91,7 +90,7 @@ CreateTablePath(const DBMetaOptions &options, const std::string &table_id) { return status; } - for (auto &path : options.slave_paths_) { + for (auto& path : options.slave_paths_) { table_path = path + TABLES_FOLDER + table_id; status = server::CommonUtil::CreateDirectory(table_path); if (!status.ok()) { @@ -104,17 +103,16 @@ CreateTablePath(const DBMetaOptions &options, const std::string &table_id) { } Status -DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool force) { +DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force) { std::vector paths = options.slave_paths_; paths.push_back(options.path_); - for (auto &path : paths) { + for (auto& path : paths) { std::string table_path = path + TABLES_FOLDER + table_id; if (force) { boost::filesystem::remove_all(table_path); ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; - } else if (boost::filesystem::exists(table_path) && - boost::filesystem::is_empty(table_path)) { + } else if (boost::filesystem::exists(table_path) && boost::filesystem::is_empty(table_path)) { boost::filesystem::remove_all(table_path); ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; } @@ -124,7 +122,7 @@ DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool } Status -CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) { +CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { std::string parent_path = GetTableFileParentFolder(options, table_file); auto status = server::CommonUtil::CreateDirectory(parent_path); @@ -139,14 +137,14 @@ CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_f } Status -GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) { +GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { std::string parent_path = ConstructParentFolder(options.path_, table_file); std::string file_path = parent_path + "/" + table_file.file_id_; if (boost::filesystem::exists(file_path)) { table_file.location_ = file_path; return Status::OK(); } else { - for (auto &path : options.slave_paths_) { + for (auto& path : options.slave_paths_) { parent_path = ConstructParentFolder(path, table_file); file_path = parent_path + "/" + table_file.file_id_; if (boost::filesystem::exists(file_path)) { @@ -157,28 +155,26 @@ GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file } std::string msg = "Table file doesn't exist: " + file_path; - ENGINE_LOG_ERROR << msg << " in path: " << options.path_ - << " for table: " << table_file.table_id_; + ENGINE_LOG_ERROR << msg << " in path: " << options.path_ << " for table: " << table_file.table_id_; return Status(DB_ERROR, msg); } Status -DeleteTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) { +DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { utils::GetTableFilePath(options, table_file); boost::filesystem::remove(table_file.location_); return Status::OK(); } bool -IsSameIndex(const TableIndex &index1, const TableIndex &index2) { - return index1.engine_type_ == index2.engine_type_ - && index1.nlist_ == index2.nlist_ - && index1.metric_type_ == index2.metric_type_; +IsSameIndex(const TableIndex& index1, const TableIndex& index2) { + return index1.engine_type_ == index2.engine_type_ && index1.nlist_ == index2.nlist_ && + index1.metric_type_ == index2.metric_type_; } meta::DateT -GetDate(const std::time_t &t, int day_delta) { +GetDate(const std::time_t& t, int day_delta) { struct tm ltm; localtime_r(&t, <m); if (day_delta > 0) { @@ -211,20 +207,15 @@ GetDate() { // URI format: dialect://username:password@host:port/database Status -ParseMetaUri(const std::string &uri, MetaUriInfo &info) { +ParseMetaUri(const std::string& uri, MetaUriInfo& info) { std::string dialect_regex = "(.*)"; std::string username_tegex = "(.*)"; std::string password_regex = "(.*)"; std::string host_regex = "(.*)"; std::string port_regex = "(.*)"; std::string db_name_regex = "(.*)"; - std::string uri_regex_str = - dialect_regex + "\\:\\/\\/" + - username_tegex + "\\:" + - password_regex + "\\@" + - host_regex + "\\:" + - port_regex + "\\/" + - db_name_regex; + std::string uri_regex_str = dialect_regex + "\\:\\/\\/" + username_tegex + "\\:" + password_regex + "\\@" + + host_regex + "\\:" + port_regex + "\\/" + db_name_regex; std::regex uri_regex(uri_regex_str); std::smatch pieces_match; @@ -237,7 +228,7 @@ ParseMetaUri(const std::string &uri, MetaUriInfo &info) { info.port_ = pieces_match[5].str(); info.db_name_ = pieces_match[6].str(); - //TODO: verify host, port... + // TODO: verify host, port... } else { return Status(DB_INVALID_META_URI, "Invalid meta uri: " + uri); } @@ -245,7 +236,7 @@ ParseMetaUri(const std::string &uri, MetaUriInfo &info) { return Status::OK(); } -} // namespace utils -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace utils +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/Utils.h b/cpp/src/db/Utils.h index fbeca84686..c161e32b95 100644 --- a/cpp/src/db/Utils.h +++ b/cpp/src/db/Utils.h @@ -18,11 +18,11 @@ #pragma once #include "Options.h" -#include "db/meta/MetaTypes.h" #include "db/Types.h" +#include "db/meta/MetaTypes.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -33,22 +33,22 @@ int64_t GetMicroSecTimeStamp(); Status -CreateTablePath(const DBMetaOptions &options, const std::string &table_id); +CreateTablePath(const DBMetaOptions& options, const std::string& table_id); Status -DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool force = true); +DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force = true); Status -CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file); +CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); Status -GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file); +GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); Status -DeleteTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file); +DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); bool -IsSameIndex(const TableIndex &index1, const TableIndex &index2); +IsSameIndex(const TableIndex& index1, const TableIndex& index2); meta::DateT -GetDate(const std::time_t &t, int day_delta = 0); +GetDate(const std::time_t& t, int day_delta = 0); meta::DateT GetDate(); meta::DateT @@ -64,9 +64,9 @@ struct MetaUriInfo { }; Status -ParseMetaUri(const std::string &uri, MetaUriInfo &info); +ParseMetaUri(const std::string& uri, MetaUriInfo& info); -} // namespace utils -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace utils +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/engine/EngineFactory.cpp b/cpp/src/db/engine/EngineFactory.cpp index 043744f803..338b7d72af 100644 --- a/cpp/src/db/engine/EngineFactory.cpp +++ b/cpp/src/db/engine/EngineFactory.cpp @@ -26,17 +26,14 @@ namespace milvus { namespace engine { ExecutionEnginePtr -EngineFactory::Build(uint16_t dimension, - const std::string &location, - EngineType index_type, - MetricType metric_type, +EngineFactory::Build(uint16_t dimension, const std::string& location, EngineType index_type, MetricType metric_type, int32_t nlist) { if (index_type == EngineType::INVALID) { ENGINE_LOG_ERROR << "Unsupported engine type"; return nullptr; } - ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int) index_type; + ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int)index_type; ExecutionEnginePtr execution_engine_ptr = std::make_shared(dimension, location, index_type, metric_type, nlist); @@ -44,6 +41,6 @@ EngineFactory::Build(uint16_t dimension, return execution_engine_ptr; } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/engine/EngineFactory.h b/cpp/src/db/engine/EngineFactory.h index 105e48f885..5d22fbb4d2 100644 --- a/cpp/src/db/engine/EngineFactory.h +++ b/cpp/src/db/engine/EngineFactory.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "ExecutionEngine.h" @@ -29,14 +28,11 @@ namespace engine { class EngineFactory { public: - static ExecutionEnginePtr Build(uint16_t dimension, - const std::string &location, - EngineType index_type, - MetricType metric_type, - int32_t nlist); + static ExecutionEnginePtr + Build(uint16_t dimension, const std::string& location, EngineType index_type, MetricType metric_type, + int32_t nlist); }; -} // namespace engine -} // namespace milvus -} // namespace zilliz - +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/engine/ExecutionEngine.h b/cpp/src/db/engine/ExecutionEngine.h index c63760686f..4ae3e872b3 100644 --- a/cpp/src/db/engine/ExecutionEngine.h +++ b/cpp/src/db/engine/ExecutionEngine.h @@ -19,9 +19,9 @@ #include "utils/Status.h" -#include #include #include +#include namespace zilliz { namespace milvus { @@ -43,52 +43,66 @@ enum class MetricType { class ExecutionEngine { public: - virtual Status AddWithIds(int64_t n, const float *xdata, const int64_t *xids) = 0; + virtual Status + AddWithIds(int64_t n, const float* xdata, const int64_t* xids) = 0; - virtual size_t Count() const = 0; + virtual size_t + Count() const = 0; - virtual size_t Size() const = 0; + virtual size_t + Size() const = 0; - virtual size_t Dimension() const = 0; + virtual size_t + Dimension() const = 0; - virtual size_t PhysicalSize() const = 0; + virtual size_t + PhysicalSize() const = 0; - virtual Status Serialize() = 0; + virtual Status + Serialize() = 0; - virtual Status Load(bool to_cache = true) = 0; + virtual Status + Load(bool to_cache = true) = 0; - virtual Status CopyToGpu(uint64_t device_id) = 0; + virtual Status + CopyToGpu(uint64_t device_id) = 0; - virtual Status CopyToCpu() = 0; + virtual Status + CopyToCpu() = 0; - virtual std::shared_ptr Clone() = 0; + virtual std::shared_ptr + Clone() = 0; - virtual Status Merge(const std::string &location) = 0; + virtual Status + Merge(const std::string& location) = 0; - virtual Status Search(int64_t n, - const float *data, - int64_t k, - int64_t nprobe, - float *distances, - int64_t *labels) const = 0; + virtual Status + Search(int64_t n, const float* data, int64_t k, int64_t nprobe, float* distances, int64_t* labels) const = 0; - virtual std::shared_ptr BuildIndex(const std::string &location, EngineType engine_type) = 0; + virtual std::shared_ptr + BuildIndex(const std::string& location, EngineType engine_type) = 0; - virtual Status Cache() = 0; + virtual Status + Cache() = 0; - virtual Status GpuCache(uint64_t gpu_id) = 0; + virtual Status + GpuCache(uint64_t gpu_id) = 0; - virtual Status Init() = 0; + virtual Status + Init() = 0; - virtual EngineType IndexEngineType() const = 0; + virtual EngineType + IndexEngineType() const = 0; - virtual MetricType IndexMetricType() const = 0; + virtual MetricType + IndexMetricType() const = 0; - virtual std::string GetLocation() const = 0; + virtual std::string + GetLocation() const = 0; }; using ExecutionEnginePtr = std::shared_ptr; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index a9937092fd..e8f9ba395d 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -16,20 +16,20 @@ // under the License. #include "db/engine/ExecutionEngineImpl.h" -#include "cache/GpuCacheMgr.h" #include "cache/CpuCacheMgr.h" +#include "cache/GpuCacheMgr.h" #include "metrics/Metrics.h" -#include "utils/Log.h" #include "utils/CommonUtil.h" #include "utils/Exception.h" +#include "utils/Log.h" -#include "src/wrapper/VecIndex.h" -#include "src/wrapper/VecImpl.h" -#include "knowhere/common/Exception.h" #include "knowhere/common/Config.h" -#include "wrapper/ConfAdapterMgr.h" -#include "wrapper/ConfAdapter.h" +#include "knowhere/common/Exception.h" #include "server/Config.h" +#include "src/wrapper/VecImpl.h" +#include "src/wrapper/VecIndex.h" +#include "wrapper/ConfAdapter.h" +#include "wrapper/ConfAdapterMgr.h" #include #include @@ -38,17 +38,9 @@ namespace zilliz { namespace milvus { namespace engine { -ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, - const std::string &location, - EngineType index_type, - MetricType metric_type, - int32_t nlist) - : location_(location), - dim_(dimension), - index_type_(index_type), - metric_type_(metric_type), - nlist_(nlist) { - +ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string& location, EngineType index_type, + MetricType metric_type, int32_t nlist) + : location_(location), dim_(dimension), index_type_(index_type), metric_type_(metric_type), nlist_(nlist) { index_ = CreatetVecIndex(EngineType::FAISS_IDMAP); if (!index_) { throw Exception(DB_ERROR, "Could not create VecIndex"); @@ -57,8 +49,7 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, TempMetaConf temp_conf; temp_conf.gpu_id = gpu_num_; temp_conf.dim = dimension; - temp_conf.metric_type = (metric_type_ == MetricType::IP) ? - knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2; + temp_conf.metric_type = (metric_type_ == MetricType::IP) ? knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2; auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType()); auto conf = adapter->Match(temp_conf); @@ -68,16 +59,9 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, } } -ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index, - const std::string &location, - EngineType index_type, - MetricType metric_type, - int32_t nlist) - : index_(std::move(index)), - location_(location), - index_type_(index_type), - metric_type_(metric_type), - nlist_(nlist) { +ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index, const std::string& location, EngineType index_type, + MetricType metric_type, int32_t nlist) + : index_(std::move(index)), location_(location), index_type_(index_type), metric_type_(metric_type), nlist_(nlist) { } VecIndexPtr @@ -109,7 +93,7 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) { } Status -ExecutionEngineImpl::AddWithIds(int64_t n, const float *xdata, const int64_t *xids) { +ExecutionEngineImpl::AddWithIds(int64_t n, const float* xdata, const int64_t* xids) { auto status = index_->Add(n, xdata, xids); return status; } @@ -125,7 +109,7 @@ ExecutionEngineImpl::Count() const { size_t ExecutionEngineImpl::Size() const { - return (size_t) (Count() * Dimension()) * sizeof(float); + return (size_t)(Count() * Dimension()) * sizeof(float); } size_t @@ -164,7 +148,7 @@ ExecutionEngineImpl::Load(bool to_cache) { } else { ENGINE_LOG_DEBUG << "Disk io from: " << location_; } - } catch (std::exception &e) { + } catch (std::exception& e) { ENGINE_LOG_ERROR << e.what(); return Status(DB_ERROR, e.what()); } @@ -191,7 +175,7 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id) { try { index_ = index_->CopyToGpu(device_id); ENGINE_LOG_DEBUG << "CPU to GPU" << device_id; - } catch (std::exception &e) { + } catch (std::exception& e) { ENGINE_LOG_ERROR << e.what(); return Status(DB_ERROR, e.what()); } @@ -219,7 +203,7 @@ ExecutionEngineImpl::CopyToCpu() { try { index_ = index_->CopyToCpu(); ENGINE_LOG_DEBUG << "GPU to CPU"; - } catch (std::exception &e) { + } catch (std::exception& e) { ENGINE_LOG_ERROR << e.what(); return Status(DB_ERROR, e.what()); } @@ -245,7 +229,7 @@ ExecutionEngineImpl::Clone() { } Status -ExecutionEngineImpl::Merge(const std::string &location) { +ExecutionEngineImpl::Merge(const std::string& location) { if (location == location_) { return Status(DB_ERROR, "Cannot Merge Self"); } @@ -257,7 +241,7 @@ ExecutionEngineImpl::Merge(const std::string &location) { double physical_size = server::CommonUtil::GetFileSize(location); server::CollectExecutionEngineMetrics metrics(physical_size); to_merge = read_index(location); - } catch (std::exception &e) { + } catch (std::exception& e) { ENGINE_LOG_ERROR << e.what(); return Status(DB_ERROR, e.what()); } @@ -280,7 +264,7 @@ ExecutionEngineImpl::Merge(const std::string &location) { } ExecutionEnginePtr -ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_type) { +ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_type) { ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_; auto from_index = std::dynamic_pointer_cast(index_); @@ -298,29 +282,23 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t temp_conf.gpu_id = gpu_num_; temp_conf.dim = Dimension(); temp_conf.nlist = nlist_; - temp_conf.metric_type = (metric_type_ == MetricType::IP) ? - knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2; + temp_conf.metric_type = (metric_type_ == MetricType::IP) ? knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2; temp_conf.size = Count(); auto adapter = AdapterMgr::GetInstance().GetAdapter(to_index->GetType()); auto conf = adapter->Match(temp_conf); - auto status = to_index->BuildAll(Count(), - from_index->GetRawVectors(), - from_index->GetRawIds(), - conf); - if (!status.ok()) { throw Exception(DB_ERROR, status.message()); } + auto status = to_index->BuildAll(Count(), from_index->GetRawVectors(), from_index->GetRawIds(), conf); + if (!status.ok()) { + throw Exception(DB_ERROR, status.message()); + } return std::make_shared(to_index, location, engine_type, metric_type_, nlist_); } Status -ExecutionEngineImpl::Search(int64_t n, - const float *data, - int64_t k, - int64_t nprobe, - float *distances, - int64_t *labels) const { +ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t nprobe, float* distances, + int64_t* labels) const { if (index_ == nullptr) { ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search"; return Status(DB_ERROR, "index is null"); @@ -362,13 +340,13 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { // TODO(linxj): remove. Status ExecutionEngineImpl::Init() { - server::Config &config = server::Config::GetInstance(); + server::Config& config = server::Config::GetInstance(); Status s = config.GetDBConfigBuildIndexGPU(gpu_num_); if (!s.ok()) return s; return Status::OK(); } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/engine/ExecutionEngineImpl.h b/cpp/src/db/engine/ExecutionEngineImpl.h index 6ea09ddb29..e8f9521460 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.h +++ b/cpp/src/db/engine/ExecutionEngineImpl.h @@ -29,71 +29,81 @@ namespace engine { class ExecutionEngineImpl : public ExecutionEngine { public: - ExecutionEngineImpl(uint16_t dimension, - const std::string &location, - EngineType index_type, - MetricType metric_type, + ExecutionEngineImpl(uint16_t dimension, const std::string& location, EngineType index_type, MetricType metric_type, int32_t nlist); - ExecutionEngineImpl(VecIndexPtr index, - const std::string &location, - EngineType index_type, - MetricType metric_type, + ExecutionEngineImpl(VecIndexPtr index, const std::string& location, EngineType index_type, MetricType metric_type, int32_t nlist); - Status AddWithIds(int64_t n, const float *xdata, const int64_t *xids) override; + Status + AddWithIds(int64_t n, const float* xdata, const int64_t* xids) override; - size_t Count() const override; + size_t + Count() const override; - size_t Size() const override; + size_t + Size() const override; - size_t Dimension() const override; + size_t + Dimension() const override; - size_t PhysicalSize() const override; + size_t + PhysicalSize() const override; - Status Serialize() override; + Status + Serialize() override; - Status Load(bool to_cache) override; + Status + Load(bool to_cache) override; - Status CopyToGpu(uint64_t device_id) override; + Status + CopyToGpu(uint64_t device_id) override; - Status CopyToCpu() override; + Status + CopyToCpu() override; - ExecutionEnginePtr Clone() override; + ExecutionEnginePtr + Clone() override; - Status Merge(const std::string &location) override; + Status + Merge(const std::string& location) override; - Status Search(int64_t n, - const float *data, - int64_t k, - int64_t nprobe, - float *distances, - int64_t *labels) const override; + Status + Search(int64_t n, const float* data, int64_t k, int64_t nprobe, float* distances, int64_t* labels) const override; - ExecutionEnginePtr BuildIndex(const std::string &location, EngineType engine_type) override; + ExecutionEnginePtr + BuildIndex(const std::string& location, EngineType engine_type) override; - Status Cache() override; + Status + Cache() override; - Status GpuCache(uint64_t gpu_id) override; + Status + GpuCache(uint64_t gpu_id) override; - Status Init() override; + Status + Init() override; - EngineType IndexEngineType() const override { + EngineType + IndexEngineType() const override { return index_type_; } - MetricType IndexMetricType() const override { + MetricType + IndexMetricType() const override { return metric_type_; } - std::string GetLocation() const override { + std::string + GetLocation() const override { return location_; } private: - VecIndexPtr CreatetVecIndex(EngineType type); + VecIndexPtr + CreatetVecIndex(EngineType type); - VecIndexPtr Load(const std::string &location); + VecIndexPtr + Load(const std::string& location); protected: VecIndexPtr index_ = nullptr; @@ -107,6 +117,6 @@ class ExecutionEngineImpl : public ExecutionEngine { int32_t gpu_num_ = 0; }; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemManager.h b/cpp/src/db/insert/MemManager.h index 751633bf67..920e6e9692 100644 --- a/cpp/src/db/insert/MemManager.h +++ b/cpp/src/db/insert/MemManager.h @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include "utils/Status.h" #include "db/Types.h" +#include "utils/Status.h" -#include #include +#include #include namespace zilliz { @@ -31,22 +30,27 @@ namespace engine { class MemManager { public: - virtual Status InsertVectors(const std::string &table_id, - size_t n, const float *vectors, IDNumbers &vector_ids) = 0; + virtual Status + InsertVectors(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids) = 0; - virtual Status Serialize(std::set &table_ids) = 0; + virtual Status + Serialize(std::set& table_ids) = 0; - virtual Status EraseMemVector(const std::string &table_id) = 0; + virtual Status + EraseMemVector(const std::string& table_id) = 0; - virtual size_t GetCurrentMutableMem() = 0; + virtual size_t + GetCurrentMutableMem() = 0; - virtual size_t GetCurrentImmutableMem() = 0; + virtual size_t + GetCurrentImmutableMem() = 0; - virtual size_t GetCurrentMem() = 0; -}; // MemManagerAbstract + virtual size_t + GetCurrentMem() = 0; +}; // MemManagerAbstract using MemManagerPtr = std::shared_ptr; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemManagerImpl.cpp b/cpp/src/db/insert/MemManagerImpl.cpp index e555acd28d..1a59c8b892 100644 --- a/cpp/src/db/insert/MemManagerImpl.cpp +++ b/cpp/src/db/insert/MemManagerImpl.cpp @@ -15,11 +15,10 @@ // specific language governing permissions and limitations // under the License. - #include "db/insert/MemManagerImpl.h" #include "VectorSource.h" -#include "utils/Log.h" #include "db/Constants.h" +#include "utils/Log.h" #include @@ -28,7 +27,7 @@ namespace milvus { namespace engine { MemTablePtr -MemManagerImpl::GetMemByTable(const std::string &table_id) { +MemManagerImpl::GetMemByTable(const std::string& table_id) { auto memIt = mem_id_map_.find(table_id); if (memIt != mem_id_map_.end()) { return memIt->second; @@ -39,10 +38,7 @@ MemManagerImpl::GetMemByTable(const std::string &table_id) { } Status -MemManagerImpl::InsertVectors(const std::string &table_id_, - size_t n_, - const float *vectors_, - IDNumbers &vector_ids_) { +MemManagerImpl::InsertVectors(const std::string& table_id_, size_t n_, const float* vectors_, IDNumbers& vector_ids_) { while (GetCurrentMem() > options_.insert_buffer_size_) { std::this_thread::sleep_for(std::chrono::milliseconds(1)); } @@ -53,10 +49,8 @@ MemManagerImpl::InsertVectors(const std::string &table_id_, } Status -MemManagerImpl::InsertVectorsNoLock(const std::string &table_id, - size_t n, - const float *vectors, - IDNumbers &vector_ids) { +MemManagerImpl::InsertVectorsNoLock(const std::string& table_id, size_t n, const float* vectors, + IDNumbers& vector_ids) { MemTablePtr mem = GetMemByTable(table_id); VectorSourcePtr source = std::make_shared(n, vectors); @@ -73,9 +67,9 @@ Status MemManagerImpl::ToImmutable() { std::unique_lock lock(mutex_); MemIdMap temp_map; - for (auto &kv : mem_id_map_) { + for (auto& kv : mem_id_map_) { if (kv.second->Empty()) { - //empty table, no need to serialize + // empty table, no need to serialize temp_map.insert(kv); } else { immu_mem_list_.push_back(kv.second); @@ -87,11 +81,11 @@ MemManagerImpl::ToImmutable() { } Status -MemManagerImpl::Serialize(std::set &table_ids) { +MemManagerImpl::Serialize(std::set& table_ids) { ToImmutable(); std::unique_lock lock(serialization_mtx_); table_ids.clear(); - for (auto &mem : immu_mem_list_) { + for (auto& mem : immu_mem_list_) { mem->Serialize(); table_ids.insert(mem->GetTableId()); } @@ -100,16 +94,16 @@ MemManagerImpl::Serialize(std::set &table_ids) { } Status -MemManagerImpl::EraseMemVector(const std::string &table_id) { - {//erase MemVector from rapid-insert cache +MemManagerImpl::EraseMemVector(const std::string& table_id) { + { // erase MemVector from rapid-insert cache std::unique_lock lock(mutex_); mem_id_map_.erase(table_id); } - {//erase MemVector from serialize cache + { // erase MemVector from serialize cache std::unique_lock lock(serialization_mtx_); MemList temp_list; - for (auto &mem : immu_mem_list_) { + for (auto& mem : immu_mem_list_) { if (mem->GetTableId() != table_id) { temp_list.push_back(mem); } @@ -123,7 +117,7 @@ MemManagerImpl::EraseMemVector(const std::string &table_id) { size_t MemManagerImpl::GetCurrentMutableMem() { size_t total_mem = 0; - for (auto &kv : mem_id_map_) { + for (auto& kv : mem_id_map_) { auto memTable = kv.second; total_mem += memTable->GetCurrentMem(); } @@ -133,7 +127,7 @@ MemManagerImpl::GetCurrentMutableMem() { size_t MemManagerImpl::GetCurrentImmutableMem() { size_t total_mem = 0; - for (auto &mem_table : immu_mem_list_) { + for (auto& mem_table : immu_mem_list_) { total_mem += mem_table->GetCurrentMem(); } return total_mem; @@ -144,6 +138,6 @@ MemManagerImpl::GetCurrentMem() { return GetCurrentMutableMem() + GetCurrentImmutableMem(); } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemManagerImpl.h b/cpp/src/db/insert/MemManagerImpl.h index 1783adec18..4de5626459 100644 --- a/cpp/src/db/insert/MemManagerImpl.h +++ b/cpp/src/db/insert/MemManagerImpl.h @@ -15,21 +15,20 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include "MemTable.h" #include "MemManager.h" +#include "MemTable.h" #include "db/meta/Meta.h" #include "utils/Status.h" -#include -#include -#include -#include #include +#include #include #include +#include +#include +#include namespace zilliz { namespace milvus { @@ -39,29 +38,35 @@ class MemManagerImpl : public MemManager { public: using Ptr = std::shared_ptr; - MemManagerImpl(const meta::MetaPtr &meta, const DBOptions &options) - : meta_(meta), options_(options) { + MemManagerImpl(const meta::MetaPtr& meta, const DBOptions& options) : meta_(meta), options_(options) { } - Status InsertVectors(const std::string &table_id, - size_t n, const float *vectors, IDNumbers &vector_ids) override; + Status + InsertVectors(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids) override; - Status Serialize(std::set &table_ids) override; + Status + Serialize(std::set& table_ids) override; - Status EraseMemVector(const std::string &table_id) override; + Status + EraseMemVector(const std::string& table_id) override; - size_t GetCurrentMutableMem() override; + size_t + GetCurrentMutableMem() override; - size_t GetCurrentImmutableMem() override; + size_t + GetCurrentImmutableMem() override; - size_t GetCurrentMem() override; + size_t + GetCurrentMem() override; private: - MemTablePtr GetMemByTable(const std::string &table_id); + MemTablePtr + GetMemByTable(const std::string& table_id); - Status InsertVectorsNoLock(const std::string &table_id, - size_t n, const float *vectors, IDNumbers &vector_ids); - Status ToImmutable(); + Status + InsertVectorsNoLock(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids); + Status + ToImmutable(); using MemIdMap = std::map; using MemList = std::vector; @@ -71,8 +76,8 @@ class MemManagerImpl : public MemManager { DBOptions options_; std::mutex mutex_; std::mutex serialization_mtx_; -}; // NewMemManager +}; // NewMemManager -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemMenagerFactory.cpp b/cpp/src/db/insert/MemMenagerFactory.cpp index 76e521e168..8f0d6704d7 100644 --- a/cpp/src/db/insert/MemMenagerFactory.cpp +++ b/cpp/src/db/insert/MemMenagerFactory.cpp @@ -17,26 +17,26 @@ #include "db/insert/MemMenagerFactory.h" #include "MemManagerImpl.h" -#include "utils/Log.h" #include "utils/Exception.h" +#include "utils/Log.h" #include #include -#include #include -#include -#include #include +#include +#include +#include namespace zilliz { namespace milvus { namespace engine { MemManagerPtr -MemManagerFactory::Build(const std::shared_ptr &meta, const DBOptions &options) { +MemManagerFactory::Build(const std::shared_ptr& meta, const DBOptions& options) { return std::make_shared(meta, options); } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemMenagerFactory.h b/cpp/src/db/insert/MemMenagerFactory.h index fa0cb5930a..d7996abe99 100644 --- a/cpp/src/db/insert/MemMenagerFactory.h +++ b/cpp/src/db/insert/MemMenagerFactory.h @@ -28,9 +28,10 @@ namespace engine { class MemManagerFactory { public: - static MemManagerPtr Build(const std::shared_ptr &meta, const DBOptions &options); + static MemManagerPtr + Build(const std::shared_ptr& meta, const DBOptions& options); }; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemTable.cpp b/cpp/src/db/insert/MemTable.cpp index 8db6a9d141..e189dbf15b 100644 --- a/cpp/src/db/insert/MemTable.cpp +++ b/cpp/src/db/insert/MemTable.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include "db/insert/MemTable.h" #include "utils/Log.h" @@ -26,16 +25,12 @@ namespace zilliz { namespace milvus { namespace engine { -MemTable::MemTable(const std::string &table_id, - const meta::MetaPtr &meta, - const DBOptions &options) : - table_id_(table_id), - meta_(meta), - options_(options) { +MemTable::MemTable(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options) + : table_id_(table_id), meta_(meta), options_(options) { } Status -MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) { +MemTable::Add(VectorSourcePtr& source, IDNumbers& vector_ids) { while (!source->AllAdded()) { MemTableFilePtr current_mem_table_file; if (!mem_table_file_list_.empty()) { @@ -63,7 +58,7 @@ MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) { } void -MemTable::GetCurrentMemTableFile(MemTableFilePtr &mem_table_file) { +MemTable::GetCurrentMemTableFile(MemTableFilePtr& mem_table_file) { mem_table_file = mem_table_file_list_.back(); } @@ -92,7 +87,7 @@ MemTable::Empty() { return mem_table_file_list_.empty(); } -const std::string & +const std::string& MemTable::GetTableId() const { return table_id_; } @@ -101,12 +96,12 @@ size_t MemTable::GetCurrentMem() { std::lock_guard lock(mutex_); size_t total_mem = 0; - for (auto &mem_table_file : mem_table_file_list_) { + for (auto& mem_table_file : mem_table_file_list_) { total_mem += mem_table_file->GetCurrentMem(); } return total_mem; } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemTable.h b/cpp/src/db/insert/MemTable.h index da7d914b41..b2b16cf472 100644 --- a/cpp/src/db/insert/MemTable.h +++ b/cpp/src/db/insert/MemTable.h @@ -15,17 +15,16 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "MemTableFile.h" #include "VectorSource.h" #include "utils/Status.h" -#include -#include #include +#include #include +#include namespace zilliz { namespace milvus { @@ -35,21 +34,28 @@ class MemTable { public: using MemTableFileList = std::vector; - MemTable(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options); + MemTable(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options); - Status Add(VectorSourcePtr &source, IDNumbers &vector_ids); + Status + Add(VectorSourcePtr& source, IDNumbers& vector_ids); - void GetCurrentMemTableFile(MemTableFilePtr &mem_table_file); + void + GetCurrentMemTableFile(MemTableFilePtr& mem_table_file); - size_t GetTableFileCount(); + size_t + GetTableFileCount(); - Status Serialize(); + Status + Serialize(); - bool Empty(); + bool + Empty(); - const std::string &GetTableId() const; + const std::string& + GetTableId() const; - size_t GetCurrentMem(); + size_t + GetCurrentMem(); private: const std::string table_id_; @@ -61,10 +67,10 @@ class MemTable { DBOptions options_; std::mutex mutex_; -}; //MemTable +}; // MemTable using MemTablePtr = std::shared_ptr; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index fc6c2b319a..8395b0b62a 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include "db/insert/MemTableFile.h" #include "db/Constants.h" #include "db/engine/EngineFactory.h" @@ -29,20 +28,14 @@ namespace zilliz { namespace milvus { namespace engine { -MemTableFile::MemTableFile(const std::string &table_id, - const meta::MetaPtr &meta, - const DBOptions &options) : - table_id_(table_id), - meta_(meta), - options_(options) { +MemTableFile::MemTableFile(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options) + : table_id_(table_id), meta_(meta), options_(options) { current_mem_ = 0; auto status = CreateTableFile(); if (status.ok()) { - execution_engine_ = EngineFactory::Build(table_file_schema_.dimension_, - table_file_schema_.location_, - (EngineType) table_file_schema_.engine_type_, - (MetricType) table_file_schema_.metric_type_, - table_file_schema_.nlist_); + execution_engine_ = EngineFactory::Build( + table_file_schema_.dimension_, table_file_schema_.location_, (EngineType)table_file_schema_.engine_type_, + (MetricType)table_file_schema_.metric_type_, table_file_schema_.nlist_); } } @@ -61,10 +54,11 @@ MemTableFile::CreateTableFile() { } Status -MemTableFile::Add(const VectorSourcePtr &source, IDNumbers &vector_ids) { +MemTableFile::Add(const VectorSourcePtr& source, IDNumbers& vector_ids) { if (table_file_schema_.dimension_ <= 0) { std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " + - std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_; + std::to_string(table_file_schema_.dimension_) + ", table_id = " + + table_file_schema_.table_id_; ENGINE_LOG_ERROR << err_msg; return Status(DB_ERROR, "Not able to create table file"); } @@ -109,11 +103,11 @@ MemTableFile::Serialize() { table_file_schema_.file_size_ = execution_engine_->PhysicalSize(); table_file_schema_.row_count_ = execution_engine_->Count(); - //if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size - //else set file type to RAW, no need to build index - if (table_file_schema_.engine_type_ != (int) EngineType::FAISS_IDMAP) { - table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ? - meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; + // if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size + // else set file type to RAW, no need to build index + if (table_file_schema_.engine_type_ != (int)EngineType::FAISS_IDMAP) { + table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ? meta::TableFileSchema::TO_INDEX + : meta::TableFileSchema::RAW; } else { table_file_schema_.file_type_ = meta::TableFileSchema::RAW; } @@ -130,6 +124,6 @@ MemTableFile::Serialize() { return status; } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/MemTableFile.h b/cpp/src/db/insert/MemTableFile.h index 1d7019a42d..72ae09c861 100644 --- a/cpp/src/db/insert/MemTableFile.h +++ b/cpp/src/db/insert/MemTableFile.h @@ -15,16 +15,15 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "VectorSource.h" -#include "db/meta/Meta.h" #include "db/engine/ExecutionEngine.h" +#include "db/meta/Meta.h" #include "utils/Status.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -32,20 +31,26 @@ namespace engine { class MemTableFile { public: - MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options); + MemTableFile(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options); - Status Add(const VectorSourcePtr &source, IDNumbers &vector_ids); + Status + Add(const VectorSourcePtr& source, IDNumbers& vector_ids); - size_t GetCurrentMem(); + size_t + GetCurrentMem(); - size_t GetMemLeft(); + size_t + GetMemLeft(); - bool IsFull(); + bool + IsFull(); - Status Serialize(); + Status + Serialize(); private: - Status CreateTableFile(); + Status + CreateTableFile(); private: const std::string table_id_; @@ -55,10 +60,10 @@ class MemTableFile { size_t current_mem_; ExecutionEnginePtr execution_engine_; -}; //MemTableFile +}; // MemTableFile using MemTableFilePtr = std::shared_ptr; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/VectorSource.cpp b/cpp/src/db/insert/VectorSource.cpp index 330ef08b98..7a9fdf5d0f 100644 --- a/cpp/src/db/insert/VectorSource.cpp +++ b/cpp/src/db/insert/VectorSource.cpp @@ -15,35 +15,28 @@ // specific language governing permissions and limitations // under the License. - #include "db/insert/VectorSource.h" -#include "db/engine/ExecutionEngine.h" #include "db/engine/EngineFactory.h" -#include "utils/Log.h" +#include "db/engine/ExecutionEngine.h" #include "metrics/Metrics.h" +#include "utils/Log.h" namespace zilliz { namespace milvus { namespace engine { -VectorSource::VectorSource(const size_t &n, - const float *vectors) : - n_(n), - vectors_(vectors), - id_generator_(std::make_shared()) { +VectorSource::VectorSource(const size_t& n, const float* vectors) + : n_(n), vectors_(vectors), id_generator_(std::make_shared()) { current_num_vectors_added = 0; } Status -VectorSource::Add(const ExecutionEnginePtr &execution_engine, - const meta::TableFileSchema &table_file_schema, - const size_t &num_vectors_to_add, - size_t &num_vectors_added, - IDNumbers &vector_ids) { +VectorSource::Add(const ExecutionEnginePtr& execution_engine, const meta::TableFileSchema& table_file_schema, + const size_t& num_vectors_to_add, size_t& num_vectors_added, IDNumbers& vector_ids) { server::CollectAddMetrics metrics(n_, table_file_schema.dimension_); - num_vectors_added = current_num_vectors_added + num_vectors_to_add <= n_ ? - num_vectors_to_add : n_ - current_num_vectors_added; + num_vectors_added = + current_num_vectors_added + num_vectors_to_add <= n_ ? num_vectors_to_add : n_ - current_num_vectors_added; IDNumbers vector_ids_to_add; if (vector_ids.empty()) { id_generator_->GetNextIDNumbers(num_vectors_added, vector_ids_to_add); @@ -58,8 +51,7 @@ VectorSource::Add(const ExecutionEnginePtr &execution_engine, vector_ids_to_add.data()); if (status.ok()) { current_num_vectors_added += num_vectors_added; - vector_ids_.insert(vector_ids_.end(), - std::make_move_iterator(vector_ids_to_add.begin()), + vector_ids_.insert(vector_ids_.end(), std::make_move_iterator(vector_ids_to_add.begin()), std::make_move_iterator(vector_ids_to_add.end())); } else { ENGINE_LOG_ERROR << "VectorSource::Add failed: " + status.ToString(); @@ -83,6 +75,6 @@ VectorSource::GetVectorIds() { return vector_ids_; } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/insert/VectorSource.h b/cpp/src/db/insert/VectorSource.h index fd31a14fa6..4b8e9d21b9 100644 --- a/cpp/src/db/insert/VectorSource.h +++ b/cpp/src/db/insert/VectorSource.h @@ -15,12 +15,11 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include "db/meta/Meta.h" #include "db/IDGenerator.h" #include "db/engine/ExecutionEngine.h" +#include "db/meta/Meta.h" #include "utils/Status.h" #include @@ -31,32 +30,33 @@ namespace engine { class VectorSource { public: - VectorSource(const size_t &n, const float *vectors); + VectorSource(const size_t& n, const float* vectors); - Status Add(const ExecutionEnginePtr &execution_engine, - const meta::TableFileSchema &table_file_schema, - const size_t &num_vectors_to_add, - size_t &num_vectors_added, - IDNumbers &vector_ids); + Status + Add(const ExecutionEnginePtr& execution_engine, const meta::TableFileSchema& table_file_schema, + const size_t& num_vectors_to_add, size_t& num_vectors_added, IDNumbers& vector_ids); - size_t GetNumVectorsAdded(); + size_t + GetNumVectorsAdded(); - bool AllAdded(); + bool + AllAdded(); - IDNumbers GetVectorIds(); + IDNumbers + GetVectorIds(); private: const size_t n_; - const float *vectors_; + const float* vectors_; IDNumbers vector_ids_; size_t current_num_vectors_added; std::shared_ptr id_generator_; -}; //VectorSource +}; // VectorSource using VectorSourcePtr = std::shared_ptr; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/Meta.h b/cpp/src/db/meta/Meta.h index 42fe6a6dd5..3e3128366c 100644 --- a/cpp/src/db/meta/Meta.h +++ b/cpp/src/db/meta/Meta.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "MetaTypes.h" @@ -25,84 +24,104 @@ #include #include -#include #include +#include namespace zilliz { namespace milvus { namespace engine { namespace meta { -static const char *META_TABLES = "Tables"; -static const char *META_TABLEFILES = "TableFiles"; +static const char* META_TABLES = "Tables"; +static const char* META_TABLEFILES = "TableFiles"; class Meta { public: virtual ~Meta() = default; - virtual Status CreateTable(TableSchema &table_schema) = 0; + virtual Status + CreateTable(TableSchema& table_schema) = 0; - virtual Status DescribeTable(TableSchema &table_schema) = 0; + virtual Status + DescribeTable(TableSchema& table_schema) = 0; - virtual Status HasTable(const std::string &table_id, bool &has_or_not) = 0; + virtual Status + HasTable(const std::string& table_id, bool& has_or_not) = 0; - virtual Status AllTables(std::vector &table_schema_array) = 0; + virtual Status + AllTables(std::vector& table_schema_array) = 0; - virtual Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) = 0; + virtual Status + UpdateTableIndex(const std::string& table_id, const TableIndex& index) = 0; - virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0; + virtual Status + UpdateTableFlag(const std::string& table_id, int64_t flag) = 0; - virtual Status DeleteTable(const std::string &table_id) = 0; + virtual Status + DeleteTable(const std::string& table_id) = 0; - virtual Status DeleteTableFiles(const std::string &table_id) = 0; + virtual Status + DeleteTableFiles(const std::string& table_id) = 0; - virtual Status CreateTableFile(TableFileSchema &file_schema) = 0; + virtual Status + CreateTableFile(TableFileSchema& file_schema) = 0; - virtual Status DropPartitionsByDates(const std::string &table_id, const DatesT &dates) = 0; + virtual Status + DropPartitionsByDates(const std::string& table_id, const DatesT& dates) = 0; - virtual Status GetTableFiles(const std::string &table_id, - const std::vector &ids, - TableFilesSchema &table_files) = 0; + virtual Status + GetTableFiles(const std::string& table_id, const std::vector& ids, TableFilesSchema& table_files) = 0; - virtual Status UpdateTableFilesToIndex(const std::string &table_id) = 0; + virtual Status + UpdateTableFilesToIndex(const std::string& table_id) = 0; - virtual Status UpdateTableFile(TableFileSchema &file_schema) = 0; + virtual Status + UpdateTableFile(TableFileSchema& file_schema) = 0; - virtual Status UpdateTableFiles(TableFilesSchema &files) = 0; + virtual Status + UpdateTableFiles(TableFilesSchema& files) = 0; - virtual Status FilesToSearch(const std::string &table_id, - const std::vector &ids, - const DatesT &partition, - DatePartionedTableFilesSchema &files) = 0; + virtual Status + FilesToSearch(const std::string& table_id, const std::vector& ids, const DatesT& partition, + DatePartionedTableFilesSchema& files) = 0; - virtual Status FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) = 0; + virtual Status + FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) = 0; - virtual Status Size(uint64_t &result) = 0; + virtual Status + Size(uint64_t& result) = 0; - virtual Status Archive() = 0; + virtual Status + Archive() = 0; - virtual Status FilesToIndex(TableFilesSchema &) = 0; + virtual Status + FilesToIndex(TableFilesSchema&) = 0; - virtual Status FilesByType(const std::string &table_id, - const std::vector &file_types, - std::vector &file_ids) = 0; + virtual Status + FilesByType(const std::string& table_id, const std::vector& file_types, + std::vector& file_ids) = 0; - virtual Status DescribeTableIndex(const std::string &table_id, TableIndex &index) = 0; + virtual Status + DescribeTableIndex(const std::string& table_id, TableIndex& index) = 0; - virtual Status DropTableIndex(const std::string &table_id) = 0; + virtual Status + DropTableIndex(const std::string& table_id) = 0; - virtual Status CleanUp() = 0; + virtual Status + CleanUp() = 0; virtual Status CleanUpFilesWithTTL(uint16_t) = 0; - virtual Status DropAll() = 0; + virtual Status + DropAll() = 0; - virtual Status Count(const std::string &table_id, uint64_t &result) = 0; -}; // MetaData + virtual Status + Count(const std::string& table_id, uint64_t& result) = 0; +}; // MetaData using MetaPtr = std::shared_ptr; -} // namespace meta -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/MetaConsts.h b/cpp/src/db/meta/MetaConsts.h index 6abae564e8..64cfeab78a 100644 --- a/cpp/src/db/meta/MetaConsts.h +++ b/cpp/src/db/meta/MetaConsts.h @@ -38,7 +38,7 @@ const size_t H_SEC = 60 * M_SEC; const size_t D_SEC = 24 * H_SEC; const size_t W_SEC = 7 * D_SEC; -} // namespace meta -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/MetaFactory.cpp b/cpp/src/db/meta/MetaFactory.cpp index 6a305a4280..d8beecbd4e 100644 --- a/cpp/src/db/meta/MetaFactory.cpp +++ b/cpp/src/db/meta/MetaFactory.cpp @@ -16,26 +16,26 @@ // under the License. #include "db/meta/MetaFactory.h" -#include "SqliteMetaImpl.h" #include "MySQLMetaImpl.h" -#include "utils/Log.h" -#include "utils/Exception.h" +#include "SqliteMetaImpl.h" #include "db/Utils.h" +#include "utils/Exception.h" +#include "utils/Log.h" #include -#include -#include -#include -#include #include +#include +#include #include +#include +#include namespace zilliz { namespace milvus { namespace engine { DBMetaOptions -MetaFactory::BuildOption(const std::string &path) { +MetaFactory::BuildOption(const std::string& path) { auto p = path; if (p == "") { srand(time(nullptr)); @@ -51,7 +51,7 @@ MetaFactory::BuildOption(const std::string &path) { } meta::MetaPtr -MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) { +MetaFactory::Build(const DBMetaOptions& metaOptions, const int& mode) { std::string uri = metaOptions.backend_uri_; utils::MetaUriInfo uri_info; @@ -73,6 +73,6 @@ MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) { } } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/MetaFactory.h b/cpp/src/db/meta/MetaFactory.h index a13a09715a..0792515ac1 100644 --- a/cpp/src/db/meta/MetaFactory.h +++ b/cpp/src/db/meta/MetaFactory.h @@ -28,11 +28,13 @@ namespace engine { class MetaFactory { public: - static DBMetaOptions BuildOption(const std::string &path = ""); + static DBMetaOptions + BuildOption(const std::string& path = ""); - static meta::MetaPtr Build(const DBMetaOptions &metaOptions, const int &mode); + static meta::MetaPtr + Build(const DBMetaOptions& metaOptions, const int& mode); }; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/MetaTypes.h b/cpp/src/db/meta/MetaTypes.h index 6fd649bc07..aad8b61c87 100644 --- a/cpp/src/db/meta/MetaTypes.h +++ b/cpp/src/db/meta/MetaTypes.h @@ -17,22 +17,22 @@ #pragma once -#include "db/engine/ExecutionEngine.h" #include "db/Constants.h" +#include "db/engine/ExecutionEngine.h" -#include #include -#include #include +#include +#include namespace zilliz { namespace milvus { namespace engine { namespace meta { -constexpr int32_t DEFAULT_ENGINE_TYPE = (int) EngineType::FAISS_IDMAP; +constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP; constexpr int32_t DEFAULT_NLIST = 16384; -constexpr int32_t DEFAULT_METRIC_TYPE = (int) MetricType::L2; +constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2; constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB; constexpr int64_t FLAG_MASK_NO_USERID = 0x1; @@ -50,7 +50,7 @@ struct TableSchema { size_t id_ = 0; std::string table_id_; - int32_t state_ = (int) NORMAL; + int32_t state_ = (int)NORMAL; uint16_t dimension_ = 0; int64_t created_on_ = 0; int64_t flag_ = 0; @@ -58,7 +58,7 @@ struct TableSchema { int32_t engine_type_ = DEFAULT_ENGINE_TYPE; int32_t nlist_ = DEFAULT_NLIST; int32_t metric_type_ = DEFAULT_METRIC_TYPE; -}; // TableSchema +}; // TableSchema struct TableFileSchema { typedef enum { @@ -83,17 +83,17 @@ struct TableFileSchema { std::string location_; int64_t updated_time_ = 0; int64_t created_on_ = 0; - int64_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; //not persist to meta + int64_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; // not persist to meta int32_t engine_type_ = DEFAULT_ENGINE_TYPE; - int32_t nlist_ = DEFAULT_NLIST; //not persist to meta - int32_t metric_type_ = DEFAULT_METRIC_TYPE; //not persist to meta -}; // TableFileSchema + int32_t nlist_ = DEFAULT_NLIST; // not persist to meta + int32_t metric_type_ = DEFAULT_METRIC_TYPE; // not persist to meta +}; // TableFileSchema using TableFileSchemaPtr = std::shared_ptr; using TableFilesSchema = std::vector; using DatePartionedTableFilesSchema = std::map; -} // namespace meta -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/MySQLConnectionPool.cpp b/cpp/src/db/meta/MySQLConnectionPool.cpp index 9d612d8b54..2d09874d3f 100644 --- a/cpp/src/db/meta/MySQLConnectionPool.cpp +++ b/cpp/src/db/meta/MySQLConnectionPool.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include "db/meta/MySQLConnectionPool.h" namespace zilliz { @@ -28,7 +27,7 @@ namespace meta { // already. Can't do this in create() because we're interested in // connections actually in use, not those created. Also note that // we keep our own count; ConnectionPool::size() isn't the same! -mysqlpp::Connection * +mysqlpp::Connection* MySQLConnectionPool::grab() { while (conns_in_use_ > max_pool_size_) { sleep(1); @@ -40,7 +39,7 @@ MySQLConnectionPool::grab() { // Other half of in-use conn count limit void -MySQLConnectionPool::release(const mysqlpp::Connection *pc) { +MySQLConnectionPool::release(const mysqlpp::Connection* pc) { mysqlpp::ConnectionPool::release(pc); if (conns_in_use_ <= 0) { ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = " @@ -64,27 +63,25 @@ MySQLConnectionPool::getDB() { } // Superclass overrides -mysqlpp::Connection * +mysqlpp::Connection* MySQLConnectionPool::create() { try { // Create connection using the parameters we were passed upon // creation. - mysqlpp::Connection *conn = new mysqlpp::Connection(); + mysqlpp::Connection* conn = new mysqlpp::Connection(); conn->set_option(new mysqlpp::ReconnectOption(true)); - conn->connect(db_.empty() ? 0 : db_.c_str(), - server_.empty() ? 0 : server_.c_str(), - user_.empty() ? 0 : user_.c_str(), - password_.empty() ? 0 : password_.c_str(), - port_); + conn->connect(db_.empty() ? 0 : db_.c_str(), server_.empty() ? 0 : server_.c_str(), + user_.empty() ? 0 : user_.c_str(), password_.empty() ? 0 : password_.c_str(), port_); return conn; - } catch (const mysqlpp::ConnectionFailed &er) { - ENGINE_LOG_ERROR << "Failed to connect to database server" << ": " << er.what(); + } catch (const mysqlpp::ConnectionFailed& er) { + ENGINE_LOG_ERROR << "Failed to connect to database server" + << ": " << er.what(); return nullptr; } } void -MySQLConnectionPool::destroy(mysqlpp::Connection *cp) { +MySQLConnectionPool::destroy(mysqlpp::Connection* cp) { // Our superclass can't know how we created the Connection, so // it delegates destruction to us, to be safe. delete cp; @@ -95,7 +92,7 @@ MySQLConnectionPool::max_idle_time() { return max_idle_time_; } -} // namespace meta -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/MySQLConnectionPool.h b/cpp/src/db/meta/MySQLConnectionPool.h index 761f272bc1..1b5345c762 100644 --- a/cpp/src/db/meta/MySQLConnectionPool.h +++ b/cpp/src/db/meta/MySQLConnectionPool.h @@ -15,12 +15,11 @@ // specific language governing permissions and limitations // under the License. - #include "mysql++/mysql++.h" -#include #include #include +#include #include "utils/Log.h" @@ -32,20 +31,16 @@ namespace meta { class MySQLConnectionPool : public mysqlpp::ConnectionPool { public: // The object's only constructor - MySQLConnectionPool(std::string dbName, - std::string userName, - std::string passWord, - std::string serverIp, - int port = 0, - int maxPoolSize = 8) : - db_(dbName), - user_(userName), - password_(passWord), - server_(serverIp), - port_(port), - max_pool_size_(maxPoolSize) { + MySQLConnectionPool(std::string dbName, std::string userName, std::string passWord, std::string serverIp, + int port = 0, int maxPoolSize = 8) + : db_(dbName), + user_(userName), + password_(passWord), + server_(serverIp), + port_(port), + max_pool_size_(maxPoolSize) { conns_in_use_ = 0; - max_idle_time_ = 10; //10 seconds + max_idle_time_ = 10; // 10 seconds } // The destructor. We _must_ call ConnectionPool::clear() here, @@ -54,24 +49,30 @@ class MySQLConnectionPool : public mysqlpp::ConnectionPool { clear(); } - mysqlpp::Connection *grab() override; + mysqlpp::Connection* + grab() override; // Other half of in-use conn count limit - void release(const mysqlpp::Connection *pc) override; + void + release(const mysqlpp::Connection* pc) override; -// int getConnectionsInUse(); -// -// void set_max_idle_time(int max_idle); + // int getConnectionsInUse(); + // + // void set_max_idle_time(int max_idle); - std::string getDB(); + std::string + getDB(); protected: // Superclass overrides - mysqlpp::Connection *create() override; + mysqlpp::Connection* + create() override; - void destroy(mysqlpp::Connection *cp) override; + void + destroy(mysqlpp::Connection* cp) override; - unsigned int max_idle_time() override; + unsigned int + max_idle_time() override; private: // Number of connections currently in use @@ -86,7 +87,7 @@ class MySQLConnectionPool : public mysqlpp::ConnectionPool { unsigned int max_idle_time_; }; -} // namespace meta -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 490eda1916..e9654c7866 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -16,27 +16,27 @@ // under the License. #include "db/meta/MySQLMetaImpl.h" +#include "MetaConsts.h" #include "db/IDGenerator.h" #include "db/Utils.h" -#include "utils/Log.h" -#include "utils/Exception.h" -#include "MetaConsts.h" #include "metrics/Metrics.h" +#include "utils/Exception.h" +#include "utils/Log.h" +#include +#include #include -#include -#include +#include #include #include -#include -#include -#include -#include -#include +#include #include +#include +#include #include -#include -#include +#include +#include +#include namespace zilliz { namespace milvus { @@ -46,7 +46,7 @@ namespace meta { namespace { Status -HandleException(const std::string &desc, const char *what = nullptr) { +HandleException(const std::string& desc, const char* what = nullptr) { if (what == nullptr) { ENGINE_LOG_ERROR << desc; return Status(DB_META_TRANSACTION_FAILED, desc); @@ -59,27 +59,28 @@ HandleException(const std::string &desc, const char *what = nullptr) { class MetaField { public: - MetaField(const std::string &name, const std::string &type, const std::string &setting) - : name_(name), - type_(type), - setting_(setting) { + MetaField(const std::string& name, const std::string& type, const std::string& setting) + : name_(name), type_(type), setting_(setting) { } - std::string name() const { + std::string + name() const { return name_; } - std::string ToString() const { + std::string + ToString() const { return name_ + " " + type_ + " " + setting_; } // mysql field type has additional information. for instance, a filed type is defined as 'BIGINT' // we get the type from sql is 'bigint(20)', so we need to ignore the '(20)' - bool IsEqual(const MetaField &field) const { + bool + IsEqual(const MetaField& field) const { size_t name_len_min = field.name_.length() > name_.length() ? name_.length() : field.name_.length(); size_t type_len_min = field.type_.length() > type_.length() ? type_.length() : field.type_.length(); return strncasecmp(field.name_.c_str(), name_.c_str(), name_len_min) == 0 && - strncasecmp(field.type_.c_str(), type_.c_str(), type_len_min) == 0; + strncasecmp(field.type_.c_str(), type_.c_str(), type_len_min) == 0; } private: @@ -91,18 +92,18 @@ class MetaField { using MetaFields = std::vector; class MetaSchema { public: - MetaSchema(const std::string &name, const MetaFields &fields) - : name_(name), - fields_(fields) { + MetaSchema(const std::string& name, const MetaFields& fields) : name_(name), fields_(fields) { } - std::string name() const { + std::string + name() const { return name_; } - std::string ToString() const { + std::string + ToString() const { std::string result; - for (auto &field : fields_) { + for (auto& field : fields_) { if (!result.empty()) { result += ","; } @@ -111,12 +112,13 @@ class MetaSchema { return result; } - //if the outer fields contains all this MetaSchema fields, return true - //otherwise return false - bool IsEqual(const MetaFields &fields) const { + // if the outer fields contains all this MetaSchema fields, return true + // otherwise return false + bool + IsEqual(const MetaFields& fields) const { std::vector found_field; - for (const auto &this_field : fields_) { - for (const auto &outer_field : fields) { + for (const auto& this_field : fields_) { + for (const auto& outer_field : fields) { if (this_field.IsEqual(outer_field)) { found_field.push_back(this_field.name()); break; @@ -132,40 +134,35 @@ class MetaSchema { MetaFields fields_; }; -//Tables schema +// Tables schema static const MetaSchema TABLES_SCHEMA(META_TABLES, { - MetaField("id", "BIGINT", "PRIMARY KEY AUTO_INCREMENT"), - MetaField("table_id", "VARCHAR(255)", "UNIQUE NOT NULL"), - MetaField("state", "INT", "NOT NULL"), - MetaField("dimension", "SMALLINT", "NOT NULL"), - MetaField("created_on", "BIGINT", "NOT NULL"), - MetaField("flag", "BIGINT", "DEFAULT 0 NOT NULL"), - MetaField("index_file_size", "BIGINT", "DEFAULT 1024 NOT NULL"), - MetaField("engine_type", "INT", "DEFAULT 1 NOT NULL"), - MetaField("nlist", "INT", "DEFAULT 16384 NOT NULL"), - MetaField("metric_type", "INT", "DEFAULT 1 NOT NULL"), -}); + MetaField("id", "BIGINT", "PRIMARY KEY AUTO_INCREMENT"), + MetaField("table_id", "VARCHAR(255)", "UNIQUE NOT NULL"), + MetaField("state", "INT", "NOT NULL"), + MetaField("dimension", "SMALLINT", "NOT NULL"), + MetaField("created_on", "BIGINT", "NOT NULL"), + MetaField("flag", "BIGINT", "DEFAULT 0 NOT NULL"), + MetaField("index_file_size", "BIGINT", "DEFAULT 1024 NOT NULL"), + MetaField("engine_type", "INT", "DEFAULT 1 NOT NULL"), + MetaField("nlist", "INT", "DEFAULT 16384 NOT NULL"), + MetaField("metric_type", "INT", "DEFAULT 1 NOT NULL"), + }); -//TableFiles schema -static const MetaSchema TABLEFILES_SCHEMA(META_TABLEFILES, { - MetaField("id", "BIGINT", "PRIMARY KEY AUTO_INCREMENT"), - MetaField("table_id", "VARCHAR(255)", "NOT NULL"), - MetaField("engine_type", "INT", "DEFAULT 1 NOT NULL"), - MetaField("file_id", "VARCHAR(255)", "NOT NULL"), - MetaField("file_type", "INT", "DEFAULT 0 NOT NULL"), - MetaField("file_size", "BIGINT", "DEFAULT 0 NOT NULL"), - MetaField("row_count", "BIGINT", "DEFAULT 0 NOT NULL"), - MetaField("updated_time", "BIGINT", "NOT NULL"), - MetaField("created_on", "BIGINT", "NOT NULL"), - MetaField("date", "INT", "DEFAULT -1 NOT NULL"), -}); +// TableFiles schema +static const MetaSchema TABLEFILES_SCHEMA( + META_TABLEFILES, + { + MetaField("id", "BIGINT", "PRIMARY KEY AUTO_INCREMENT"), MetaField("table_id", "VARCHAR(255)", "NOT NULL"), + MetaField("engine_type", "INT", "DEFAULT 1 NOT NULL"), MetaField("file_id", "VARCHAR(255)", "NOT NULL"), + MetaField("file_type", "INT", "DEFAULT 0 NOT NULL"), MetaField("file_size", "BIGINT", "DEFAULT 0 NOT NULL"), + MetaField("row_count", "BIGINT", "DEFAULT 0 NOT NULL"), MetaField("updated_time", "BIGINT", "NOT NULL"), + MetaField("created_on", "BIGINT", "NOT NULL"), MetaField("date", "INT", "DEFAULT -1 NOT NULL"), + }); -} // namespace +} // namespace //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options, const int &mode) - : options_(options), - mode_(mode) { +MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions& options, const int& mode) : options_(options), mode_(mode) { Initialize(); } @@ -173,7 +170,7 @@ MySQLMetaImpl::~MySQLMetaImpl() { } Status -MySQLMetaImpl::NextTableId(std::string &table_id) { +MySQLMetaImpl::NextTableId(std::string& table_id) { std::stringstream ss; SimpleIDGenerator g; ss << g.GetNextIDNumber(); @@ -182,7 +179,7 @@ MySQLMetaImpl::NextTableId(std::string &table_id) { } Status -MySQLMetaImpl::NextFileId(std::string &file_id) { +MySQLMetaImpl::NextFileId(std::string& file_id) { std::stringstream ss; SimpleIDGenerator g; ss << g.GetNextIDNumber(); @@ -201,7 +198,7 @@ MySQLMetaImpl::ValidateMetaSchema() { return; } - auto validate_func = [&](const MetaSchema &schema) { + auto validate_func = [&](const MetaSchema& schema) { mysqlpp::Query query_statement = connectionPtr->query(); query_statement << "DESC " << schema.name() << ";"; @@ -210,14 +207,14 @@ MySQLMetaImpl::ValidateMetaSchema() { try { mysqlpp::StoreQueryResult res = query_statement.store(); for (size_t i = 0; i < res.num_rows(); i++) { - const mysqlpp::Row &row = res[i]; + const mysqlpp::Row& row = res[i]; std::string name, type; row["Field"].to_string(name); row["Type"].to_string(type); exist_fields.push_back(MetaField(name, type, "")); } - } catch (std::exception &e) { + } catch (std::exception& e) { ENGINE_LOG_DEBUG << "Meta table '" << schema.name() << "' not exist and will be created"; } @@ -228,12 +225,12 @@ MySQLMetaImpl::ValidateMetaSchema() { return schema.IsEqual(exist_fields); }; - //verify Tables + // verify Tables if (!validate_func(TABLES_SCHEMA)) { throw Exception(DB_INCOMPATIB_META, "Meta Tables schema is created by Milvus old version"); } - //verufy TableFiles + // verufy TableFiles if (!validate_func(TABLEFILES_SCHEMA)) { throw Exception(DB_INCOMPATIB_META, "Meta TableFiles schema is created by Milvus old version"); } @@ -241,7 +238,7 @@ MySQLMetaImpl::ValidateMetaSchema() { Status MySQLMetaImpl::Initialize() { - //step 1: create db root path + // step 1: create db root path if (!boost::filesystem::is_directory(options_.path_)) { auto ret = boost::filesystem::create_directory(options_.path_); if (!ret) { @@ -253,7 +250,7 @@ MySQLMetaImpl::Initialize() { std::string uri = options_.backend_uri_; - //step 2: parse and check meta uri + // step 2: parse and check meta uri utils::MetaUriInfo uri_info; auto status = utils::ParseMetaUri(uri, uri_info); if (!status.ok()) { @@ -268,7 +265,7 @@ MySQLMetaImpl::Initialize() { throw Exception(DB_INVALID_META_URI, msg); } - //step 3: connect mysql + // step 3: connect mysql int thread_hint = std::thread::hardware_concurrency(); int max_pool_size = (thread_hint == 0) ? 8 : thread_hint; unsigned int port = 0; @@ -276,15 +273,14 @@ MySQLMetaImpl::Initialize() { port = std::stoi(uri_info.port_); } - mysql_connection_pool_ = - std::make_shared(uri_info.db_name_, uri_info.username_, - uri_info.password_, uri_info.host_, port, max_pool_size); + mysql_connection_pool_ = std::make_shared( + uri_info.db_name_, uri_info.username_, uri_info.password_, uri_info.host_, port, max_pool_size); ENGINE_LOG_DEBUG << "MySQL connection pool: maximum pool size = " << std::to_string(max_pool_size); - //step 4: validate to avoid open old version schema + // step 4: validate to avoid open old version schema ValidateMetaSchema(); - //step 5: create meta tables + // step 5: create meta tables try { if (mode_ != DBOptions::MODE::CLUSTER_READONLY) { CleanUp(); @@ -303,8 +299,8 @@ MySQLMetaImpl::Initialize() { } mysqlpp::Query InitializeQuery = connectionPtr->query(); - InitializeQuery << "CREATE TABLE IF NOT EXISTS " << - TABLES_SCHEMA.name() << " (" << TABLES_SCHEMA.ToString() + ");"; + InitializeQuery << "CREATE TABLE IF NOT EXISTS " << TABLES_SCHEMA.name() << " (" + << TABLES_SCHEMA.ToString() + ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); @@ -312,16 +308,16 @@ MySQLMetaImpl::Initialize() { return HandleException("Initialization Error", InitializeQuery.error()); } - InitializeQuery << "CREATE TABLE IF NOT EXISTS " << - TABLEFILES_SCHEMA.name() << " (" << TABLEFILES_SCHEMA.ToString() + ");"; + InitializeQuery << "CREATE TABLE IF NOT EXISTS " << TABLEFILES_SCHEMA.name() << " (" + << TABLEFILES_SCHEMA.ToString() + ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); if (!InitializeQuery.exec()) { return HandleException("Initialization Error", InitializeQuery.error()); } - } //Scoped Connection - } catch (std::exception &e) { + } // Scoped Connection + } catch (std::exception& e) { return HandleException("GENERAL ERROR DURING INITIALIZATION", e.what()); } @@ -330,8 +326,7 @@ MySQLMetaImpl::Initialize() { // PXU TODO: Temp solution. Will fix later Status -MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, - const DatesT &dates) { +MySQLMetaImpl::DropPartitionsByDates(const std::string& table_id, const DatesT& dates) { if (dates.empty()) { return Status::OK(); } @@ -345,11 +340,11 @@ MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, try { std::stringstream dateListSS; - for (auto &date : dates) { + for (auto& date : dates) { dateListSS << std::to_string(date) << ", "; } std::string dateListStr = dateListSS.str(); - dateListStr = dateListStr.substr(0, dateListStr.size() - 2); //remove the last ", " + dateListStr = dateListStr.substr(0, dateListStr.size() - 2); // remove the last ", " { mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); @@ -360,12 +355,11 @@ MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, mysqlpp::Query dropPartitionsByDatesQuery = connectionPtr->query(); - dropPartitionsByDatesQuery << "UPDATE " << - META_TABLEFILES << " " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," << - "updated_time = " << utils::GetMicroSecTimeStamp() << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "date in (" << dateListStr << ");"; + dropPartitionsByDatesQuery << "UPDATE " << META_TABLEFILES << " " + << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," + << "updated_time = " << utils::GetMicroSecTimeStamp() << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "date in (" << dateListStr << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str(); @@ -373,17 +367,17 @@ MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, return HandleException("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", dropPartitionsByDatesQuery.error()); } - } //Scoped Connection + } // Scoped Connection ENGINE_LOG_DEBUG << "Successfully drop partitions, table id = " << table_schema.table_id_; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES", e.what()); } return Status::OK(); } Status -MySQLMetaImpl::CreateTable(TableSchema &table_schema) { +MySQLMetaImpl::CreateTable(TableSchema& table_schema) { try { server::MetricCollector metric; { @@ -398,9 +392,8 @@ MySQLMetaImpl::CreateTable(TableSchema &table_schema) { if (table_schema.table_id_.empty()) { NextTableId(table_schema.table_id_); } else { - createTableQuery << "SELECT state FROM " << - META_TABLES << " " << - "WHERE table_id = " << mysqlpp::quote << table_schema.table_id_ << ";"; + createTableQuery << "SELECT state FROM " << META_TABLES << " " + << "WHERE table_id = " << mysqlpp::quote << table_schema.table_id_ << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); @@ -419,7 +412,7 @@ MySQLMetaImpl::CreateTable(TableSchema &table_schema) { table_schema.id_ = -1; table_schema.created_on_ = utils::GetMicroSecTimeStamp(); - std::string id = "NULL"; //auto-increment + std::string id = "NULL"; // auto-increment std::string table_id = table_schema.table_id_; std::string state = std::to_string(table_schema.state_); std::string dimension = std::to_string(table_schema.dimension_); @@ -430,35 +423,32 @@ MySQLMetaImpl::CreateTable(TableSchema &table_schema) { std::string nlist = std::to_string(table_schema.nlist_); std::string metric_type = std::to_string(table_schema.metric_type_); - createTableQuery << "INSERT INTO " << - META_TABLES << " " << - "VALUES(" << id << ", " << mysqlpp::quote << table_id << ", " << - state << ", " << dimension << ", " << created_on << ", " << - flag << ", " << index_file_size << ", " << engine_type << ", " << - nlist << ", " << metric_type << ");"; + createTableQuery << "INSERT INTO " << META_TABLES << " " + << "VALUES(" << id << ", " << mysqlpp::quote << table_id << ", " << state << ", " + << dimension << ", " << created_on << ", " << flag << ", " << index_file_size << ", " + << engine_type << ", " << nlist << ", " << metric_type << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); if (mysqlpp::SimpleResult res = createTableQuery.execute()) { - table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? + table_schema.id_ = res.insert_id(); // Might need to use SELECT LAST_INSERT_ID()? - //Consume all results to avoid "Commands out of sync" error + // Consume all results to avoid "Commands out of sync" error } else { return HandleException("Add Table Error", createTableQuery.error()); } - } //Scoped Connection + } // Scoped Connection ENGINE_LOG_DEBUG << "Successfully create table: " << table_schema.table_id_; return utils::CreateTablePath(options_, table_schema.table_id_); - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN CREATING TABLE", e.what()); } } Status -MySQLMetaImpl::FilesByType(const std::string &table_id, - const std::vector &file_types, - std::vector &file_ids) { +MySQLMetaImpl::FilesByType(const std::string& table_id, const std::vector& file_types, + std::vector& file_ids) { if (file_types.empty()) { return Status(DB_ERROR, "file types array is empty"); } @@ -483,42 +473,49 @@ MySQLMetaImpl::FilesByType(const std::string &table_id, } mysqlpp::Query hasNonIndexFilesQuery = connectionPtr->query(); - //since table_id is a unique column we just need to check whether it exists or not - hasNonIndexFilesQuery << "SELECT file_id, file_type FROM " << - META_TABLEFILES << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "file_type in (" << types << ");"; + // since table_id is a unique column we just need to check whether it exists or not + hasNonIndexFilesQuery << "SELECT file_id, file_type FROM " << META_TABLEFILES << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "file_type in (" << types << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesByType: " << hasNonIndexFilesQuery.str(); res = hasNonIndexFilesQuery.store(); - } //Scoped Connection + } // Scoped Connection if (res.num_rows() > 0) { int raw_count = 0, new_count = 0, new_merge_count = 0, new_index_count = 0; int to_index_count = 0, index_count = 0, backup_count = 0; - for (auto &resRow : res) { + for (auto& resRow : res) { std::string file_id; resRow["file_id"].to_string(file_id); file_ids.push_back(file_id); int32_t file_type = resRow["file_type"]; switch (file_type) { - case (int) TableFileSchema::RAW:raw_count++; + case (int)TableFileSchema::RAW: + raw_count++; break; - case (int) TableFileSchema::NEW:new_count++; + case (int)TableFileSchema::NEW: + new_count++; break; - case (int) TableFileSchema::NEW_MERGE:new_merge_count++; + case (int)TableFileSchema::NEW_MERGE: + new_merge_count++; break; - case (int) TableFileSchema::NEW_INDEX:new_index_count++; + case (int)TableFileSchema::NEW_INDEX: + new_index_count++; break; - case (int) TableFileSchema::TO_INDEX:to_index_count++; + case (int)TableFileSchema::TO_INDEX: + to_index_count++; break; - case (int) TableFileSchema::INDEX:index_count++; + case (int)TableFileSchema::INDEX: + index_count++; break; - case (int) TableFileSchema::BACKUP:backup_count++; + case (int)TableFileSchema::BACKUP: + backup_count++; + break; + default: break; - default:break; } } @@ -527,7 +524,7 @@ MySQLMetaImpl::FilesByType(const std::string &table_id, << " new_index files:" << new_index_count << " to_index files:" << to_index_count << " index files:" << index_count << " backup files:" << backup_count; } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN GET FILE BY TYPE", e.what()); } @@ -535,7 +532,7 @@ MySQLMetaImpl::FilesByType(const std::string &table_id, } Status -MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex &index) { +MySQLMetaImpl::UpdateTableIndex(const std::string& table_id, const TableIndex& index) { try { server::MetricCollector metric; @@ -547,33 +544,31 @@ MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex &i } mysqlpp::Query updateTableIndexParamQuery = connectionPtr->query(); - updateTableIndexParamQuery << "SELECT id, state, dimension, created_on FROM " << - META_TABLES << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + updateTableIndexParamQuery << "SELECT id, state, dimension, created_on FROM " << META_TABLES << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableIndex: " << updateTableIndexParamQuery.str(); mysqlpp::StoreQueryResult res = updateTableIndexParamQuery.store(); if (res.num_rows() == 1) { - const mysqlpp::Row &resRow = res[0]; + const mysqlpp::Row& resRow = res[0]; size_t id = resRow["id"]; int32_t state = resRow["state"]; uint16_t dimension = resRow["dimension"]; int64_t created_on = resRow["created_on"]; - updateTableIndexParamQuery << "UPDATE " << - META_TABLES << " " << - "SET id = " << id << ", " << - "state = " << state << ", " << - "dimension = " << dimension << ", " << - "created_on = " << created_on << ", " << - "engine_type = " << index.engine_type_ << ", " << - "nlist = " << index.nlist_ << ", " << - "metric_type = " << index.metric_type_ << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << ";"; + updateTableIndexParamQuery << "UPDATE " << META_TABLES << " " + << "SET id = " << id << ", " + << "state = " << state << ", " + << "dimension = " << dimension << ", " + << "created_on = " << created_on << ", " + << "engine_type = " << index.engine_type_ << ", " + << "nlist = " << index.nlist_ << ", " + << "metric_type = " << index.metric_type_ << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableIndex: " << updateTableIndexParamQuery.str(); @@ -584,10 +579,10 @@ MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex &i } else { return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); } - } //Scoped Connection + } // Scoped Connection ENGINE_LOG_DEBUG << "Successfully update table index, table id = " << table_id; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE INDEX PARAM", e.what()); } @@ -595,7 +590,7 @@ MySQLMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex &i } Status -MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { +MySQLMetaImpl::UpdateTableFlag(const std::string& table_id, int64_t flag) { try { server::MetricCollector metric; @@ -607,20 +602,19 @@ MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { } mysqlpp::Query updateTableFlagQuery = connectionPtr->query(); - updateTableFlagQuery << "UPDATE " << - META_TABLES << " " << - "SET flag = " << flag << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << ";"; + updateTableFlagQuery << "UPDATE " << META_TABLES << " " + << "SET flag = " << flag << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFlag: " << updateTableFlagQuery.str(); if (!updateTableFlagQuery.exec()) { return HandleException("QUERY ERROR WHEN UPDATING TABLE FLAG", updateTableFlagQuery.error()); } - } //Scoped Connection + } // Scoped Connection ENGINE_LOG_DEBUG << "Successfully update table flag, table id = " << table_id; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FLAG", e.what()); } @@ -628,7 +622,7 @@ MySQLMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { } Status -MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex &index) { +MySQLMetaImpl::DescribeTableIndex(const std::string& table_id, TableIndex& index) { try { server::MetricCollector metric; @@ -640,17 +634,17 @@ MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex &index } mysqlpp::Query describeTableIndexQuery = connectionPtr->query(); - describeTableIndexQuery << "SELECT engine_type, nlist, index_file_size, metric_type FROM " << - META_TABLES << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + describeTableIndexQuery << "SELECT engine_type, nlist, index_file_size, metric_type FROM " << META_TABLES + << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTableIndex: " << describeTableIndexQuery.str(); mysqlpp::StoreQueryResult res = describeTableIndexQuery.store(); if (res.num_rows() == 1) { - const mysqlpp::Row &resRow = res[0]; + const mysqlpp::Row& resRow = res[0]; index.engine_type_ = resRow["engine_type"]; index.nlist_ = resRow["nlist"]; @@ -658,8 +652,8 @@ MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex &index } else { return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); } - } //Scoped Connection - } catch (std::exception &e) { + } // Scoped Connection + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FLAG", e.what()); } @@ -667,7 +661,7 @@ MySQLMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex &index } Status -MySQLMetaImpl::DropTableIndex(const std::string &table_id) { +MySQLMetaImpl::DropTableIndex(const std::string& table_id) { try { server::MetricCollector metric; @@ -680,13 +674,12 @@ MySQLMetaImpl::DropTableIndex(const std::string &table_id) { mysqlpp::Query dropTableIndexQuery = connectionPtr->query(); - //soft delete index files - dropTableIndexQuery << "UPDATE " << - META_TABLEFILES << " " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," << - "updated_time = " << utils::GetMicroSecTimeStamp() << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "file_type = " << std::to_string(TableFileSchema::INDEX) << ";"; + // soft delete index files + dropTableIndexQuery << "UPDATE " << META_TABLEFILES << " " + << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," + << "updated_time = " << utils::GetMicroSecTimeStamp() << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "file_type = " << std::to_string(TableFileSchema::INDEX) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); @@ -694,13 +687,12 @@ MySQLMetaImpl::DropTableIndex(const std::string &table_id) { return HandleException("QUERY ERROR WHEN DROPPING TABLE INDEX", dropTableIndexQuery.error()); } - //set all backup file to raw - dropTableIndexQuery << "UPDATE " << - META_TABLEFILES << " " << - "SET file_type = " << std::to_string(TableFileSchema::RAW) << "," << - "updated_time = " << utils::GetMicroSecTimeStamp() << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "file_type = " << std::to_string(TableFileSchema::BACKUP) << ";"; + // set all backup file to raw + dropTableIndexQuery << "UPDATE " << META_TABLEFILES << " " + << "SET file_type = " << std::to_string(TableFileSchema::RAW) << "," + << "updated_time = " << utils::GetMicroSecTimeStamp() << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "file_type = " << std::to_string(TableFileSchema::BACKUP) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); @@ -708,23 +700,22 @@ MySQLMetaImpl::DropTableIndex(const std::string &table_id) { return HandleException("QUERY ERROR WHEN DROPPING TABLE INDEX", dropTableIndexQuery.error()); } - //set table index type to raw - dropTableIndexQuery << "UPDATE " << - META_TABLES << " " << - "SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) << "," << - "nlist = " << std::to_string(DEFAULT_NLIST) << ", " << - "metric_type = " << std::to_string(DEFAULT_METRIC_TYPE) << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << ";"; + // set table index type to raw + dropTableIndexQuery << "UPDATE " << META_TABLES << " " + << "SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) << "," + << "nlist = " << std::to_string(DEFAULT_NLIST) << ", " + << "metric_type = " << std::to_string(DEFAULT_METRIC_TYPE) << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); if (!dropTableIndexQuery.exec()) { return HandleException("QUERY ERROR WHEN DROPPING TABLE INDEX", dropTableIndexQuery.error()); } - } //Scoped Connection + } // Scoped Connection ENGINE_LOG_DEBUG << "Successfully drop table index, table id = " << table_id; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN DROPPING TABLE INDEX", e.what()); } @@ -732,7 +723,7 @@ MySQLMetaImpl::DropTableIndex(const std::string &table_id) { } Status -MySQLMetaImpl::DeleteTable(const std::string &table_id) { +MySQLMetaImpl::DeleteTable(const std::string& table_id) { try { server::MetricCollector metric; { @@ -742,27 +733,26 @@ MySQLMetaImpl::DeleteTable(const std::string &table_id) { return Status(DB_ERROR, "Failed to connect to database server"); } - //soft delete table + // soft delete table mysqlpp::Query deleteTableQuery = connectionPtr->query(); -// - deleteTableQuery << "UPDATE " << - META_TABLES << " " << - "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << ";"; + // + deleteTableQuery << "UPDATE " << META_TABLES << " " + << "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str(); if (!deleteTableQuery.exec()) { return HandleException("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error()); } - } //Scoped Connection + } // Scoped Connection if (mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { DeleteTableFiles(table_id); } ENGINE_LOG_DEBUG << "Successfully delete table, table id = " << table_id; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN DELETING TABLE", e.what()); } @@ -770,7 +760,7 @@ MySQLMetaImpl::DeleteTable(const std::string &table_id) { } Status -MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { +MySQLMetaImpl::DeleteTableFiles(const std::string& table_id) { try { server::MetricCollector metric; { @@ -780,25 +770,24 @@ MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { return Status(DB_ERROR, "Failed to connect to database server"); } - //soft delete table files + // soft delete table files mysqlpp::Query deleteTableFilesQuery = connectionPtr->query(); // - deleteTableFilesQuery << "UPDATE " << - META_TABLEFILES << " " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << - "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + deleteTableFilesQuery << "UPDATE " << META_TABLEFILES << " " + << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " + << "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str(); if (!deleteTableFilesQuery.exec()) { return HandleException("QUERY ERROR WHEN DELETING TABLE FILES", deleteTableFilesQuery.error()); } - } //Scoped Connection + } // Scoped Connection ENGINE_LOG_DEBUG << "Successfully delete table files, table id = " << table_id; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN DELETING TABLE FILES", e.what()); } @@ -806,7 +795,7 @@ MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { } Status -MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { +MySQLMetaImpl::DescribeTable(TableSchema& table_schema) { try { server::MetricCollector metric; mysqlpp::StoreQueryResult res; @@ -820,19 +809,19 @@ MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { mysqlpp::Query describeTableQuery = connectionPtr->query(); describeTableQuery << "SELECT id, state, dimension, created_on, flag, index_file_size, engine_type, nlist, metric_type " - << " FROM " << META_TABLES << " " << - "WHERE table_id = " << mysqlpp::quote << table_schema.table_id_ << " " << - "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + << " FROM " << META_TABLES << " " + << "WHERE table_id = " << mysqlpp::quote << table_schema.table_id_ << " " + << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str(); res = describeTableQuery.store(); - } //Scoped Connection + } // Scoped Connection if (res.num_rows() == 1) { - const mysqlpp::Row &resRow = res[0]; + const mysqlpp::Row& resRow = res[0]; - table_schema.id_ = resRow["id"]; //implicit conversion + table_schema.id_ = resRow["id"]; // implicit conversion table_schema.state_ = resRow["state"]; @@ -852,7 +841,7 @@ MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { } else { return Status(DB_NOT_FOUND, "Table " + table_schema.table_id_ + " not found"); } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN DESCRIBING TABLE", e.what()); } @@ -860,7 +849,7 @@ MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { } Status -MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { +MySQLMetaImpl::HasTable(const std::string& table_id, bool& has_or_not) { try { server::MetricCollector metric; mysqlpp::StoreQueryResult res; @@ -872,22 +861,22 @@ MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { } mysqlpp::Query hasTableQuery = connectionPtr->query(); - //since table_id is a unique column we just need to check whether it exists or not - hasTableQuery << "SELECT EXISTS " << - "(SELECT 1 FROM " << - META_TABLES << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " " << - "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << - "AS " << mysqlpp::quote << "check" << ";"; + // since table_id is a unique column we just need to check whether it exists or not + hasTableQuery << "SELECT EXISTS " + << "(SELECT 1 FROM " << META_TABLES << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " " + << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " + << "AS " << mysqlpp::quote << "check" + << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str(); res = hasTableQuery.store(); - } //Scoped Connection + } // Scoped Connection int check = res[0]["check"]; has_or_not = (check == 1); - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN CHECKING IF TABLE EXISTS", e.what()); } @@ -895,7 +884,7 @@ MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { } Status -MySQLMetaImpl::AllTables(std::vector &table_schema_array) { +MySQLMetaImpl::AllTables(std::vector& table_schema_array) { try { server::MetricCollector metric; mysqlpp::StoreQueryResult res; @@ -908,19 +897,18 @@ MySQLMetaImpl::AllTables(std::vector &table_schema_array) { mysqlpp::Query allTablesQuery = connectionPtr->query(); allTablesQuery << "SELECT id, table_id, dimension, engine_type, nlist, index_file_size, metric_type FROM " - << - META_TABLES << " " << - "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + << META_TABLES << " " + << "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str(); res = allTablesQuery.store(); - } //Scoped Connection + } // Scoped Connection - for (auto &resRow : res) { + for (auto& resRow : res) { TableSchema table_schema; - table_schema.id_ = resRow["id"]; //implicit conversion + table_schema.id_ = resRow["id"]; // implicit conversion std::string table_id; resRow["table_id"].to_string(table_id); @@ -938,7 +926,7 @@ MySQLMetaImpl::AllTables(std::vector &table_schema_array) { table_schema_array.emplace_back(table_schema); } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN DESCRIBING ALL TABLES", e.what()); } @@ -946,7 +934,7 @@ MySQLMetaImpl::AllTables(std::vector &table_schema_array) { } Status -MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { +MySQLMetaImpl::CreateTableFile(TableFileSchema& file_schema) { if (file_schema.date_ == EmptyDate) { file_schema.date_ = utils::GetDate(); } @@ -971,7 +959,7 @@ MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { file_schema.nlist_ = table_schema.nlist_; file_schema.metric_type_ = table_schema.metric_type_; - std::string id = "NULL"; //auto-increment + std::string id = "NULL"; // auto-increment std::string table_id = file_schema.table_id_; std::string engine_type = std::to_string(file_schema.engine_type_); std::string file_id = file_schema.file_id_; @@ -991,33 +979,31 @@ MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { mysqlpp::Query createTableFileQuery = connectionPtr->query(); - createTableFileQuery << "INSERT INTO " << - META_TABLEFILES << " " << - "VALUES(" << id << ", " << mysqlpp::quote << table_id << - ", " << engine_type << ", " << - mysqlpp::quote << file_id << ", " << file_type << ", " << file_size << ", " << - row_count << ", " << updated_time << ", " << created_on << ", " << date << ");"; + createTableFileQuery << "INSERT INTO " << META_TABLEFILES << " " + << "VALUES(" << id << ", " << mysqlpp::quote << table_id << ", " << engine_type << ", " + << mysqlpp::quote << file_id << ", " << file_type << ", " << file_size << ", " + << row_count << ", " << updated_time << ", " << created_on << ", " << date << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); if (mysqlpp::SimpleResult res = createTableFileQuery.execute()) { - file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? + file_schema.id_ = res.insert_id(); // Might need to use SELECT LAST_INSERT_ID()? - //Consume all results to avoid "Commands out of sync" error + // Consume all results to avoid "Commands out of sync" error } else { return HandleException("QUERY ERROR WHEN CREATING TABLE FILE", createTableFileQuery.error()); } - } // Scoped Connection + } // Scoped Connection ENGINE_LOG_DEBUG << "Successfully create table file, file id = " << file_schema.file_id_; return utils::CreateTableFilePath(options_, file_schema); - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN CREATING TABLE FILE", e.what()); } } Status -MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { +MySQLMetaImpl::FilesToIndex(TableFilesSchema& files) { files.clear(); try { @@ -1033,20 +1019,19 @@ MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { mysqlpp::Query filesToIndexQuery = connectionPtr->query(); filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " - << - META_TABLEFILES << " " << - "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; + << META_TABLEFILES << " " + << "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); res = filesToIndexQuery.store(); - } //Scoped Connection + } // Scoped Connection Status ret; std::map groups; TableFileSchema table_file; - for (auto &resRow : res) { - table_file.id_ = resRow["id"]; //implicit conversion + for (auto& resRow : res) { + table_file.id_ = resRow["id"]; // implicit conversion std::string table_id; resRow["table_id"].to_string(table_id); @@ -1095,16 +1080,14 @@ MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-index files"; } return ret; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", e.what()); } } Status -MySQLMetaImpl::FilesToSearch(const std::string &table_id, - const std::vector &ids, - const DatesT &partition, - DatePartionedTableFilesSchema &files) { +MySQLMetaImpl::FilesToSearch(const std::string& table_id, const std::vector& ids, const DatesT& partition, + DatePartionedTableFilesSchema& files) { files.clear(); try { @@ -1119,41 +1102,43 @@ MySQLMetaImpl::FilesToSearch(const std::string &table_id, mysqlpp::Query filesToSearchQuery = connectionPtr->query(); filesToSearchQuery - << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date FROM " << - META_TABLEFILES << " " << - "WHERE table_id = " << mysqlpp::quote << table_id; + << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date FROM " + << META_TABLEFILES << " " + << "WHERE table_id = " << mysqlpp::quote << table_id; if (!partition.empty()) { std::stringstream partitionListSS; - for (auto &date : partition) { + for (auto& date : partition) { partitionListSS << std::to_string(date) << ", "; } std::string partitionListStr = partitionListSS.str(); - partitionListStr = partitionListStr.substr(0, partitionListStr.size() - 2); //remove the last ", " - filesToSearchQuery << " AND " << "date IN (" << partitionListStr << ")"; + partitionListStr = partitionListStr.substr(0, partitionListStr.size() - 2); // remove the last ", " + filesToSearchQuery << " AND " + << "date IN (" << partitionListStr << ")"; } if (!ids.empty()) { std::stringstream idSS; - for (auto &id : ids) { + for (auto& id : ids) { idSS << "id = " << std::to_string(id) << " OR "; } std::string idStr = idSS.str(); - idStr = idStr.substr(0, idStr.size() - 4); //remove the last " OR " + idStr = idStr.substr(0, idStr.size() - 4); // remove the last " OR " - filesToSearchQuery << " AND " << "(" << idStr << ")"; + filesToSearchQuery << " AND " + << "(" << idStr << ")"; } // End - filesToSearchQuery << " AND " << - "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << - "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << - "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + filesToSearchQuery << " AND " + << "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " + << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " + << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); res = filesToSearchQuery.store(); - } //Scoped Connection + } // Scoped Connection TableSchema table_schema; table_schema.table_id_ = table_id; @@ -1164,8 +1149,8 @@ MySQLMetaImpl::FilesToSearch(const std::string &table_id, Status ret; TableFileSchema table_file; - for (auto &resRow : res) { - table_file.id_ = resRow["id"]; //implicit conversion + for (auto& resRow : res) { + table_file.id_ = resRow["id"]; // implicit conversion std::string table_id_str; resRow["table_id"].to_string(table_id_str); @@ -1210,20 +1195,19 @@ MySQLMetaImpl::FilesToSearch(const std::string &table_id, ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-search files"; } return ret; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH", e.what()); } } Status -MySQLMetaImpl::FilesToMerge(const std::string &table_id, - DatePartionedTableFilesSchema &files) { +MySQLMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) { files.clear(); try { server::MetricCollector metric; - //check table existence + // check table existence TableSchema table_schema; table_schema.table_id_ = table_id; auto status = DescribeTable(table_schema); @@ -1242,26 +1226,26 @@ MySQLMetaImpl::FilesToMerge(const std::string &table_id, mysqlpp::Query filesToMergeQuery = connectionPtr->query(); filesToMergeQuery << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, created_on FROM " - << - META_TABLEFILES << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "file_type = " << std::to_string(TableFileSchema::RAW) << " " << - "ORDER BY row_count DESC" << ";"; + << META_TABLEFILES << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "file_type = " << std::to_string(TableFileSchema::RAW) << " " + << "ORDER BY row_count DESC" + << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str(); res = filesToMergeQuery.store(); - } //Scoped Connection + } // Scoped Connection Status ret; - for (auto &resRow : res) { + for (auto& resRow : res) { TableFileSchema table_file; table_file.file_size_ = resRow["file_size"]; if (table_file.file_size_ >= table_schema.index_file_size_) { - continue;//skip large file + continue; // skip large file } - table_file.id_ = resRow["id"]; //implicit conversion + table_file.id_ = resRow["id"]; // implicit conversion std::string table_id_str; resRow["table_id"].to_string(table_id_str); @@ -1306,25 +1290,24 @@ MySQLMetaImpl::FilesToMerge(const std::string &table_id, ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-merge files"; } return ret; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", e.what()); } } Status -MySQLMetaImpl::GetTableFiles(const std::string &table_id, - const std::vector &ids, - TableFilesSchema &table_files) { +MySQLMetaImpl::GetTableFiles(const std::string& table_id, const std::vector& ids, + TableFilesSchema& table_files) { if (ids.empty()) { return Status::OK(); } std::stringstream idSS; - for (auto &id : ids) { + for (auto& id : ids) { idSS << "id = " << std::to_string(id) << " OR "; } std::string idStr = idSS.str(); - idStr = idStr.substr(0, idStr.size() - 4); //remove the last " OR " + idStr = idStr.substr(0, idStr.size() - 4); // remove the last " OR " try { mysqlpp::StoreQueryResult res; @@ -1337,23 +1320,23 @@ MySQLMetaImpl::GetTableFiles(const std::string &table_id, mysqlpp::Query getTableFileQuery = connectionPtr->query(); getTableFileQuery - << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " << - META_TABLEFILES << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "(" << idStr << ") AND " << - "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " + << META_TABLEFILES << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "(" << idStr << ") AND " + << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); res = getTableFileQuery.store(); - } //Scoped Connection + } // Scoped Connection TableSchema table_schema; table_schema.table_id_ = table_id; DescribeTable(table_schema); Status ret; - for (auto &resRow : res) { + for (auto& resRow : res) { TableFileSchema file_schema; file_schema.id_ = resRow["id"]; @@ -1391,7 +1374,7 @@ MySQLMetaImpl::GetTableFiles(const std::string &table_id, ENGINE_LOG_DEBUG << "Get table files by id"; return ret; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN RETRIEVING TABLE FILES", e.what()); } } @@ -1399,14 +1382,14 @@ MySQLMetaImpl::GetTableFiles(const std::string &table_id, // PXU TODO: Support Swap Status MySQLMetaImpl::Archive() { - auto &criterias = options_.archive_conf_.GetCriterias(); + auto& criterias = options_.archive_conf_.GetCriterias(); if (criterias.empty()) { return Status::OK(); } - for (auto &kv : criterias) { - auto &criteria = kv.first; - auto &limit = kv.second; + for (auto& kv : criterias) { + auto& criteria = kv.first; + auto& limit = kv.second; if (criteria == engine::ARCHIVE_CONF_DAYS) { size_t usecs = limit * D_SEC * US_PS; int64_t now = utils::GetMicroSecTimeStamp(); @@ -1419,11 +1402,10 @@ MySQLMetaImpl::Archive() { } mysqlpp::Query archiveQuery = connectionPtr->query(); - archiveQuery << "UPDATE " << - META_TABLEFILES << " " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << - "WHERE created_on < " << std::to_string(now - usecs) << " AND " << - "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + archiveQuery << "UPDATE " << META_TABLEFILES << " " + << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " + << "WHERE created_on < " << std::to_string(now - usecs) << " AND " + << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::Archive: " << archiveQuery.str(); @@ -1432,7 +1414,7 @@ MySQLMetaImpl::Archive() { } ENGINE_LOG_DEBUG << "Archive old files"; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN DURING ARCHIVE", e.what()); } } @@ -1451,7 +1433,7 @@ MySQLMetaImpl::Archive() { } Status -MySQLMetaImpl::Size(uint64_t &result) { +MySQLMetaImpl::Size(uint64_t& result) { result = 0; try { @@ -1464,21 +1446,20 @@ MySQLMetaImpl::Size(uint64_t &result) { } mysqlpp::Query getSizeQuery = connectionPtr->query(); - getSizeQuery << "SELECT IFNULL(SUM(file_size),0) AS sum FROM " << - META_TABLEFILES << " " << - "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + getSizeQuery << "SELECT IFNULL(SUM(file_size),0) AS sum FROM " << META_TABLEFILES << " " + << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str(); res = getSizeQuery.store(); - } //Scoped Connection + } // Scoped Connection if (res.empty()) { result = 0; } else { result = res[0]["sum"]; } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN RETRIEVING SIZE", e.what()); } @@ -1503,11 +1484,10 @@ MySQLMetaImpl::DiscardFiles(int64_t to_discard_size) { } mysqlpp::Query discardFilesQuery = connectionPtr->query(); - discardFilesQuery << "SELECT id, file_size FROM " << - META_TABLEFILES << " " << - "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " << - "ORDER BY id ASC " << - "LIMIT 10;"; + discardFilesQuery << "SELECT id, file_size FROM " << META_TABLEFILES << " " + << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " + << "ORDER BY id ASC " + << "LIMIT 10;"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); @@ -1518,7 +1498,7 @@ MySQLMetaImpl::DiscardFiles(int64_t to_discard_size) { TableFileSchema table_file; std::stringstream idsToDiscardSS; - for (auto &resRow : res) { + for (auto& resRow : res) { if (to_discard_size <= 0) { break; } @@ -1531,13 +1511,12 @@ MySQLMetaImpl::DiscardFiles(int64_t to_discard_size) { } std::string idsToDiscardStr = idsToDiscardSS.str(); - idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); //remove the last " OR " + idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); // remove the last " OR " - discardFilesQuery << "UPDATE " << - META_TABLEFILES << " " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << - "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << - "WHERE " << idsToDiscardStr << ";"; + discardFilesQuery << "UPDATE " << META_TABLEFILES << " " + << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " + << "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " + << "WHERE " << idsToDiscardStr << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); @@ -1545,17 +1524,17 @@ MySQLMetaImpl::DiscardFiles(int64_t to_discard_size) { if (!status) { return HandleException("QUERY ERROR WHEN DISCARDING FILES", discardFilesQuery.error()); } - } //Scoped Connection + } // Scoped Connection return DiscardFiles(to_discard_size); - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN DISCARDING FILES", e.what()); } } -//ZR: this function assumes all fields in file_schema have value +// ZR: this function assumes all fields in file_schema have value Status -MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { +MySQLMetaImpl::UpdateTableFile(TableFileSchema& file_schema) { file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); try { @@ -1569,11 +1548,10 @@ MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { mysqlpp::Query updateTableFileQuery = connectionPtr->query(); - //if the table has been deleted, just mark the table file as TO_DELETE - //clean thread will delete the file later - updateTableFileQuery << "SELECT state FROM " << - META_TABLES << " " << - "WHERE table_id = " << mysqlpp::quote << file_schema.table_id_ << ";"; + // if the table has been deleted, just mark the table file as TO_DELETE + // clean thread will delete the file later + updateTableFileQuery << "SELECT state FROM " << META_TABLES << " " + << "WHERE table_id = " << mysqlpp::quote << file_schema.table_id_ << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); @@ -1599,18 +1577,17 @@ MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { std::string created_on = std::to_string(file_schema.created_on_); std::string date = std::to_string(file_schema.date_); - updateTableFileQuery << "UPDATE " << - META_TABLEFILES << " " << - "SET table_id = " << mysqlpp::quote << table_id << ", " << - "engine_type = " << engine_type << ", " << - "file_id = " << mysqlpp::quote << file_id << ", " << - "file_type = " << file_type << ", " << - "file_size = " << file_size << ", " << - "row_count = " << row_count << ", " << - "updated_time = " << updated_time << ", " << - "created_on = " << created_on << ", " << - "date = " << date << " " << - "WHERE id = " << id << ";"; + updateTableFileQuery << "UPDATE " << META_TABLEFILES << " " + << "SET table_id = " << mysqlpp::quote << table_id << ", " + << "engine_type = " << engine_type << ", " + << "file_id = " << mysqlpp::quote << file_id << ", " + << "file_type = " << file_type << ", " + << "file_size = " << file_size << ", " + << "row_count = " << row_count << ", " + << "updated_time = " << updated_time << ", " + << "created_on = " << created_on << ", " + << "date = " << date << " " + << "WHERE id = " << id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); @@ -1618,10 +1595,10 @@ MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; return HandleException("QUERY ERROR WHEN UPDATING TABLE FILE", updateTableFileQuery.error()); } - } //Scoped Connection + } // Scoped Connection ENGINE_LOG_DEBUG << "Update single table file, file id = " << file_schema.file_id_; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILE", e.what()); } @@ -1629,7 +1606,7 @@ MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { } Status -MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { +MySQLMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { try { mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); @@ -1639,11 +1616,10 @@ MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { mysqlpp::Query updateTableFilesToIndexQuery = connectionPtr->query(); - updateTableFilesToIndexQuery << "UPDATE " << - META_TABLEFILES << " " << - "SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "file_type = " << std::to_string(TableFileSchema::RAW) << ";"; + updateTableFilesToIndexQuery << "UPDATE " << META_TABLEFILES << " " + << "SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "file_type = " << std::to_string(TableFileSchema::RAW) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFilesToIndex: " << updateTableFilesToIndexQuery.str(); @@ -1653,7 +1629,7 @@ MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { } ENGINE_LOG_DEBUG << "Update files to to_index, table id = " << table_id; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES TO INDEX", e.what()); } @@ -1661,7 +1637,7 @@ MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { } Status -MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { +MySQLMetaImpl::UpdateTableFiles(TableFilesSchema& files) { try { server::MetricCollector metric; { @@ -1674,17 +1650,17 @@ MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { mysqlpp::Query updateTableFilesQuery = connectionPtr->query(); std::map has_tables; - for (auto &file_schema : files) { + for (auto& file_schema : files) { if (has_tables.find(file_schema.table_id_) != has_tables.end()) { continue; } - updateTableFilesQuery << "SELECT EXISTS " << - "(SELECT 1 FROM " << - META_TABLES << " " << - "WHERE table_id = " << mysqlpp::quote << file_schema.table_id_ << " " << - "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << - "AS " << mysqlpp::quote << "check" << ";"; + updateTableFilesQuery << "SELECT EXISTS " + << "(SELECT 1 FROM " << META_TABLES << " " + << "WHERE table_id = " << mysqlpp::quote << file_schema.table_id_ << " " + << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " + << "AS " << mysqlpp::quote << "check" + << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); @@ -1694,7 +1670,7 @@ MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { has_tables[file_schema.table_id_] = (check == 1); } - for (auto &file_schema : files) { + for (auto& file_schema : files) { if (!has_tables[file_schema.table_id_]) { file_schema.file_type_ = TableFileSchema::TO_DELETE; } @@ -1711,18 +1687,17 @@ MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { std::string created_on = std::to_string(file_schema.created_on_); std::string date = std::to_string(file_schema.date_); - updateTableFilesQuery << "UPDATE " << - META_TABLEFILES << " " << - "SET table_id = " << mysqlpp::quote << table_id << ", " << - "engine_type = " << engine_type << ", " << - "file_id = " << mysqlpp::quote << file_id << ", " << - "file_type = " << file_type << ", " << - "file_size = " << file_size << ", " << - "row_count = " << row_count << ", " << - "updated_time = " << updated_time << ", " << - "created_on = " << created_on << ", " << - "date = " << date << " " << - "WHERE id = " << id << ";"; + updateTableFilesQuery << "UPDATE " << META_TABLEFILES << " " + << "SET table_id = " << mysqlpp::quote << table_id << ", " + << "engine_type = " << engine_type << ", " + << "file_id = " << mysqlpp::quote << file_id << ", " + << "file_type = " << file_type << ", " + << "file_size = " << file_size << ", " + << "row_count = " << row_count << ", " + << "updated_time = " << updated_time << ", " + << "created_on = " << created_on << ", " + << "date = " << date << " " + << "WHERE id = " << id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); @@ -1730,10 +1705,10 @@ MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { return HandleException("QUERY ERROR WHEN UPDATING TABLE FILES", updateTableFilesQuery.error()); } } - } //Scoped Connection + } // Scoped Connection ENGINE_LOG_DEBUG << "Update " << files.size() << " table files"; - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES", e.what()); } @@ -1745,7 +1720,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { auto now = utils::GetMicroSecTimeStamp(); std::set table_ids; - //remove to_delete files + // remove to_delete files try { server::MetricCollector metric; @@ -1757,10 +1732,9 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date FROM " << - META_TABLEFILES << " " << - "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << - "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; + cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date FROM " << META_TABLEFILES << " " + << "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " + << "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1769,8 +1743,8 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { TableFileSchema table_file; std::vector idsToDelete; - for (auto &resRow : res) { - table_file.id_ = resRow["id"]; //implicit conversion + for (auto& resRow : res) { + table_file.id_ = resRow["id"]; // implicit conversion std::string table_id; resRow["table_id"].to_string(table_id); @@ -1793,15 +1767,14 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { if (!idsToDelete.empty()) { std::stringstream idsToDeleteSS; - for (auto &id : idsToDelete) { + for (auto& id : idsToDelete) { idsToDeleteSS << "id = " << id << " OR "; } std::string idsToDeleteStr = idsToDeleteSS.str(); - idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM " << - META_TABLEFILES << " " << - "WHERE " << idsToDeleteStr << ";"; + idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); // remove the last " OR " + cleanUpFilesWithTTLQuery << "DELETE FROM " << META_TABLEFILES << " " + << "WHERE " << idsToDeleteStr << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1814,12 +1787,12 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { if (res.size() > 0) { ENGINE_LOG_DEBUG << "Clean " << res.size() << " files deleted in " << seconds << " seconds"; } - } //Scoped Connection - } catch (std::exception &e) { + } // Scoped Connection + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", e.what()); } - //remove to_delete tables + // remove to_delete tables try { server::MetricCollector metric; @@ -1831,9 +1804,8 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { } mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id FROM " << - META_TABLES << " " << - "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; + cleanUpFilesWithTTLQuery << "SELECT id, table_id FROM " << META_TABLES << " " + << "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1841,20 +1813,19 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { if (!res.empty()) { std::stringstream idsToDeleteSS; - for (auto &resRow : res) { + for (auto& resRow : res) { size_t id = resRow["id"]; std::string table_id; resRow["table_id"].to_string(table_id); - utils::DeleteTablePath(options_, table_id, false);//only delete empty folder + utils::DeleteTablePath(options_, table_id, false); // only delete empty folder idsToDeleteSS << "id = " << std::to_string(id) << " OR "; } std::string idsToDeleteStr = idsToDeleteSS.str(); - idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM " << - META_TABLES << " " << - "WHERE " << idsToDeleteStr << ";"; + idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); // remove the last " OR " + cleanUpFilesWithTTLQuery << "DELETE FROM " << META_TABLES << " " + << "WHERE " << idsToDeleteStr << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1867,13 +1838,13 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { if (res.size() > 0) { ENGINE_LOG_DEBUG << "Remove " << res.size() << " tables from meta"; } - } //Scoped Connection - } catch (std::exception &e) { + } // Scoped Connection + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what()); } - //remove deleted table folder - //don't remove table folder until all its files has been deleted + // remove deleted table folder + // don't remove table folder until all its files has been deleted try { server::MetricCollector metric; @@ -1884,11 +1855,10 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { return Status(DB_ERROR, "Failed to connect to database server"); } - for (auto &table_id : table_ids) { + for (auto& table_id : table_ids) { mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT file_id FROM " << - META_TABLEFILES << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << ";"; + cleanUpFilesWithTTLQuery << "SELECT file_id FROM " << META_TABLEFILES << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); @@ -1903,7 +1873,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { ENGINE_LOG_DEBUG << "Remove " << table_ids.size() << " tables folder"; } } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what()); } @@ -1920,10 +1890,10 @@ MySQLMetaImpl::CleanUp() { } mysqlpp::Query cleanUpQuery = connectionPtr->query(); - cleanUpQuery << "SELECT table_name " << - "FROM information_schema.tables " << - "WHERE table_schema = " << mysqlpp::quote << mysql_connection_pool_->getDB() << " " << - "AND table_name = " << mysqlpp::quote << META_TABLEFILES << ";"; + cleanUpQuery << "SELECT table_name " + << "FROM information_schema.tables " + << "WHERE table_schema = " << mysqlpp::quote << mysql_connection_pool_->getDB() << " " + << "AND table_name = " << mysqlpp::quote << META_TABLEFILES << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); @@ -1932,9 +1902,8 @@ MySQLMetaImpl::CleanUp() { if (!res.empty()) { ENGINE_LOG_DEBUG << "Remove table file type as NEW"; cleanUpQuery << "DELETE FROM " << META_TABLEFILES << " WHERE file_type IN (" - << std::to_string(TableFileSchema::NEW) << "," - << std::to_string(TableFileSchema::NEW_MERGE) << "," - << std::to_string(TableFileSchema::NEW_INDEX) << ");"; + << std::to_string(TableFileSchema::NEW) << "," << std::to_string(TableFileSchema::NEW_MERGE) + << "," << std::to_string(TableFileSchema::NEW_INDEX) << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); @@ -1946,7 +1915,7 @@ MySQLMetaImpl::CleanUp() { if (res.size() > 0) { ENGINE_LOG_DEBUG << "Clean " << res.size() << " files"; } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN CLEANING UP FILES", e.what()); } @@ -1954,7 +1923,7 @@ MySQLMetaImpl::CleanUp() { } Status -MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { +MySQLMetaImpl::Count(const std::string& table_id, uint64_t& result) { try { server::MetricCollector metric; @@ -1975,24 +1944,23 @@ MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { } mysqlpp::Query countQuery = connectionPtr->query(); - countQuery << "SELECT row_count FROM " << - META_TABLEFILES << " " << - "WHERE table_id = " << mysqlpp::quote << table_id << " AND " << - "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << - "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << - "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + countQuery << "SELECT row_count FROM " << META_TABLEFILES << " " + << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " + << "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " + << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " + << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::Count: " << countQuery.str(); res = countQuery.store(); - } //Scoped Connection + } // Scoped Connection result = 0; - for (auto &resRow : res) { + for (auto& resRow : res) { size_t size = resRow["row_count"]; result += size; } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN RETRIEVING COUNT", e.what()); } @@ -2019,12 +1987,12 @@ MySQLMetaImpl::DropAll() { } else { return HandleException("QUERY ERROR WHEN DROPPING ALL", dropTableQuery.error()); } - } catch (std::exception &e) { + } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN DROPPING ALL", e.what()); } } -} // namespace meta -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/MySQLMetaImpl.h b/cpp/src/db/meta/MySQLMetaImpl.h index d9d67fd748..48bdf5b9ab 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.h +++ b/cpp/src/db/meta/MySQLMetaImpl.h @@ -18,14 +18,14 @@ #pragma once #include "Meta.h" -#include "db/Options.h" #include "MySQLConnectionPool.h" +#include "db/Options.h" #include -#include -#include -#include #include +#include +#include +#include namespace zilliz { namespace milvus { @@ -34,77 +34,101 @@ namespace meta { class MySQLMetaImpl : public Meta { public: - MySQLMetaImpl(const DBMetaOptions &options, const int &mode); + MySQLMetaImpl(const DBMetaOptions& options, const int& mode); ~MySQLMetaImpl(); - Status CreateTable(TableSchema &table_schema) override; + Status + CreateTable(TableSchema& table_schema) override; - Status DescribeTable(TableSchema &table_schema) override; + Status + DescribeTable(TableSchema& table_schema) override; - Status HasTable(const std::string &table_id, bool &has_or_not) override; + Status + HasTable(const std::string& table_id, bool& has_or_not) override; - Status AllTables(std::vector &table_schema_array) override; + Status + AllTables(std::vector& table_schema_array) override; - Status DeleteTable(const std::string &table_id) override; + Status + DeleteTable(const std::string& table_id) override; - Status DeleteTableFiles(const std::string &table_id) override; + Status + DeleteTableFiles(const std::string& table_id) override; - Status CreateTableFile(TableFileSchema &file_schema) override; + Status + CreateTableFile(TableFileSchema& file_schema) override; - Status DropPartitionsByDates(const std::string &table_id, - const DatesT &dates) override; + Status + DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override; - Status GetTableFiles(const std::string &table_id, - const std::vector &ids, - TableFilesSchema &table_files) override; + Status + GetTableFiles(const std::string& table_id, const std::vector& ids, TableFilesSchema& table_files) override; - Status FilesByType(const std::string &table_id, - const std::vector &file_types, - std::vector &file_ids) override; + Status + FilesByType(const std::string& table_id, const std::vector& file_types, + std::vector& file_ids) override; - Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) override; + Status + UpdateTableIndex(const std::string& table_id, const TableIndex& index) override; - Status UpdateTableFlag(const std::string &table_id, int64_t flag) override; + Status + UpdateTableFlag(const std::string& table_id, int64_t flag) override; - Status DescribeTableIndex(const std::string &table_id, TableIndex &index) override; + Status + DescribeTableIndex(const std::string& table_id, TableIndex& index) override; - Status DropTableIndex(const std::string &table_id) override; + Status + DropTableIndex(const std::string& table_id) override; - Status UpdateTableFile(TableFileSchema &file_schema) override; + Status + UpdateTableFile(TableFileSchema& file_schema) override; - Status UpdateTableFilesToIndex(const std::string &table_id) override; + Status + UpdateTableFilesToIndex(const std::string& table_id) override; - Status UpdateTableFiles(TableFilesSchema &files) override; + Status + UpdateTableFiles(TableFilesSchema& files) override; - Status FilesToSearch(const std::string &table_id, - const std::vector &ids, - const DatesT &partition, - DatePartionedTableFilesSchema &files) override; + Status + FilesToSearch(const std::string& table_id, const std::vector& ids, const DatesT& partition, + DatePartionedTableFilesSchema& files) override; - Status FilesToMerge(const std::string &table_id, - DatePartionedTableFilesSchema &files) override; + Status + FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) override; - Status FilesToIndex(TableFilesSchema &) override; + Status + FilesToIndex(TableFilesSchema&) override; - Status Archive() override; + Status + Archive() override; - Status Size(uint64_t &result) override; + Status + Size(uint64_t& result) override; - Status CleanUp() override; + Status + CleanUp() override; - Status CleanUpFilesWithTTL(uint16_t seconds) override; + Status + CleanUpFilesWithTTL(uint16_t seconds) override; - Status DropAll() override; + Status + DropAll() override; - Status Count(const std::string &table_id, uint64_t &result) override; + Status + Count(const std::string& table_id, uint64_t& result) override; private: - Status NextFileId(std::string &file_id); - Status NextTableId(std::string &table_id); - Status DiscardFiles(int64_t to_discard_size); + Status + NextFileId(std::string& file_id); + Status + NextTableId(std::string& table_id); + Status + DiscardFiles(int64_t to_discard_size); - void ValidateMetaSchema(); - Status Initialize(); + void + ValidateMetaSchema(); + Status + Initialize(); private: const DBMetaOptions options_; @@ -113,10 +137,10 @@ class MySQLMetaImpl : public Meta { std::shared_ptr mysql_connection_pool_; bool safe_grab_ = false; -// std::mutex connectionMutex_; -}; // DBMetaImpl + // std::mutex connectionMutex_; +}; // DBMetaImpl -} // namespace meta -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/meta/SqliteMetaImpl.h b/cpp/src/db/meta/SqliteMetaImpl.h index 978e5e5a4b..70f22db281 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.h +++ b/cpp/src/db/meta/SqliteMetaImpl.h @@ -21,8 +21,8 @@ #include "db/Options.h" #include -#include #include +#include namespace zilliz { namespace milvus { @@ -30,86 +30,112 @@ namespace engine { namespace meta { auto -StoragePrototype(const std::string &path); +StoragePrototype(const std::string& path); class SqliteMetaImpl : public Meta { public: - explicit SqliteMetaImpl(const DBMetaOptions &options); + explicit SqliteMetaImpl(const DBMetaOptions& options); ~SqliteMetaImpl(); - Status CreateTable(TableSchema &table_schema) override; + Status + CreateTable(TableSchema& table_schema) override; - Status DescribeTable(TableSchema &table_schema) override; + Status + DescribeTable(TableSchema& table_schema) override; - Status HasTable(const std::string &table_id, bool &has_or_not) override; + Status + HasTable(const std::string& table_id, bool& has_or_not) override; - Status AllTables(std::vector &table_schema_array) override; + Status + AllTables(std::vector& table_schema_array) override; - Status DeleteTable(const std::string &table_id) override; + Status + DeleteTable(const std::string& table_id) override; - Status DeleteTableFiles(const std::string &table_id) override; + Status + DeleteTableFiles(const std::string& table_id) override; - Status CreateTableFile(TableFileSchema &file_schema) override; + Status + CreateTableFile(TableFileSchema& file_schema) override; - Status DropPartitionsByDates(const std::string &table_id, const DatesT &dates) override; + Status + DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override; - Status GetTableFiles(const std::string &table_id, - const std::vector &ids, - TableFilesSchema &table_files) override; + Status + GetTableFiles(const std::string& table_id, const std::vector& ids, TableFilesSchema& table_files) override; - Status FilesByType(const std::string &table_id, - const std::vector &file_types, - std::vector &file_ids) override; + Status + FilesByType(const std::string& table_id, const std::vector& file_types, + std::vector& file_ids) override; - Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) override; + Status + UpdateTableIndex(const std::string& table_id, const TableIndex& index) override; - Status UpdateTableFlag(const std::string &table_id, int64_t flag) override; + Status + UpdateTableFlag(const std::string& table_id, int64_t flag) override; - Status DescribeTableIndex(const std::string &table_id, TableIndex &index) override; + Status + DescribeTableIndex(const std::string& table_id, TableIndex& index) override; - Status DropTableIndex(const std::string &table_id) override; + Status + DropTableIndex(const std::string& table_id) override; - Status UpdateTableFilesToIndex(const std::string &table_id) override; + Status + UpdateTableFilesToIndex(const std::string& table_id) override; - Status UpdateTableFile(TableFileSchema &file_schema) override; + Status + UpdateTableFile(TableFileSchema& file_schema) override; - Status UpdateTableFiles(TableFilesSchema &files) override; + Status + UpdateTableFiles(TableFilesSchema& files) override; - Status FilesToSearch(const std::string &table_id, - const std::vector &ids, - const DatesT &partition, - DatePartionedTableFilesSchema &files) override; + Status + FilesToSearch(const std::string& table_id, const std::vector& ids, const DatesT& partition, + DatePartionedTableFilesSchema& files) override; - Status FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) override; + Status + FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) override; - Status FilesToIndex(TableFilesSchema &) override; + Status + FilesToIndex(TableFilesSchema&) override; - Status Archive() override; + Status + Archive() override; - Status Size(uint64_t &result) override; + Status + Size(uint64_t& result) override; - Status CleanUp() override; + Status + CleanUp() override; - Status CleanUpFilesWithTTL(uint16_t seconds) override; + Status + CleanUpFilesWithTTL(uint16_t seconds) override; - Status DropAll() override; + Status + DropAll() override; - Status Count(const std::string &table_id, uint64_t &result) override; + Status + Count(const std::string& table_id, uint64_t& result) override; private: - Status NextFileId(std::string &file_id); - Status NextTableId(std::string &table_id); - Status DiscardFiles(int64_t to_discard_size); + Status + NextFileId(std::string& file_id); + Status + NextTableId(std::string& table_id); + Status + DiscardFiles(int64_t to_discard_size); - void ValidateMetaSchema(); - Status Initialize(); + void + ValidateMetaSchema(); + Status + Initialize(); private: const DBMetaOptions options_; std::mutex meta_mutex_; -}; // DBMetaImpl +}; // DBMetaImpl -} // namespace meta -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index 22c282d4ea..bced85af6d 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -17,25 +17,25 @@ #include #include -#include -#include #include #include +#include +#include -#include "utils/easylogging++.h" -#include "utils/SignalUtil.h" -#include "utils/CommonUtil.h" +#include "../version.h" #include "metrics/Metrics.h" #include "server/Server.h" -#include "../version.h" - +#include "utils/CommonUtil.h" +#include "utils/SignalUtil.h" +#include "utils/easylogging++.h" INITIALIZE_EASYLOGGINGPP -void print_help(const std::string &app_name); +void +print_help(const std::string& app_name); int -main(int argc, char *argv[]) { +main(int argc, char* argv[]) { std::cout << std::endl << "Welcome to use Milvus by Zilliz!" << std::endl; std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << " built at " << BUILD_TIME << std::endl; @@ -63,21 +63,21 @@ main(int argc, char *argv[]) { while ((value = getopt_long(argc, argv, "c:l:p:dh", long_options, &option_index)) != -1) { switch (value) { case 'c': { - char *config_filename_ptr = strdup(optarg); + char* config_filename_ptr = strdup(optarg); config_filename = config_filename_ptr; free(config_filename_ptr); std::cout << "Loading configuration from: " << config_filename << std::endl; break; } case 'l': { - char *log_filename_ptr = strdup(optarg); + char* log_filename_ptr = strdup(optarg); log_config_file = log_filename_ptr; free(log_filename_ptr); std::cout << "Initial log config from: " << log_config_file << std::endl; break; } case 'p': { - char *pid_filename_ptr = strdup(optarg); + char* pid_filename_ptr = strdup(optarg); pid_filename = pid_filename_ptr; free(pid_filename_ptr); std::cout << pid_filename << std::endl; @@ -106,7 +106,7 @@ main(int argc, char *argv[]) { signal(SIGUSR2, zilliz::milvus::server::SignalUtil::HandleSignal); signal(SIGTERM, zilliz::milvus::server::SignalUtil::HandleSignal); - zilliz::milvus::server::Server &server = zilliz::milvus::server::Server::GetInstance(); + zilliz::milvus::server::Server& server = zilliz::milvus::server::Server::GetInstance(); server.Init(start_daemonized, pid_filename, config_filename, log_config_file); server.Start(); @@ -117,7 +117,7 @@ main(int argc, char *argv[]) { } void -print_help(const std::string &app_name) { +print_help(const std::string& app_name) { std::cout << std::endl << "Usage: " << app_name << " [OPTIONS]" << std::endl << std::endl; std::cout << " Options:" << std::endl; std::cout << " -h --help Print this help" << std::endl; diff --git a/cpp/src/metrics/MetricBase.h b/cpp/src/metrics/MetricBase.h index 32a0e715e6..5442b9b714 100644 --- a/cpp/src/metrics/MetricBase.h +++ b/cpp/src/metrics/MetricBase.h @@ -15,11 +15,10 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include "utils/Error.h" #include "SystemInfo.h" +#include "utils/Error.h" #include @@ -28,142 +27,185 @@ namespace milvus { namespace server { class MetricsBase { public: - static MetricsBase & + static MetricsBase& GetInstance() { static MetricsBase instance; return instance; } - virtual ErrorCode Init() { + virtual ErrorCode + Init() { } - virtual void AddVectorsSuccessTotalIncrement(double value = 1) { + virtual void + AddVectorsSuccessTotalIncrement(double value = 1) { } - virtual void AddVectorsFailTotalIncrement(double value = 1) { + virtual void + AddVectorsFailTotalIncrement(double value = 1) { } - virtual void AddVectorsDurationHistogramOberve(double value) { + virtual void + AddVectorsDurationHistogramOberve(double value) { } - virtual void RawFileSizeHistogramObserve(double value) { + virtual void + RawFileSizeHistogramObserve(double value) { } - virtual void IndexFileSizeHistogramObserve(double value) { + virtual void + IndexFileSizeHistogramObserve(double value) { } - virtual void BuildIndexDurationSecondsHistogramObserve(double value) { + virtual void + BuildIndexDurationSecondsHistogramObserve(double value) { } - virtual void CpuCacheUsageGaugeSet(double value) { + virtual void + CpuCacheUsageGaugeSet(double value) { } - virtual void GpuCacheUsageGaugeSet() { + virtual void + GpuCacheUsageGaugeSet() { } - virtual void MetaAccessTotalIncrement(double value = 1) { + virtual void + MetaAccessTotalIncrement(double value = 1) { } - virtual void MetaAccessDurationSecondsHistogramObserve(double value) { + virtual void + MetaAccessDurationSecondsHistogramObserve(double value) { } - virtual void FaissDiskLoadDurationSecondsHistogramObserve(double value) { + virtual void + FaissDiskLoadDurationSecondsHistogramObserve(double value) { } - virtual void FaissDiskLoadSizeBytesHistogramObserve(double value) { + virtual void + FaissDiskLoadSizeBytesHistogramObserve(double value) { } - virtual void CacheAccessTotalIncrement(double value = 1) { + virtual void + CacheAccessTotalIncrement(double value = 1) { } - virtual void MemTableMergeDurationSecondsHistogramObserve(double value) { + virtual void + MemTableMergeDurationSecondsHistogramObserve(double value) { } - virtual void SearchIndexDataDurationSecondsHistogramObserve(double value) { + virtual void + SearchIndexDataDurationSecondsHistogramObserve(double value) { } - virtual void SearchRawDataDurationSecondsHistogramObserve(double value) { + virtual void + SearchRawDataDurationSecondsHistogramObserve(double value) { } - virtual void IndexFileSizeTotalIncrement(double value = 1) { + virtual void + IndexFileSizeTotalIncrement(double value = 1) { } - virtual void RawFileSizeTotalIncrement(double value = 1) { + virtual void + RawFileSizeTotalIncrement(double value = 1) { } - virtual void IndexFileSizeGaugeSet(double value) { + virtual void + IndexFileSizeGaugeSet(double value) { } - virtual void RawFileSizeGaugeSet(double value) { + virtual void + RawFileSizeGaugeSet(double value) { } - virtual void FaissDiskLoadIOSpeedGaugeSet(double value) { + virtual void + FaissDiskLoadIOSpeedGaugeSet(double value) { } - virtual void QueryResponseSummaryObserve(double value) { + virtual void + QueryResponseSummaryObserve(double value) { } - virtual void DiskStoreIOSpeedGaugeSet(double value) { + virtual void + DiskStoreIOSpeedGaugeSet(double value) { } - virtual void DataFileSizeGaugeSet(double value) { + virtual void + DataFileSizeGaugeSet(double value) { } - virtual void AddVectorsSuccessGaugeSet(double value) { + virtual void + AddVectorsSuccessGaugeSet(double value) { } - virtual void AddVectorsFailGaugeSet(double value) { + virtual void + AddVectorsFailGaugeSet(double value) { } - virtual void QueryVectorResponseSummaryObserve(double value, int count = 1) { + virtual void + QueryVectorResponseSummaryObserve(double value, int count = 1) { } - virtual void QueryVectorResponsePerSecondGaugeSet(double value) { + virtual void + QueryVectorResponsePerSecondGaugeSet(double value) { } - virtual void CPUUsagePercentSet() { + virtual void + CPUUsagePercentSet() { } - virtual void RAMUsagePercentSet() { + virtual void + RAMUsagePercentSet() { } - virtual void QueryResponsePerSecondGaugeSet(double value) { + virtual void + QueryResponsePerSecondGaugeSet(double value) { } - virtual void GPUPercentGaugeSet() { + virtual void + GPUPercentGaugeSet() { } - virtual void GPUMemoryUsageGaugeSet() { + virtual void + GPUMemoryUsageGaugeSet() { } - virtual void AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) { + virtual void + AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) { } - virtual void QueryIndexTypePerSecondSet(std::string type, double value) { + virtual void + QueryIndexTypePerSecondSet(std::string type, double value) { } - virtual void ConnectionGaugeIncrement() { + virtual void + ConnectionGaugeIncrement() { } - virtual void ConnectionGaugeDecrement() { + virtual void + ConnectionGaugeDecrement() { } - virtual void KeepingAliveCounterIncrement(double value = 1) { + virtual void + KeepingAliveCounterIncrement(double value = 1) { } - virtual void OctetsSet() { + virtual void + OctetsSet() { } - virtual void CPUCoreUsagePercentSet() { + virtual void + CPUCoreUsagePercentSet() { } - virtual void GPUTemperature() { + virtual void + GPUTemperature() { } - virtual void CPUTemperature() { + virtual void + CPUTemperature() { } }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/metrics/Metrics.cpp b/cpp/src/metrics/Metrics.cpp index 0a1b333b42..9a182ce571 100644 --- a/cpp/src/metrics/Metrics.cpp +++ b/cpp/src/metrics/Metrics.cpp @@ -16,8 +16,8 @@ // under the License. #include "metrics/Metrics.h" -#include "server/Config.h" #include "PrometheusMetrics.h" +#include "server/Config.h" #include @@ -25,15 +25,15 @@ namespace zilliz { namespace milvus { namespace server { -MetricsBase & +MetricsBase& Metrics::GetInstance() { - static MetricsBase &instance = CreateMetricsCollector(); + static MetricsBase& instance = CreateMetricsCollector(); return instance; } -MetricsBase & +MetricsBase& Metrics::CreateMetricsCollector() { - Config &config = Config::GetInstance(); + Config& config = Config::GetInstance(); std::string collector_type_str; config.GetMetricConfigCollector(collector_type_str); @@ -45,6 +45,6 @@ Metrics::CreateMetricsCollector() { } } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/metrics/Metrics.h b/cpp/src/metrics/Metrics.h index 66e06b8ba1..f644004f14 100644 --- a/cpp/src/metrics/Metrics.h +++ b/cpp/src/metrics/Metrics.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "MetricBase.h" @@ -26,20 +25,18 @@ namespace milvus { namespace server { #define METRICS_NOW_TIME std::chrono::system_clock::now() -#define METRICS_MICROSECONDS(a, b) (std::chrono::duration_cast (b-a)).count(); +#define METRICS_MICROSECONDS(a, b) (std::chrono::duration_cast(b - a)).count(); -enum class MetricCollectorType { - INVALID, - PROMETHEUS, - ZABBIX -}; +enum class MetricCollectorType { INVALID, PROMETHEUS, ZABBIX }; class Metrics { public: - static MetricsBase &GetInstance(); + static MetricsBase& + GetInstance(); private: - static MetricsBase &CreateMetricsCollector(); + static MetricsBase& + CreateMetricsCollector(); }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CollectMetricsBase { @@ -50,7 +47,8 @@ class CollectMetricsBase { virtual ~CollectMetricsBase() = default; - double TimeFromBegine() { + double + TimeFromBegine() { auto end_time = METRICS_NOW_TIME; return METRICS_MICROSECONDS(start_time_, end_time); } @@ -63,7 +61,7 @@ class CollectMetricsBase { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CollectInsertMetrics : CollectMetricsBase { public: - CollectInsertMetrics(size_t n, Status &status) : n_(n), status_(status) { + CollectInsertMetrics(size_t n, Status& status) : n_(n), status_(status) { } ~CollectInsertMetrics() { @@ -87,7 +85,7 @@ class CollectInsertMetrics : CollectMetricsBase { private: size_t n_; - Status &status_; + Status& status_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -162,7 +160,7 @@ class CollectSerializeMetrics : CollectMetricsBase { ~CollectSerializeMetrics() { auto total_time = TimeFromBegine(); - server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet((double) size_ / total_time); + server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet((double)size_ / total_time); } private: @@ -177,8 +175,7 @@ class CollectAddMetrics : CollectMetricsBase { ~CollectAddMetrics() { auto total_time = TimeFromBegine(); - server::Metrics::GetInstance().AddVectorsPerSecondGaugeSet(static_cast(n_), - static_cast(dimension_), + server::Metrics::GetInstance().AddVectorsPerSecondGaugeSet(static_cast(n_), static_cast(dimension_), total_time); } @@ -256,6 +253,6 @@ class MetricCollector : CollectMetricsBase { } }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index 8dcc97aa8b..dd6410b46a 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -15,12 +15,11 @@ // specific language governing permissions and limitations // under the License. - #include "metrics/PrometheusMetrics.h" +#include "SystemInfo.h" #include "cache/GpuCacheMgr.h" #include "server/Config.h" #include "utils/Log.h" -#include "SystemInfo.h" #include #include @@ -32,7 +31,7 @@ namespace server { ErrorCode PrometheusMetrics::Init() { try { - Config &config = Config::GetInstance(); + Config& config = Config::GetInstance(); Status s = config.GetMetricConfigEnableMonitor(startup_); if (!s.ok()) return s.code(); if (!startup_) return SERVER_SUCCESS; @@ -49,7 +48,7 @@ PrometheusMetrics::Init() { // Exposer Registry exposer_ptr_->RegisterCollectable(registry_); - } catch (std::exception &ex) { + } catch (std::exception& ex) { SERVER_LOG_ERROR << "Failed to connect prometheus server: " << std::string(ex.what()); return SERVER_UNEXPECTED_ERROR; } @@ -79,8 +78,8 @@ PrometheusMetrics::GPUPercentGaugeSet() { std::vector used_memory = server::SystemInfo::GetInstance().GPUMemoryUsed(); for (int i = 0; i < numDevice; ++i) { - prometheus::Gauge &GPU_percent = GPU_percent_.Add({{"DeviceNum", std::to_string(i)}}); - double percent = (double) used_memory[i] / (double) used_total[i]; + prometheus::Gauge& GPU_percent = GPU_percent_.Add({{"DeviceNum", std::to_string(i)}}); + double percent = (double)used_memory[i] / (double)used_total[i]; GPU_percent.Set(percent * 100); } } @@ -93,7 +92,7 @@ PrometheusMetrics::GPUMemoryUsageGaugeSet() { int numDevice = server::SystemInfo::GetInstance().num_device(); for (int i = 0; i < numDevice; ++i) { - prometheus::Gauge &GPU_memory = GPU_memory_usage_.Add({{"DeviceNum", std::to_string(i)}}); + prometheus::Gauge& GPU_memory = GPU_memory_usage_.Add({{"DeviceNum", std::to_string(i)}}); GPU_memory.Set(values[i] / MtoB); } } @@ -155,54 +154,51 @@ PrometheusMetrics::OctetsSet() { void PrometheusMetrics::CPUCoreUsagePercentSet() { - if (!startup_) - return; + if (!startup_) return; std::vector cpu_core_percent = server::SystemInfo::GetInstance().CPUCorePercent(); for (int i = 0; i < cpu_core_percent.size(); ++i) { - prometheus::Gauge &core_percent = CPU_.Add({{"CPU", std::to_string(i)}}); + prometheus::Gauge& core_percent = CPU_.Add({{"CPU", std::to_string(i)}}); core_percent.Set(cpu_core_percent[i]); } } void PrometheusMetrics::GPUTemperature() { - if (!startup_) - return; + if (!startup_) return; std::vector GPU_temperatures = server::SystemInfo::GetInstance().GPUTemperature(); for (int i = 0; i < GPU_temperatures.size(); ++i) { - prometheus::Gauge &gpu_temp = GPU_temperature_.Add({{"GPU", std::to_string(i)}}); + prometheus::Gauge& gpu_temp = GPU_temperature_.Add({{"GPU", std::to_string(i)}}); gpu_temp.Set(GPU_temperatures[i]); } } void PrometheusMetrics::CPUTemperature() { - if (!startup_) - return; + if (!startup_) return; std::vector CPU_temperatures = server::SystemInfo::GetInstance().CPUTemperature(); for (int i = 0; i < CPU_temperatures.size(); ++i) { - prometheus::Gauge &cpu_temp = CPU_temperature_.Add({{"CPU", std::to_string(i)}}); + prometheus::Gauge& cpu_temp = CPU_temperature_.Add({{"CPU", std::to_string(i)}}); cpu_temp.Set(CPU_temperatures[i]); } } void PrometheusMetrics::GpuCacheUsageGaugeSet() { -// std::vector gpu_ids = {0}; -// for(auto i = 0; i < gpu_ids.size(); ++i) { -// uint64_t cache_usage = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheUsage(); -// uint64_t cache_capacity = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheCapacity(); -// prometheus::Gauge &gpu_cache = gpu_cache_usage_.Add({{"GPU_Cache", std::to_string(i)}}); -// gpu_cache.Set(cache_usage * 100 / cache_capacity); -// } + // std::vector gpu_ids = {0}; + // for(auto i = 0; i < gpu_ids.size(); ++i) { + // uint64_t cache_usage = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheUsage(); + // uint64_t cache_capacity = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheCapacity(); + // prometheus::Gauge &gpu_cache = gpu_cache_usage_.Add({{"GPU_Cache", std::to_string(i)}}); + // gpu_cache.Set(cache_usage * 100 / cache_capacity); + // } } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/metrics/PrometheusMetrics.h b/cpp/src/metrics/PrometheusMetrics.h index ce45c5e711..9e009186d2 100644 --- a/cpp/src/metrics/PrometheusMetrics.h +++ b/cpp/src/metrics/PrometheusMetrics.h @@ -17,19 +17,19 @@ #pragma once -#include -#include -#include #include +#include #include +#include #include +#include -#include "utils/Error.h" #include "MetricBase.h" +#include "utils/Error.h" #define METRICS_NOW_TIME std::chrono::system_clock::now() //#define server::Metrics::GetInstance() server::GetInstance() -#define METRICS_MICROSECONDS(a, b) (std::chrono::duration_cast (b-a)).count(); +#define METRICS_MICROSECONDS(a, b) (std::chrono::duration_cast(b - a)).count(); namespace zilliz { namespace milvus { @@ -37,13 +37,14 @@ namespace server { class PrometheusMetrics : public MetricsBase { public: - static PrometheusMetrics & + static PrometheusMetrics& GetInstance() { static PrometheusMetrics instance; return instance; } - ErrorCode Init(); + ErrorCode + Init(); private: std::shared_ptr exposer_ptr_; @@ -51,163 +52,191 @@ class PrometheusMetrics : public MetricsBase { bool startup_ = false; public: - void SetStartup(bool startup) { + void + SetStartup(bool startup) { startup_ = startup; } - void AddVectorsSuccessTotalIncrement(double value = 1.0) override { + void + AddVectorsSuccessTotalIncrement(double value = 1.0) override { if (startup_) { add_vectors_success_total_.Increment(value); } } - void AddVectorsFailTotalIncrement(double value = 1.0) override { + void + AddVectorsFailTotalIncrement(double value = 1.0) override { if (startup_) { add_vectors_fail_total_.Increment(value); } } - void AddVectorsDurationHistogramOberve(double value) override { + void + AddVectorsDurationHistogramOberve(double value) override { if (startup_) { add_vectors_duration_histogram_.Observe(value); } } - void RawFileSizeHistogramObserve(double value) override { + void + RawFileSizeHistogramObserve(double value) override { if (startup_) { raw_files_size_histogram_.Observe(value); } } - void IndexFileSizeHistogramObserve(double value) override { + void + IndexFileSizeHistogramObserve(double value) override { if (startup_) { index_files_size_histogram_.Observe(value); } } - void BuildIndexDurationSecondsHistogramObserve(double value) override { + void + BuildIndexDurationSecondsHistogramObserve(double value) override { if (startup_) { build_index_duration_seconds_histogram_.Observe(value); } } - void CpuCacheUsageGaugeSet(double value) override { + void + CpuCacheUsageGaugeSet(double value) override { if (startup_) { cpu_cache_usage_gauge_.Set(value); } } - void GpuCacheUsageGaugeSet() override; + void + GpuCacheUsageGaugeSet() override; - void MetaAccessTotalIncrement(double value = 1) override { + void + MetaAccessTotalIncrement(double value = 1) override { if (startup_) { meta_access_total_.Increment(value); } } - void MetaAccessDurationSecondsHistogramObserve(double value) override { + void + MetaAccessDurationSecondsHistogramObserve(double value) override { if (startup_) { meta_access_duration_seconds_histogram_.Observe(value); } } - void FaissDiskLoadDurationSecondsHistogramObserve(double value) override { + void + FaissDiskLoadDurationSecondsHistogramObserve(double value) override { if (startup_) { faiss_disk_load_duration_seconds_histogram_.Observe(value); } } - void FaissDiskLoadSizeBytesHistogramObserve(double value) override { + void + FaissDiskLoadSizeBytesHistogramObserve(double value) override { if (startup_) { faiss_disk_load_size_bytes_histogram_.Observe(value); } } - void FaissDiskLoadIOSpeedGaugeSet(double value) override { + void + FaissDiskLoadIOSpeedGaugeSet(double value) override { if (startup_) { faiss_disk_load_IO_speed_gauge_.Set(value); } } - void CacheAccessTotalIncrement(double value = 1) override { + void + CacheAccessTotalIncrement(double value = 1) override { if (startup_) { cache_access_total_.Increment(value); } } - void MemTableMergeDurationSecondsHistogramObserve(double value) override { + void + MemTableMergeDurationSecondsHistogramObserve(double value) override { if (startup_) { mem_table_merge_duration_seconds_histogram_.Observe(value); } } - void SearchIndexDataDurationSecondsHistogramObserve(double value) override { + void + SearchIndexDataDurationSecondsHistogramObserve(double value) override { if (startup_) { search_index_data_duration_seconds_histogram_.Observe(value); } } - void SearchRawDataDurationSecondsHistogramObserve(double value) override { + void + SearchRawDataDurationSecondsHistogramObserve(double value) override { if (startup_) { search_raw_data_duration_seconds_histogram_.Observe(value); } } - void IndexFileSizeTotalIncrement(double value = 1) override { + void + IndexFileSizeTotalIncrement(double value = 1) override { if (startup_) { index_file_size_total_.Increment(value); } } - void RawFileSizeTotalIncrement(double value = 1) override { + void + RawFileSizeTotalIncrement(double value = 1) override { if (startup_) { raw_file_size_total_.Increment(value); } } - void IndexFileSizeGaugeSet(double value) override { + void + IndexFileSizeGaugeSet(double value) override { if (startup_) { index_file_size_gauge_.Set(value); } } - void RawFileSizeGaugeSet(double value) override { + void + RawFileSizeGaugeSet(double value) override { if (startup_) { raw_file_size_gauge_.Set(value); } } - void QueryResponseSummaryObserve(double value) override { + void + QueryResponseSummaryObserve(double value) override { if (startup_) { query_response_summary_.Observe(value); } } - void DiskStoreIOSpeedGaugeSet(double value) override { + void + DiskStoreIOSpeedGaugeSet(double value) override { if (startup_) { disk_store_IO_speed_gauge_.Set(value); } } - void DataFileSizeGaugeSet(double value) override { + void + DataFileSizeGaugeSet(double value) override { if (startup_) { data_file_size_gauge_.Set(value); } } - void AddVectorsSuccessGaugeSet(double value) override { + void + AddVectorsSuccessGaugeSet(double value) override { if (startup_) { add_vectors_success_gauge_.Set(value); } } - void AddVectorsFailGaugeSet(double value) override { + void + AddVectorsFailGaugeSet(double value) override { if (startup_) { add_vectors_fail_gauge_.Set(value); } } - void QueryVectorResponseSummaryObserve(double value, int count = 1) override { + void + QueryVectorResponseSummaryObserve(double value, int count = 1) override { if (startup_) { for (int i = 0; i < count; ++i) { query_vector_response_summary_.Observe(value); @@ -215,412 +244,414 @@ class PrometheusMetrics : public MetricsBase { } } - void QueryVectorResponsePerSecondGaugeSet(double value) override { + void + QueryVectorResponsePerSecondGaugeSet(double value) override { if (startup_) { query_vector_response_per_second_gauge_.Set(value); } } - void CPUUsagePercentSet() override; - void CPUCoreUsagePercentSet() override; + void + CPUUsagePercentSet() override; + void + CPUCoreUsagePercentSet() override; - void RAMUsagePercentSet() override; + void + RAMUsagePercentSet() override; - void QueryResponsePerSecondGaugeSet(double value) override { + void + QueryResponsePerSecondGaugeSet(double value) override { if (startup_) { query_response_per_second_gauge.Set(value); } } - void GPUPercentGaugeSet() override; - void GPUMemoryUsageGaugeSet() override; - void AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) override; - void QueryIndexTypePerSecondSet(std::string type, double value) override; - void ConnectionGaugeIncrement() override; - void ConnectionGaugeDecrement() override; + void + GPUPercentGaugeSet() override; + void + GPUMemoryUsageGaugeSet() override; + void + AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) override; + void + QueryIndexTypePerSecondSet(std::string type, double value) override; + void + ConnectionGaugeIncrement() override; + void + ConnectionGaugeDecrement() override; - void KeepingAliveCounterIncrement(double value = 1) override { + void + KeepingAliveCounterIncrement(double value = 1) override { if (startup_) { keeping_alive_counter_.Increment(value); } } - void OctetsSet() override; + void + OctetsSet() override; - void GPUTemperature() override; - void CPUTemperature() override; + void + GPUTemperature() override; + void + CPUTemperature() override; - std::shared_ptr &exposer_ptr() { + std::shared_ptr& + exposer_ptr() { return exposer_ptr_; } -// prometheus::Exposer& exposer() { return exposer_;} - std::shared_ptr ®istry_ptr() { + // prometheus::Exposer& exposer() { return exposer_;} + std::shared_ptr& + registry_ptr() { return registry_; } // ..... private: ////all from db_connection.cpp -// prometheus::Family &connect_request_ = prometheus::BuildCounter() -// .Name("connection_total") -// .Help("total number of connection has been made") -// .Register(*registry_); -// prometheus::Counter &connection_total_ = connect_request_.Add({}); + // prometheus::Family &connect_request_ = prometheus::BuildCounter() + // .Name("connection_total") + // .Help("total number of connection has been made") + // .Register(*registry_); + // prometheus::Counter &connection_total_ = connect_request_.Add({}); ////all from DBImpl.cpp using BucketBoundaries = std::vector; - //record add_group request - prometheus::Family &add_group_request_ = prometheus::BuildCounter() - .Name("add_group_request_total") - .Help("the number of add_group request") - .Register(*registry_); + // record add_group request + prometheus::Family& add_group_request_ = prometheus::BuildCounter() + .Name("add_group_request_total") + .Help("the number of add_group request") + .Register(*registry_); - prometheus::Counter &add_group_success_total_ = add_group_request_.Add({{"outcome", "success"}}); - prometheus::Counter &add_group_fail_total_ = add_group_request_.Add({{"outcome", "fail"}}); + prometheus::Counter& add_group_success_total_ = add_group_request_.Add({{"outcome", "success"}}); + prometheus::Counter& add_group_fail_total_ = add_group_request_.Add({{"outcome", "fail"}}); - //record get_group request - prometheus::Family &get_group_request_ = prometheus::BuildCounter() - .Name("get_group_request_total") - .Help("the number of get_group request") - .Register(*registry_); + // record get_group request + prometheus::Family& get_group_request_ = prometheus::BuildCounter() + .Name("get_group_request_total") + .Help("the number of get_group request") + .Register(*registry_); - prometheus::Counter &get_group_success_total_ = get_group_request_.Add({{"outcome", "success"}}); - prometheus::Counter &get_group_fail_total_ = get_group_request_.Add({{"outcome", "fail"}}); + prometheus::Counter& get_group_success_total_ = get_group_request_.Add({{"outcome", "success"}}); + prometheus::Counter& get_group_fail_total_ = get_group_request_.Add({{"outcome", "fail"}}); - //record has_group request - prometheus::Family &has_group_request_ = prometheus::BuildCounter() - .Name("has_group_request_total") - .Help("the number of has_group request") - .Register(*registry_); + // record has_group request + prometheus::Family& has_group_request_ = prometheus::BuildCounter() + .Name("has_group_request_total") + .Help("the number of has_group request") + .Register(*registry_); - prometheus::Counter &has_group_success_total_ = has_group_request_.Add({{"outcome", "success"}}); - prometheus::Counter &has_group_fail_total_ = has_group_request_.Add({{"outcome", "fail"}}); + prometheus::Counter& has_group_success_total_ = has_group_request_.Add({{"outcome", "success"}}); + prometheus::Counter& has_group_fail_total_ = has_group_request_.Add({{"outcome", "fail"}}); - //record get_group_files - prometheus::Family &get_group_files_request_ = prometheus::BuildCounter() - .Name("get_group_files_request_total") - .Help("the number of get_group_files request") - .Register(*registry_); + // record get_group_files + prometheus::Family& get_group_files_request_ = + prometheus::BuildCounter() + .Name("get_group_files_request_total") + .Help("the number of get_group_files request") + .Register(*registry_); - prometheus::Counter &get_group_files_success_total_ = get_group_files_request_.Add({{"outcome", "success"}}); - prometheus::Counter &get_group_files_fail_total_ = get_group_files_request_.Add({{"outcome", "fail"}}); + prometheus::Counter& get_group_files_success_total_ = get_group_files_request_.Add({{"outcome", "success"}}); + prometheus::Counter& get_group_files_fail_total_ = get_group_files_request_.Add({{"outcome", "fail"}}); - //record add_vectors count and average time - //need to be considered - prometheus::Family &add_vectors_request_ = prometheus::BuildCounter() - .Name("add_vectors_request_total") - .Help("the number of vectors added") - .Register(*registry_); - prometheus::Counter &add_vectors_success_total_ = add_vectors_request_.Add({{"outcome", "success"}}); - prometheus::Counter &add_vectors_fail_total_ = add_vectors_request_.Add({{"outcome", "fail"}}); + // record add_vectors count and average time + // need to be considered + prometheus::Family& add_vectors_request_ = prometheus::BuildCounter() + .Name("add_vectors_request_total") + .Help("the number of vectors added") + .Register(*registry_); + prometheus::Counter& add_vectors_success_total_ = add_vectors_request_.Add({{"outcome", "success"}}); + prometheus::Counter& add_vectors_fail_total_ = add_vectors_request_.Add({{"outcome", "fail"}}); - prometheus::Family &add_vectors_duration_seconds_ = prometheus::BuildHistogram() - .Name("add_vector_duration_microseconds") - .Help("average time of adding every vector") - .Register(*registry_); - prometheus::Histogram &add_vectors_duration_histogram_ = + prometheus::Family& add_vectors_duration_seconds_ = + prometheus::BuildHistogram() + .Name("add_vector_duration_microseconds") + .Help("average time of adding every vector") + .Register(*registry_); + prometheus::Histogram& add_vectors_duration_histogram_ = add_vectors_duration_seconds_.Add({}, BucketBoundaries{0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.08, 0.1, 0.5, 1}); - //record search count and average time - prometheus::Family &search_request_ = prometheus::BuildCounter() - .Name("search_request_total") - .Help("the number of search request") - .Register(*registry_); - prometheus::Counter &search_success_total_ = search_request_.Add({{"outcome", "success"}}); - prometheus::Counter &search_fail_total_ = search_request_.Add({{"outcome", "fail"}}); + // record search count and average time + prometheus::Family& search_request_ = prometheus::BuildCounter() + .Name("search_request_total") + .Help("the number of search request") + .Register(*registry_); + prometheus::Counter& search_success_total_ = search_request_.Add({{"outcome", "success"}}); + prometheus::Counter& search_fail_total_ = search_request_.Add({{"outcome", "fail"}}); - prometheus::Family &search_request_duration_seconds_ = prometheus::BuildHistogram() - .Name("search_request_duration_microsecond") - .Help("histogram of processing time for each search") - .Register(*registry_); - prometheus::Histogram - &search_duration_histogram_ = search_request_duration_seconds_.Add({}, BucketBoundaries{0.1, 1.0, 10.0}); + prometheus::Family& search_request_duration_seconds_ = + prometheus::BuildHistogram() + .Name("search_request_duration_microsecond") + .Help("histogram of processing time for each search") + .Register(*registry_); + prometheus::Histogram& search_duration_histogram_ = + search_request_duration_seconds_.Add({}, BucketBoundaries{0.1, 1.0, 10.0}); - //record raw_files size histogram - prometheus::Family &raw_files_size_ = prometheus::BuildHistogram() - .Name("search_raw_files_bytes") - .Help("histogram of raw files size by bytes") - .Register(*registry_); - prometheus::Histogram - &raw_files_size_histogram_ = raw_files_size_.Add({}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9, 1e10}); + // record raw_files size histogram + prometheus::Family& raw_files_size_ = prometheus::BuildHistogram() + .Name("search_raw_files_bytes") + .Help("histogram of raw files size by bytes") + .Register(*registry_); + prometheus::Histogram& raw_files_size_histogram_ = + raw_files_size_.Add({}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9, 1e10}); - //record index_files size histogram - prometheus::Family &index_files_size_ = prometheus::BuildHistogram() - .Name("search_index_files_bytes") - .Help("histogram of index files size by bytes") - .Register(*registry_); - prometheus::Histogram - &index_files_size_histogram_ = index_files_size_.Add({}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9, 1e10}); + // record index_files size histogram + prometheus::Family& index_files_size_ = prometheus::BuildHistogram() + .Name("search_index_files_bytes") + .Help("histogram of index files size by bytes") + .Register(*registry_); + prometheus::Histogram& index_files_size_histogram_ = + index_files_size_.Add({}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9, 1e10}); - //record index and raw files size counter - prometheus::Family &file_size_total_ = prometheus::BuildCounter() - .Name("search_file_size_total") - .Help("searched index and raw file size") - .Register(*registry_); - prometheus::Counter &index_file_size_total_ = file_size_total_.Add({{"type", "index"}}); - prometheus::Counter &raw_file_size_total_ = file_size_total_.Add({{"type", "raw"}}); + // record index and raw files size counter + prometheus::Family& file_size_total_ = prometheus::BuildCounter() + .Name("search_file_size_total") + .Help("searched index and raw file size") + .Register(*registry_); + prometheus::Counter& index_file_size_total_ = file_size_total_.Add({{"type", "index"}}); + prometheus::Counter& raw_file_size_total_ = file_size_total_.Add({{"type", "raw"}}); - //record index and raw files size counter - prometheus::Family &file_size_gauge_ = prometheus::BuildGauge() - .Name("search_file_size_gauge") - .Help("searched current index and raw file size") - .Register(*registry_); - prometheus::Gauge &index_file_size_gauge_ = file_size_gauge_.Add({{"type", "index"}}); - prometheus::Gauge &raw_file_size_gauge_ = file_size_gauge_.Add({{"type", "raw"}}); + // record index and raw files size counter + prometheus::Family& file_size_gauge_ = prometheus::BuildGauge() + .Name("search_file_size_gauge") + .Help("searched current index and raw file size") + .Register(*registry_); + prometheus::Gauge& index_file_size_gauge_ = file_size_gauge_.Add({{"type", "index"}}); + prometheus::Gauge& raw_file_size_gauge_ = file_size_gauge_.Add({{"type", "raw"}}); - //record processing time for building index - prometheus::Family &build_index_duration_seconds_ = prometheus::BuildHistogram() - .Name("build_index_duration_microseconds") - .Help("histogram of processing time for building index") - .Register(*registry_); - prometheus::Histogram &build_index_duration_seconds_histogram_ = + // record processing time for building index + prometheus::Family& build_index_duration_seconds_ = + prometheus::BuildHistogram() + .Name("build_index_duration_microseconds") + .Help("histogram of processing time for building index") + .Register(*registry_); + prometheus::Histogram& build_index_duration_seconds_histogram_ = build_index_duration_seconds_.Add({}, BucketBoundaries{5e5, 2e6, 4e6, 6e6, 8e6, 1e7}); - //record processing time for all building index - prometheus::Family &all_build_index_duration_seconds_ = prometheus::BuildHistogram() - .Name("all_build_index_duration_microseconds") - .Help("histogram of processing time for building index") - .Register(*registry_); - prometheus::Histogram &all_build_index_duration_seconds_histogram_ = + // record processing time for all building index + prometheus::Family& all_build_index_duration_seconds_ = + prometheus::BuildHistogram() + .Name("all_build_index_duration_microseconds") + .Help("histogram of processing time for building index") + .Register(*registry_); + prometheus::Histogram& all_build_index_duration_seconds_histogram_ = all_build_index_duration_seconds_.Add({}, BucketBoundaries{2e6, 4e6, 6e6, 8e6, 1e7}); - //record duration of merging mem table - prometheus::Family &mem_table_merge_duration_seconds_ = prometheus::BuildHistogram() - .Name("mem_table_merge_duration_microseconds") - .Help("histogram of processing time for merging mem tables") - .Register(*registry_); - prometheus::Histogram &mem_table_merge_duration_seconds_histogram_ = + // record duration of merging mem table + prometheus::Family& mem_table_merge_duration_seconds_ = + prometheus::BuildHistogram() + .Name("mem_table_merge_duration_microseconds") + .Help("histogram of processing time for merging mem tables") + .Register(*registry_); + prometheus::Histogram& mem_table_merge_duration_seconds_histogram_ = mem_table_merge_duration_seconds_.Add({}, BucketBoundaries{5e4, 1e5, 2e5, 4e5, 6e5, 8e5, 1e6}); - //record search index and raw data duration - prometheus::Family &search_data_duration_seconds_ = prometheus::BuildHistogram() - .Name("search_data_duration_microseconds") - .Help("histograms of processing time for search index and raw data") - .Register(*registry_); - prometheus::Histogram &search_index_data_duration_seconds_histogram_ = + // record search index and raw data duration + prometheus::Family& search_data_duration_seconds_ = + prometheus::BuildHistogram() + .Name("search_data_duration_microseconds") + .Help("histograms of processing time for search index and raw data") + .Register(*registry_); + prometheus::Histogram& search_index_data_duration_seconds_histogram_ = search_data_duration_seconds_.Add({{"type", "index"}}, BucketBoundaries{1e5, 2e5, 4e5, 6e5, 8e5}); - prometheus::Histogram &search_raw_data_duration_seconds_histogram_ = + prometheus::Histogram& search_raw_data_duration_seconds_histogram_ = search_data_duration_seconds_.Add({{"type", "raw"}}, BucketBoundaries{1e5, 2e5, 4e5, 6e5, 8e5}); - ////all form Cache.cpp - //record cache usage, when insert/erase/clear/free - + // record cache usage, when insert/erase/clear/free ////all from Meta.cpp - //record meta visit count and time -// prometheus::Family &meta_visit_ = prometheus::BuildCounter() -// .Name("meta_visit_total") -// .Help("the number of accessing Meta") -// .Register(*registry_); -// prometheus::Counter &meta_visit_total_ = meta_visit_.Add({{}}); -// -// prometheus::Family &meta_visit_duration_seconds_ = prometheus::BuildHistogram() -// .Name("meta_visit_duration_seconds") -// .Help("histogram of processing time to get data from mata") -// .Register(*registry_); -// prometheus::Histogram &meta_visit_duration_seconds_histogram_ = -// meta_visit_duration_seconds_.Add({{}}, BucketBoundaries{0.1, 1.0, 10.0}); - + // record meta visit count and time + // prometheus::Family &meta_visit_ = prometheus::BuildCounter() + // .Name("meta_visit_total") + // .Help("the number of accessing Meta") + // .Register(*registry_); + // prometheus::Counter &meta_visit_total_ = meta_visit_.Add({{}}); + // + // prometheus::Family &meta_visit_duration_seconds_ = prometheus::BuildHistogram() + // .Name("meta_visit_duration_seconds") + // .Help("histogram of processing time to get data from mata") + // .Register(*registry_); + // prometheus::Histogram &meta_visit_duration_seconds_histogram_ = + // meta_visit_duration_seconds_.Add({{}}, BucketBoundaries{0.1, 1.0, 10.0}); ////all from MemManager.cpp - //record memory usage percent - prometheus::Family &mem_usage_percent_ = prometheus::BuildGauge() - .Name("memory_usage_percent") - .Help("memory usage percent") - .Register(*registry_); - prometheus::Gauge &mem_usage_percent_gauge_ = mem_usage_percent_.Add({}); + // record memory usage percent + prometheus::Family& mem_usage_percent_ = + prometheus::BuildGauge().Name("memory_usage_percent").Help("memory usage percent").Register(*registry_); + prometheus::Gauge& mem_usage_percent_gauge_ = mem_usage_percent_.Add({}); - //record memory usage toal - prometheus::Family &mem_usage_total_ = prometheus::BuildGauge() - .Name("memory_usage_total") - .Help("memory usage total") - .Register(*registry_); - prometheus::Gauge &mem_usage_total_gauge_ = mem_usage_total_.Add({}); + // record memory usage toal + prometheus::Family& mem_usage_total_ = + prometheus::BuildGauge().Name("memory_usage_total").Help("memory usage total").Register(*registry_); + prometheus::Gauge& mem_usage_total_gauge_ = mem_usage_total_.Add({}); ////all from DBMetaImpl.cpp - //record meta access count - prometheus::Family &meta_access_ = prometheus::BuildCounter() - .Name("meta_access_total") - .Help("the number of meta accessing") - .Register(*registry_); - prometheus::Counter &meta_access_total_ = meta_access_.Add({}); + // record meta access count + prometheus::Family& meta_access_ = + prometheus::BuildCounter().Name("meta_access_total").Help("the number of meta accessing").Register(*registry_); + prometheus::Counter& meta_access_total_ = meta_access_.Add({}); - //record meta access duration - prometheus::Family &meta_access_duration_seconds_ = prometheus::BuildHistogram() - .Name("meta_access_duration_microseconds") - .Help("histogram of processing time for accessing mata") - .Register(*registry_); - prometheus::Histogram &meta_access_duration_seconds_histogram_ = + // record meta access duration + prometheus::Family& meta_access_duration_seconds_ = + prometheus::BuildHistogram() + .Name("meta_access_duration_microseconds") + .Help("histogram of processing time for accessing mata") + .Register(*registry_); + prometheus::Histogram& meta_access_duration_seconds_histogram_ = meta_access_duration_seconds_.Add({}, BucketBoundaries{100, 300, 500, 700, 900, 2000, 4000, 6000, 8000, 20000}); ////all from FaissExecutionEngine.cpp - //record data loading from disk count, size, duration, IO speed - prometheus::Family &disk_load_duration_second_ = prometheus::BuildHistogram() - .Name("disk_load_duration_microseconds") - .Help("Histogram of processing time for loading data from disk") - .Register(*registry_); - prometheus::Histogram &faiss_disk_load_duration_seconds_histogram_ = + // record data loading from disk count, size, duration, IO speed + prometheus::Family& disk_load_duration_second_ = + prometheus::BuildHistogram() + .Name("disk_load_duration_microseconds") + .Help("Histogram of processing time for loading data from disk") + .Register(*registry_); + prometheus::Histogram& faiss_disk_load_duration_seconds_histogram_ = disk_load_duration_second_.Add({{"DB", "Faiss"}}, BucketBoundaries{2e5, 4e5, 6e5, 8e5}); - prometheus::Family &disk_load_size_bytes_ = prometheus::BuildHistogram() - .Name("disk_load_size_bytes") - .Help("Histogram of data size by bytes for loading data from disk") - .Register(*registry_); - prometheus::Histogram &faiss_disk_load_size_bytes_histogram_ = + prometheus::Family& disk_load_size_bytes_ = + prometheus::BuildHistogram() + .Name("disk_load_size_bytes") + .Help("Histogram of data size by bytes for loading data from disk") + .Register(*registry_); + prometheus::Histogram& faiss_disk_load_size_bytes_histogram_ = disk_load_size_bytes_.Add({{"DB", "Faiss"}}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9}); -// prometheus::Family &disk_load_IO_speed_ = prometheus::BuildHistogram() -// .Name("disk_load_IO_speed_byte_per_sec") -// .Help("Histogram of IO speed for loading data from disk") -// .Register(*registry_); -// prometheus::Histogram &faiss_disk_load_IO_speed_histogram_ = -// disk_load_IO_speed_.Add({{"DB","Faiss"}},BucketBoundaries{1000, 2000, 3000, 4000, 6000, 8000}); + // prometheus::Family &disk_load_IO_speed_ = prometheus::BuildHistogram() + // .Name("disk_load_IO_speed_byte_per_sec") + // .Help("Histogram of IO speed for loading data from disk") + // .Register(*registry_); + // prometheus::Histogram &faiss_disk_load_IO_speed_histogram_ = + // disk_load_IO_speed_.Add({{"DB","Faiss"}},BucketBoundaries{1000, 2000, 3000, 4000, 6000, 8000}); - prometheus::Family &faiss_disk_load_IO_speed_ = prometheus::BuildGauge() - .Name("disk_load_IO_speed_byte_per_microsec") - .Help("disk IO speed ") - .Register(*registry_); - prometheus::Gauge &faiss_disk_load_IO_speed_gauge_ = faiss_disk_load_IO_speed_.Add({{"DB", "Faiss"}}); + prometheus::Family& faiss_disk_load_IO_speed_ = prometheus::BuildGauge() + .Name("disk_load_IO_speed_byte_per_microsec") + .Help("disk IO speed ") + .Register(*registry_); + prometheus::Gauge& faiss_disk_load_IO_speed_gauge_ = faiss_disk_load_IO_speed_.Add({{"DB", "Faiss"}}); ////all from CacheMgr.cpp - //record cache access count - prometheus::Family &cache_access_ = prometheus::BuildCounter() - .Name("cache_access_total") - .Help("the count of accessing cache ") - .Register(*registry_); - prometheus::Counter &cache_access_total_ = cache_access_.Add({}); + // record cache access count + prometheus::Family& cache_access_ = prometheus::BuildCounter() + .Name("cache_access_total") + .Help("the count of accessing cache ") + .Register(*registry_); + prometheus::Counter& cache_access_total_ = cache_access_.Add({}); // record CPU cache usage and % - prometheus::Family &cpu_cache_usage_ = prometheus::BuildGauge() - .Name("cache_usage_bytes") - .Help("current cache usage by bytes") - .Register(*registry_); - prometheus::Gauge &cpu_cache_usage_gauge_ = cpu_cache_usage_.Add({}); + prometheus::Family& cpu_cache_usage_ = + prometheus::BuildGauge().Name("cache_usage_bytes").Help("current cache usage by bytes").Register(*registry_); + prometheus::Gauge& cpu_cache_usage_gauge_ = cpu_cache_usage_.Add({}); - //record GPU cache usage and % - prometheus::Family &gpu_cache_usage_ = prometheus::BuildGauge() - .Name("gpu_cache_usage_bytes") - .Help("current gpu cache usage by bytes") - .Register(*registry_); + // record GPU cache usage and % + prometheus::Family& gpu_cache_usage_ = prometheus::BuildGauge() + .Name("gpu_cache_usage_bytes") + .Help("current gpu cache usage by bytes") + .Register(*registry_); // record query response using Quantiles = std::vector; - prometheus::Family &query_response_ = prometheus::BuildSummary() - .Name("query_response_summary") - .Help("query response summary") - .Register(*registry_); - prometheus::Summary - &query_response_summary_ = query_response_.Add({}, Quantiles{{0.95, 0.00}, {0.9, 0.05}, {0.8, 0.1}}); + prometheus::Family& query_response_ = + prometheus::BuildSummary().Name("query_response_summary").Help("query response summary").Register(*registry_); + prometheus::Summary& query_response_summary_ = + query_response_.Add({}, Quantiles{{0.95, 0.00}, {0.9, 0.05}, {0.8, 0.1}}); - prometheus::Family &query_vector_response_ = prometheus::BuildSummary() - .Name("query_vector_response_summary") - .Help("query each vector response summary") - .Register(*registry_); - prometheus::Summary &query_vector_response_summary_ = + prometheus::Family& query_vector_response_ = prometheus::BuildSummary() + .Name("query_vector_response_summary") + .Help("query each vector response summary") + .Register(*registry_); + prometheus::Summary& query_vector_response_summary_ = query_vector_response_.Add({}, Quantiles{{0.95, 0.00}, {0.9, 0.05}, {0.8, 0.1}}); - prometheus::Family &query_vector_response_per_second_ = prometheus::BuildGauge() - .Name("query_vector_response_per_microsecond") - .Help("the number of vectors can be queried every second ") - .Register(*registry_); - prometheus::Gauge &query_vector_response_per_second_gauge_ = query_vector_response_per_second_.Add({}); + prometheus::Family& query_vector_response_per_second_ = + prometheus::BuildGauge() + .Name("query_vector_response_per_microsecond") + .Help("the number of vectors can be queried every second ") + .Register(*registry_); + prometheus::Gauge& query_vector_response_per_second_gauge_ = query_vector_response_per_second_.Add({}); - prometheus::Family &query_response_per_second_ = prometheus::BuildGauge() - .Name("query_response_per_microsecond") - .Help("the number of queries can be processed every microsecond") - .Register(*registry_); - prometheus::Gauge &query_response_per_second_gauge = query_response_per_second_.Add({}); + prometheus::Family& query_response_per_second_ = + prometheus::BuildGauge() + .Name("query_response_per_microsecond") + .Help("the number of queries can be processed every microsecond") + .Register(*registry_); + prometheus::Gauge& query_response_per_second_gauge = query_response_per_second_.Add({}); - prometheus::Family &disk_store_IO_speed_ = prometheus::BuildGauge() - .Name("disk_store_IO_speed_bytes_per_microseconds") - .Help("disk_store_IO_speed") - .Register(*registry_); - prometheus::Gauge &disk_store_IO_speed_gauge_ = disk_store_IO_speed_.Add({}); + prometheus::Family& disk_store_IO_speed_ = + prometheus::BuildGauge() + .Name("disk_store_IO_speed_bytes_per_microseconds") + .Help("disk_store_IO_speed") + .Register(*registry_); + prometheus::Gauge& disk_store_IO_speed_gauge_ = disk_store_IO_speed_.Add({}); - prometheus::Family &data_file_size_ = prometheus::BuildGauge() - .Name("data_file_size_bytes") - .Help("data file size by bytes") - .Register(*registry_); - prometheus::Gauge &data_file_size_gauge_ = data_file_size_.Add({}); + prometheus::Family& data_file_size_ = + prometheus::BuildGauge().Name("data_file_size_bytes").Help("data file size by bytes").Register(*registry_); + prometheus::Gauge& data_file_size_gauge_ = data_file_size_.Add({}); - prometheus::Family &add_vectors_ = prometheus::BuildGauge() - .Name("add_vectors") - .Help("current added vectors") - .Register(*registry_); - prometheus::Gauge &add_vectors_success_gauge_ = add_vectors_.Add({{"outcome", "success"}}); - prometheus::Gauge &add_vectors_fail_gauge_ = add_vectors_.Add({{"outcome", "fail"}}); + prometheus::Family& add_vectors_ = + prometheus::BuildGauge().Name("add_vectors").Help("current added vectors").Register(*registry_); + prometheus::Gauge& add_vectors_success_gauge_ = add_vectors_.Add({{"outcome", "success"}}); + prometheus::Gauge& add_vectors_fail_gauge_ = add_vectors_.Add({{"outcome", "fail"}}); - prometheus::Family &add_vectors_per_second_ = prometheus::BuildGauge() - .Name("add_vectors_throughput_per_microsecond") - .Help("add vectors throughput per microsecond") - .Register(*registry_); - prometheus::Gauge &add_vectors_per_second_gauge_ = add_vectors_per_second_.Add({}); + prometheus::Family& add_vectors_per_second_ = prometheus::BuildGauge() + .Name("add_vectors_throughput_per_microsecond") + .Help("add vectors throughput per microsecond") + .Register(*registry_); + prometheus::Gauge& add_vectors_per_second_gauge_ = add_vectors_per_second_.Add({}); - prometheus::Family &CPU_ = prometheus::BuildGauge() - .Name("CPU_usage_percent") - .Help("CPU usage percent by this this process") - .Register(*registry_); - prometheus::Gauge &CPU_usage_percent_ = CPU_.Add({{"CPU", "avg"}}); + prometheus::Family& CPU_ = prometheus::BuildGauge() + .Name("CPU_usage_percent") + .Help("CPU usage percent by this this process") + .Register(*registry_); + prometheus::Gauge& CPU_usage_percent_ = CPU_.Add({{"CPU", "avg"}}); - prometheus::Family &RAM_ = prometheus::BuildGauge() - .Name("RAM_usage_percent") - .Help("RAM usage percent by this process") - .Register(*registry_); - prometheus::Gauge &RAM_usage_percent_ = RAM_.Add({}); + prometheus::Family& RAM_ = prometheus::BuildGauge() + .Name("RAM_usage_percent") + .Help("RAM usage percent by this process") + .Register(*registry_); + prometheus::Gauge& RAM_usage_percent_ = RAM_.Add({}); - //GPU Usage Percent - prometheus::Family &GPU_percent_ = prometheus::BuildGauge() - .Name("Gpu_usage_percent") - .Help("GPU_usage_percent ") - .Register(*registry_); + // GPU Usage Percent + prometheus::Family& GPU_percent_ = + prometheus::BuildGauge().Name("Gpu_usage_percent").Help("GPU_usage_percent ").Register(*registry_); - //GPU Mempry used - prometheus::Family &GPU_memory_usage_ = prometheus::BuildGauge() - .Name("GPU_memory_usage_total") - .Help("GPU memory usage total ") - .Register(*registry_); + // GPU Mempry used + prometheus::Family& GPU_memory_usage_ = + prometheus::BuildGauge().Name("GPU_memory_usage_total").Help("GPU memory usage total ").Register(*registry_); - prometheus::Family &query_index_type_per_second_ = prometheus::BuildGauge() - .Name("query_index_throughtout_per_microsecond") - .Help("query index throughtout per microsecond") - .Register(*registry_); - prometheus::Gauge - &query_index_IVF_type_per_second_gauge_ = query_index_type_per_second_.Add({{"IndexType", "IVF"}}); - prometheus::Gauge - &query_index_IDMAP_type_per_second_gauge_ = query_index_type_per_second_.Add({{"IndexType", "IDMAP"}}); + prometheus::Family& query_index_type_per_second_ = + prometheus::BuildGauge() + .Name("query_index_throughtout_per_microsecond") + .Help("query index throughtout per microsecond") + .Register(*registry_); + prometheus::Gauge& query_index_IVF_type_per_second_gauge_ = + query_index_type_per_second_.Add({{"IndexType", "IVF"}}); + prometheus::Gauge& query_index_IDMAP_type_per_second_gauge_ = + query_index_type_per_second_.Add({{"IndexType", "IDMAP"}}); - prometheus::Family &connection_ = prometheus::BuildGauge() - .Name("connection_number") - .Help("the number of connections") - .Register(*registry_); - prometheus::Gauge &connection_gauge_ = connection_.Add({}); + prometheus::Family& connection_ = + prometheus::BuildGauge().Name("connection_number").Help("the number of connections").Register(*registry_); + prometheus::Gauge& connection_gauge_ = connection_.Add({}); - prometheus::Family &keeping_alive_ = prometheus::BuildCounter() - .Name("keeping_alive_seconds_total") - .Help("total seconds of the serve alive") - .Register(*registry_); - prometheus::Counter &keeping_alive_counter_ = keeping_alive_.Add({}); + prometheus::Family& keeping_alive_ = prometheus::BuildCounter() + .Name("keeping_alive_seconds_total") + .Help("total seconds of the serve alive") + .Register(*registry_); + prometheus::Counter& keeping_alive_counter_ = keeping_alive_.Add({}); - prometheus::Family &octets_ = prometheus::BuildGauge() - .Name("octets_bytes_per_second") - .Help("octets bytes per second") - .Register(*registry_); - prometheus::Gauge &inoctets_gauge_ = octets_.Add({{"type", "inoctets"}}); - prometheus::Gauge &outoctets_gauge_ = octets_.Add({{"type", "outoctets"}}); + prometheus::Family& octets_ = + prometheus::BuildGauge().Name("octets_bytes_per_second").Help("octets bytes per second").Register(*registry_); + prometheus::Gauge& inoctets_gauge_ = octets_.Add({{"type", "inoctets"}}); + prometheus::Gauge& outoctets_gauge_ = octets_.Add({{"type", "outoctets"}}); - prometheus::Family &GPU_temperature_ = prometheus::BuildGauge() - .Name("GPU_temperature") - .Help("GPU temperature") - .Register(*registry_); + prometheus::Family& GPU_temperature_ = + prometheus::BuildGauge().Name("GPU_temperature").Help("GPU temperature").Register(*registry_); - prometheus::Family &CPU_temperature_ = prometheus::BuildGauge() - .Name("CPU_temperature") - .Help("CPU temperature") - .Register(*registry_); + prometheus::Family& CPU_temperature_ = + prometheus::BuildGauge().Name("CPU_temperature").Help("CPU temperature").Register(*registry_); }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/metrics/SystemInfo.cpp b/cpp/src/metrics/SystemInfo.cpp index 54d0c51943..68c1182726 100644 --- a/cpp/src/metrics/SystemInfo.cpp +++ b/cpp/src/metrics/SystemInfo.cpp @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. - #include "metrics/SystemInfo.h" +#include #include #include -#include #include -#include +#include #include #include @@ -37,7 +36,7 @@ SystemInfo::Init() { initialized_ = true; // initialize CPU information - FILE *file; + FILE* file; struct tms time_sample; char line[128]; last_cpu_ = times(&time_sample); @@ -54,7 +53,7 @@ SystemInfo::Init() { total_ram_ = GetPhysicalMemory(); fclose(file); - //initialize GPU information + // initialize GPU information nvmlReturn_t nvmlresult; nvmlresult = nvmlInit(); if (NVML_SUCCESS != nvmlresult) { @@ -67,7 +66,7 @@ SystemInfo::Init() { return; } - //initialize network traffic information + // initialize network traffic information std::pair in_and_out_octets = Octets(); in_octets_ = in_and_out_octets.first; out_octets_ = in_and_out_octets.second; @@ -75,10 +74,10 @@ SystemInfo::Init() { } uint64_t -SystemInfo::ParseLine(char *line) { +SystemInfo::ParseLine(char* line) { // This assumes that a digit will be found and the line ends in " Kb". int i = strlen(line); - const char *p = line; + const char* p = line; while (*p < '0' || *p > '9') p++; line[i - 3] = '\0'; i = atoi(p); @@ -90,15 +89,15 @@ SystemInfo::GetPhysicalMemory() { struct sysinfo memInfo; sysinfo(&memInfo); uint64_t totalPhysMem = memInfo.totalram; - //Multiply in next statement to avoid int overflow on right hand side... + // Multiply in next statement to avoid int overflow on right hand side... totalPhysMem *= memInfo.mem_unit; return totalPhysMem; } uint64_t SystemInfo::GetProcessUsedMemory() { - //Note: this value is in KB! - FILE *file = fopen("/proc/self/status", "r"); + // Note: this value is in KB! + FILE* file = fopen("/proc/self/status", "r"); constexpr uint64_t line_length = 128; uint64_t result = -1; constexpr uint64_t KB_SIZE = 1024; @@ -118,7 +117,7 @@ SystemInfo::GetProcessUsedMemory() { double SystemInfo::MemoryPercent() { if (!initialized_) Init(); - return (double) (GetProcessUsedMemory() * 100) / (double) total_ram_; + return (double)(GetProcessUsedMemory() * 100) / (double)total_ram_; } std::vector @@ -139,9 +138,9 @@ SystemInfo::CPUCorePercent() { } std::vector -SystemInfo::getTotalCpuTime(std::vector &work_time_array) { +SystemInfo::getTotalCpuTime(std::vector& work_time_array) { std::vector total_time_array; - FILE *file = fopen("/proc/stat", "r"); + FILE* file = fopen("/proc/stat", "r"); if (file == NULL) { perror("Could not open stat file"); return total_time_array; @@ -152,16 +151,15 @@ SystemInfo::getTotalCpuTime(std::vector &work_time_array) { for (int i = 0; i < num_processors_; i++) { char buffer[1024]; - char *ret = fgets(buffer, sizeof(buffer) - 1, file); + char* ret = fgets(buffer, sizeof(buffer) - 1, file); if (ret == NULL) { perror("Could not read stat file"); fclose(file); return total_time_array; } - sscanf(buffer, - "cpu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu", - &user, &nice, &system, &idle, &iowait, &irq, &softirq, &steal, &guest, &guestnice); + sscanf(buffer, "cpu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu", &user, &nice, &system, &idle, + &iowait, &irq, &softirq, &steal, &guest, &guestnice); work_time_array.push_back(user + nice + system); total_time_array.push_back(user + nice + system + idle + iowait + irq + softirq + steal); @@ -179,13 +177,11 @@ SystemInfo::CPUPercent() { double percent; now = times(&time_sample); - if (now <= last_cpu_ || time_sample.tms_stime < last_sys_cpu_ || - time_sample.tms_utime < last_user_cpu_) { - //Overflow detection. Just skip this value. + if (now <= last_cpu_ || time_sample.tms_stime < last_sys_cpu_ || time_sample.tms_utime < last_user_cpu_) { + // Overflow detection. Just skip this value. percent = -1.0; } else { - percent = (time_sample.tms_stime - last_sys_cpu_) + - (time_sample.tms_utime - last_user_cpu_); + percent = (time_sample.tms_stime - last_sys_cpu_) + (time_sample.tms_utime - last_user_cpu_); percent /= (now - last_cpu_); percent *= 100; } @@ -230,7 +226,7 @@ SystemInfo::CPUTemperature() { std::vector result; for (int i = 0; i <= num_physical_processors_; ++i) { std::string path = "/sys/class/thermal/thermal_zone" + std::to_string(i) + "/temp"; - FILE *file = fopen(path.data(), "r"); + FILE* file = fopen(path.data(), "r"); if (file == NULL) { perror("Could not open thermal file"); return result; @@ -261,7 +257,7 @@ SystemInfo::GPUMemoryUsed() { std::pair SystemInfo::Octets() { pid_t pid = getpid(); -// const std::string filename = "/proc/"+std::to_string(pid)+"/net/netstat"; + // const std::string filename = "/proc/"+std::to_string(pid)+"/net/netstat"; const std::string filename = "/proc/net/netstat"; std::ifstream file(filename); std::string lastline = ""; @@ -293,6 +289,6 @@ SystemInfo::Octets() { return res; } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/metrics/SystemInfo.h b/cpp/src/metrics/SystemInfo.h index 802cbb0cce..7fda0730e6 100644 --- a/cpp/src/metrics/SystemInfo.h +++ b/cpp/src/metrics/SystemInfo.h @@ -15,20 +15,19 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include +#include +#include +#include #include #include +#include #include -#include -#include -#include #include #include -#include #include +#include namespace zilliz { namespace milvus { @@ -43,72 +42,94 @@ class SystemInfo { std::chrono::system_clock::time_point net_time_ = std::chrono::system_clock::now(); int num_processors_ = 0; int num_physical_processors_ = 0; - //number of GPU + // number of GPU uint32_t num_device_ = 0; uint64_t in_octets_ = 0; uint64_t out_octets_ = 0; bool initialized_ = false; public: - static SystemInfo & + static SystemInfo& GetInstance() { static SystemInfo instance; return instance; } - void Init(); + void + Init(); - int num_processor() const { + int + num_processor() const { return num_processors_; } - int num_physical_processors() const { + int + num_physical_processors() const { return num_physical_processors_; } - uint32_t num_device() const { + uint32_t + num_device() const { return num_device_; } - uint64_t get_inoctets() { + uint64_t + get_inoctets() { return in_octets_; } - uint64_t get_octets() { + uint64_t + get_octets() { return out_octets_; } - std::chrono::system_clock::time_point get_nettime() { + std::chrono::system_clock::time_point + get_nettime() { return net_time_; } - void set_inoctets(uint64_t value) { + void + set_inoctets(uint64_t value) { in_octets_ = value; } - void set_outoctets(uint64_t value) { + void + set_outoctets(uint64_t value) { out_octets_ = value; } - void set_nettime() { + void + set_nettime() { net_time_ = std::chrono::system_clock::now(); } - uint64_t ParseLine(char *line); - uint64_t GetPhysicalMemory(); - uint64_t GetProcessUsedMemory(); - double MemoryPercent(); - double CPUPercent(); - std::pair Octets(); - std::vector GPUMemoryTotal(); - std::vector GPUMemoryUsed(); + uint64_t + ParseLine(char* line); + uint64_t + GetPhysicalMemory(); + uint64_t + GetProcessUsedMemory(); + double + MemoryPercent(); + double + CPUPercent(); + std::pair + Octets(); + std::vector + GPUMemoryTotal(); + std::vector + GPUMemoryUsed(); - std::vector CPUCorePercent(); - std::vector getTotalCpuTime(std::vector &workTime); - std::vector GPUTemperature(); - std::vector CPUTemperature(); + std::vector + CPUCorePercent(); + std::vector + getTotalCpuTime(std::vector& workTime); + std::vector + GPUTemperature(); + std::vector + CPUTemperature(); }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Algorithm.cpp b/cpp/src/scheduler/Algorithm.cpp index 10a585304a..48fee94d9c 100644 --- a/cpp/src/scheduler/Algorithm.cpp +++ b/cpp/src/scheduler/Algorithm.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include "scheduler/Algorithm.h" #include @@ -29,10 +28,8 @@ namespace scheduler { constexpr uint64_t MAXINT = std::numeric_limits::max(); uint64_t -ShortestPath(const ResourcePtr &src, - const ResourcePtr &dest, - const ResourceMgrPtr &res_mgr, - std::vector &path) { +ShortestPath(const ResourcePtr& src, const ResourcePtr& dest, const ResourceMgrPtr& res_mgr, + std::vector& path) { std::vector> paths; uint64_t num_of_resources = res_mgr->GetAllResources().size(); @@ -43,7 +40,7 @@ ShortestPath(const ResourcePtr &src, name_id_map.insert(std::make_pair(res_mgr->GetAllResources().at(i)->name(), i)); } - std::vector > dis_matrix; + std::vector> dis_matrix; dis_matrix.resize(num_of_resources); for (uint64_t i = 0; i < num_of_resources; ++i) { dis_matrix[i].resize(num_of_resources); @@ -55,11 +52,11 @@ ShortestPath(const ResourcePtr &src, std::vector vis(num_of_resources, false); std::vector dis(num_of_resources, MAXINT); - for (auto &res : res_mgr->GetAllResources()) { + for (auto& res : res_mgr->GetAllResources()) { auto cur_node = std::static_pointer_cast(res); auto cur_neighbours = cur_node->GetNeighbours(); - for (auto &neighbour : cur_neighbours) { + for (auto& neighbour : cur_neighbours) { auto neighbour_res = std::static_pointer_cast(neighbour.neighbour_node.lock()); dis_matrix[name_id_map.at(res->name())][name_id_map.at(neighbour_res->name())] = neighbour.connection.transport_cost(); @@ -107,6 +104,6 @@ ShortestPath(const ResourcePtr &src, return dis[name_id_map.at(dest->name())]; } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Algorithm.h b/cpp/src/scheduler/Algorithm.h index d7e0233ba0..ed0f17d762 100644 --- a/cpp/src/scheduler/Algorithm.h +++ b/cpp/src/scheduler/Algorithm.h @@ -15,23 +15,20 @@ // specific language governing permissions and limitations // under the License. - -#include "resource/Resource.h" #include "ResourceMgr.h" +#include "resource/Resource.h" -#include #include +#include namespace zilliz { namespace milvus { namespace scheduler { uint64_t -ShortestPath(const ResourcePtr &src, - const ResourcePtr &dest, - const ResourceMgrPtr &res_mgr, - std::vector &path); +ShortestPath(const ResourcePtr& src, const ResourcePtr& dest, const ResourceMgrPtr& res_mgr, + std::vector& path); -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Definition.h b/cpp/src/scheduler/Definition.h index ce41aca48d..c886aafdac 100644 --- a/cpp/src/scheduler/Definition.h +++ b/cpp/src/scheduler/Definition.h @@ -15,20 +15,20 @@ // specific language governing permissions and limitations // under the License. -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include -#include "db/meta/MetaTypes.h" #include "db/engine/EngineFactory.h" #include "db/engine/ExecutionEngine.h" +#include "db/meta/MetaTypes.h" namespace zilliz { namespace milvus { @@ -42,6 +42,6 @@ using EngineFactory = engine::EngineFactory; using EngineType = engine::EngineType; using MetricType = engine::MetricType; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/JobMgr.cpp b/cpp/src/scheduler/JobMgr.cpp index 0406e98c49..92cf1189fd 100644 --- a/cpp/src/scheduler/JobMgr.cpp +++ b/cpp/src/scheduler/JobMgr.cpp @@ -16,8 +16,8 @@ // under the License. #include "scheduler/JobMgr.h" -#include "task/Task.h" #include "TaskCreator.h" +#include "task/Task.h" #include @@ -25,8 +25,7 @@ namespace zilliz { namespace milvus { namespace scheduler { -JobMgr::JobMgr(ResourceMgrPtr res_mgr) - : res_mgr_(std::move(res_mgr)) { +JobMgr::JobMgr(ResourceMgrPtr res_mgr) : res_mgr_(std::move(res_mgr)) { } void @@ -47,7 +46,7 @@ JobMgr::Stop() { } void -JobMgr::Put(const JobPtr &job) { +JobMgr::Put(const JobPtr& job) { { std::lock_guard lock(mutex_); queue_.push(job); @@ -59,9 +58,7 @@ void JobMgr::worker_function() { while (running_) { std::unique_lock lock(mutex_); - cv_.wait(lock, [this] { - return !queue_.empty(); - }); + cv_.wait(lock, [this] { return !queue_.empty(); }); auto job = queue_.front(); queue_.pop(); lock.unlock(); @@ -73,7 +70,7 @@ JobMgr::worker_function() { auto disk_list = res_mgr_->GetDiskResources(); if (!disk_list.empty()) { if (auto disk = disk_list[0].lock()) { - for (auto &task : tasks) { + for (auto& task : tasks) { disk->task_table().Put(task); } } @@ -82,10 +79,10 @@ JobMgr::worker_function() { } std::vector -JobMgr::build_task(const JobPtr &job) { +JobMgr::build_task(const JobPtr& job) { return TaskCreator::Create(job); } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/JobMgr.h b/cpp/src/scheduler/JobMgr.h index 49ba9154e3..66adff18d2 100644 --- a/cpp/src/scheduler/JobMgr.h +++ b/cpp/src/scheduler/JobMgr.h @@ -16,20 +16,20 @@ // under the License. #pragma once -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include "ResourceMgr.h" #include "job/Job.h" #include "task/Task.h" -#include "ResourceMgr.h" namespace zilliz { namespace milvus { @@ -47,14 +47,14 @@ class JobMgr { public: void - Put(const JobPtr &job); + Put(const JobPtr& job); private: void worker_function(); std::vector - build_task(const JobPtr &job); + build_task(const JobPtr& job); private: bool running_ = false; @@ -70,6 +70,6 @@ class JobMgr { using JobMgrPtr = std::shared_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/ResourceFactory.cpp b/cpp/src/scheduler/ResourceFactory.cpp index de9b5bc717..f89e8207e4 100644 --- a/cpp/src/scheduler/ResourceFactory.cpp +++ b/cpp/src/scheduler/ResourceFactory.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include "scheduler/ResourceFactory.h" namespace zilliz { @@ -23,10 +22,7 @@ namespace milvus { namespace scheduler { std::shared_ptr -ResourceFactory::Create(const std::string &name, - const std::string &type, - uint64_t device_id, - bool enable_loader, +ResourceFactory::Create(const std::string& name, const std::string& type, uint64_t device_id, bool enable_loader, bool enable_executor) { if (type == "DISK") { return std::make_shared(name, device_id, enable_loader, enable_executor); @@ -39,6 +35,6 @@ ResourceFactory::Create(const std::string &name, } } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/ResourceFactory.h b/cpp/src/scheduler/ResourceFactory.h index f7a47ef1e5..bf663a7db8 100644 --- a/cpp/src/scheduler/ResourceFactory.h +++ b/cpp/src/scheduler/ResourceFactory.h @@ -17,13 +17,13 @@ #pragma once -#include #include +#include -#include "resource/Resource.h" #include "resource/CpuResource.h" -#include "resource/GpuResource.h" #include "resource/DiskResource.h" +#include "resource/GpuResource.h" +#include "resource/Resource.h" namespace zilliz { namespace milvus { @@ -32,13 +32,10 @@ namespace scheduler { class ResourceFactory { public: static std::shared_ptr - Create(const std::string &name, - const std::string &type, - uint64_t device_id, - bool enable_loader = true, + Create(const std::string& name, const std::string& type, uint64_t device_id, bool enable_loader = true, bool enable_executor = true); }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/ResourceMgr.cpp b/cpp/src/scheduler/ResourceMgr.cpp index 6067b2eb01..eefa30560c 100644 --- a/cpp/src/scheduler/ResourceMgr.cpp +++ b/cpp/src/scheduler/ResourceMgr.cpp @@ -26,7 +26,7 @@ namespace scheduler { void ResourceMgr::Start() { std::lock_guard lck(resources_mutex_); - for (auto &resource : resources_) { + for (auto& resource : resources_) { resource->Start(); } running_ = true; @@ -44,13 +44,13 @@ ResourceMgr::Stop() { worker_thread_.join(); std::lock_guard lck(resources_mutex_); - for (auto &resource : resources_) { + for (auto& resource : resources_) { resource->Stop(); } } ResourceWPtr -ResourceMgr::Add(ResourcePtr &&resource) { +ResourceMgr::Add(ResourcePtr&& resource) { ResourceWPtr ret(resource); std::lock_guard lck(resources_mutex_); @@ -70,13 +70,13 @@ ResourceMgr::Add(ResourcePtr &&resource) { } bool -ResourceMgr::Connect(const std::string &name1, const std::string &name2, Connection &connection) { +ResourceMgr::Connect(const std::string& name1, const std::string& name2, Connection& connection) { auto res1 = GetResource(name1); auto res2 = GetResource(name2); if (res1 && res2) { res1->AddNeighbour(std::static_pointer_cast(res2), connection); // TODO: enable when task balance supported -// res2->AddNeighbour(std::static_pointer_cast(res1), connection); + // res2->AddNeighbour(std::static_pointer_cast(res1), connection); return true; } return false; @@ -92,7 +92,7 @@ ResourceMgr::Clear() { std::vector ResourceMgr::GetComputeResources() { std::vector result; - for (auto &resource : resources_) { + for (auto& resource : resources_) { if (resource->HasExecutor()) { result.emplace_back(resource); } @@ -102,7 +102,7 @@ ResourceMgr::GetComputeResources() { ResourcePtr ResourceMgr::GetResource(ResourceType type, uint64_t device_id) { - for (auto &resource : resources_) { + for (auto& resource : resources_) { if (resource->type() == type && resource->device_id() == device_id) { return resource; } @@ -111,8 +111,8 @@ ResourceMgr::GetResource(ResourceType type, uint64_t device_id) { } ResourcePtr -ResourceMgr::GetResource(const std::string &name) { - for (auto &resource : resources_) { +ResourceMgr::GetResource(const std::string& name) { + for (auto& resource : resources_) { if (resource->name() == name) { return resource; } @@ -128,7 +128,7 @@ ResourceMgr::GetNumOfResource() const { uint64_t ResourceMgr::GetNumOfComputeResource() const { uint64_t count = 0; - for (auto &res : resources_) { + for (auto& res : resources_) { if (res->HasExecutor()) { ++count; } @@ -139,7 +139,7 @@ ResourceMgr::GetNumOfComputeResource() const { uint64_t ResourceMgr::GetNumGpuResource() const { uint64_t num = 0; - for (auto &res : resources_) { + for (auto& res : resources_) { if (res->type() == ResourceType::GPU) { num++; } @@ -153,7 +153,7 @@ ResourceMgr::Dump() { for (uint64_t i = 0; i < resources_.size(); ++i) { str += "Resource No." + std::to_string(i) + ":\n"; - //str += resources_[i]->Dump(); + // str += resources_[i]->Dump(); } return str; @@ -163,7 +163,7 @@ std::string ResourceMgr::DumpTaskTables() { std::stringstream ss; ss << ">>>>>>>>>>>>>>>ResourceMgr::DumpTaskTable<<<<<<<<<<<<<<<" << std::endl; - for (auto &resource : resources_) { + for (auto& resource : resources_) { ss << resource->Dump() << std::endl; ss << resource->task_table().Dump(); ss << resource->Dump() << std::endl << std::endl; @@ -172,7 +172,7 @@ ResourceMgr::DumpTaskTables() { } void -ResourceMgr::post_event(const EventPtr &event) { +ResourceMgr::post_event(const EventPtr& event) { { std::lock_guard lock(event_mutex_); queue_.emplace(event); @@ -184,9 +184,7 @@ void ResourceMgr::event_process() { while (running_) { std::unique_lock lock(event_mutex_); - event_cv_.wait(lock, [this] { - return !queue_.empty(); - }); + event_cv_.wait(lock, [this] { return !queue_.empty(); }); auto event = queue_.front(); queue_.pop(); @@ -201,6 +199,6 @@ ResourceMgr::event_process() { } } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/ResourceMgr.h b/cpp/src/scheduler/ResourceMgr.h index d03408a7df..b6720a2054 100644 --- a/cpp/src/scheduler/ResourceMgr.h +++ b/cpp/src/scheduler/ResourceMgr.h @@ -17,13 +17,13 @@ #pragma once -#include -#include +#include #include #include #include +#include #include -#include +#include #include "resource/Resource.h" #include "utils/Log.h" @@ -45,10 +45,10 @@ class ResourceMgr { Stop(); ResourceWPtr - Add(ResourcePtr &&resource); + Add(ResourcePtr&& resource); bool - Connect(const std::string &res1, const std::string &res2, Connection &connection); + Connect(const std::string& res1, const std::string& res2, Connection& connection); void Clear(); @@ -60,7 +60,7 @@ class ResourceMgr { public: /******** Management Interface ********/ - inline std::vector & + inline std::vector& GetDiskResources() { return disk_resources_; } @@ -78,7 +78,7 @@ class ResourceMgr { GetResource(ResourceType type, uint64_t device_id); ResourcePtr - GetResource(const std::string &name); + GetResource(const std::string& name); uint64_t GetNumOfResource() const; @@ -102,7 +102,7 @@ class ResourceMgr { private: void - post_event(const EventPtr &event); + post_event(const EventPtr& event); void event_process(); @@ -125,6 +125,6 @@ class ResourceMgr { using ResourceMgrPtr = std::shared_ptr; using ResourceMgrWPtr = std::weak_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index 71b40de9ee..5c04eeb931 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -15,17 +15,16 @@ // specific language governing permissions and limitations // under the License. - #include "scheduler/SchedInst.h" -#include "server/Config.h" #include "ResourceFactory.h" -#include "knowhere/index/vector_index/IndexGPUIVF.h" #include "Utils.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "server/Config.h" -#include #include -#include #include +#include +#include namespace zilliz { namespace milvus { @@ -42,7 +41,7 @@ std::mutex JobMgrInst::mutex_; void load_simple_config() { - server::Config &config = server::Config::GetInstance(); + server::Config& config = server::Config::GetInstance(); std::string mode; config.GetResourceConfigMode(mode); std::vector pool; @@ -50,7 +49,7 @@ load_simple_config() { bool cpu = false; std::set gpu_ids; - for (auto &resource : pool) { + for (auto& resource : pool) { if (resource == "cpu") { cpu = true; break; @@ -78,7 +77,7 @@ load_simple_config() { ResMgrInst::GetInstance()->Connect("disk", "cpu", io); auto pcie = Connection("pcie", 12000); - for (auto &gpu_id : gpu_ids) { + for (auto& gpu_id : gpu_ids) { ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true)); ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), io); } @@ -87,77 +86,77 @@ load_simple_config() { void load_advance_config() { -// try { -// server::ConfigNode &config = server::Config::GetInstance().GetConfig(server::CONFIG_RESOURCE); -// -// if (config.GetChildren().empty()) throw "resource_config null exception"; -// -// auto resources = config.GetChild(server::CONFIG_RESOURCES).GetChildren(); -// -// if (resources.empty()) throw "Children of resource_config null exception"; -// -// for (auto &resource : resources) { -// auto &resname = resource.first; -// auto &resconf = resource.second; -// auto type = resconf.GetValue(server::CONFIG_RESOURCE_TYPE); -//// auto memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_MEMORY); -// auto device_id = resconf.GetInt64Value(server::CONFIG_RESOURCE_DEVICE_ID); -//// auto enable_loader = resconf.GetBoolValue(server::CONFIG_RESOURCE_ENABLE_LOADER); -// auto enable_loader = true; -// auto enable_executor = resconf.GetBoolValue(server::CONFIG_RESOURCE_ENABLE_EXECUTOR); -// auto pinned_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_PIN_MEMORY); -// auto temp_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_TEMP_MEMORY); -// auto resource_num = resconf.GetInt64Value(server::CONFIG_RESOURCE_NUM); -// -// auto res = ResMgrInst::GetInstance()->Add(ResourceFactory::Create(resname, -// type, -// device_id, -// enable_loader, -// enable_executor)); -// -// if (res.lock()->type() == ResourceType::GPU) { -// auto pinned_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_PIN_MEMORY, 300); -// auto temp_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_TEMP_MEMORY, 300); -// auto resource_num = resconf.GetInt64Value(server::CONFIG_RESOURCE_NUM, 2); -// pinned_memory = 1024 * 1024 * pinned_memory; -// temp_memory = 1024 * 1024 * temp_memory; -// knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(device_id, -// pinned_memory, -// temp_memory, -// resource_num); -// } -// } -// -// knowhere::FaissGpuResourceMgr::GetInstance().InitResource(); -// -// auto connections = config.GetChild(server::CONFIG_RESOURCE_CONNECTIONS).GetChildren(); -// if (connections.empty()) throw "connections config null exception"; -// for (auto &conn : connections) { -// auto &connect_name = conn.first; -// auto &connect_conf = conn.second; -// auto connect_speed = connect_conf.GetInt64Value(server::CONFIG_SPEED_CONNECTIONS); -// auto connect_endpoint = connect_conf.GetValue(server::CONFIG_ENDPOINT_CONNECTIONS); -// -// std::string delimiter = "==="; -// std::string left = connect_endpoint.substr(0, connect_endpoint.find(delimiter)); -// std::string right = connect_endpoint.substr(connect_endpoint.find(delimiter) + 3, -// connect_endpoint.length()); -// -// auto connection = Connection(connect_name, connect_speed); -// ResMgrInst::GetInstance()->Connect(left, right, connection); -// } -// } catch (const char *msg) { -// SERVER_LOG_ERROR << msg; -// // TODO: throw exception instead -// exit(-1); -//// throw std::exception(); -// } + // try { + // server::ConfigNode &config = server::Config::GetInstance().GetConfig(server::CONFIG_RESOURCE); + // + // if (config.GetChildren().empty()) throw "resource_config null exception"; + // + // auto resources = config.GetChild(server::CONFIG_RESOURCES).GetChildren(); + // + // if (resources.empty()) throw "Children of resource_config null exception"; + // + // for (auto &resource : resources) { + // auto &resname = resource.first; + // auto &resconf = resource.second; + // auto type = resconf.GetValue(server::CONFIG_RESOURCE_TYPE); + //// auto memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_MEMORY); + // auto device_id = resconf.GetInt64Value(server::CONFIG_RESOURCE_DEVICE_ID); + //// auto enable_loader = resconf.GetBoolValue(server::CONFIG_RESOURCE_ENABLE_LOADER); + // auto enable_loader = true; + // auto enable_executor = resconf.GetBoolValue(server::CONFIG_RESOURCE_ENABLE_EXECUTOR); + // auto pinned_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_PIN_MEMORY); + // auto temp_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_TEMP_MEMORY); + // auto resource_num = resconf.GetInt64Value(server::CONFIG_RESOURCE_NUM); + // + // auto res = ResMgrInst::GetInstance()->Add(ResourceFactory::Create(resname, + // type, + // device_id, + // enable_loader, + // enable_executor)); + // + // if (res.lock()->type() == ResourceType::GPU) { + // auto pinned_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_PIN_MEMORY, 300); + // auto temp_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_TEMP_MEMORY, 300); + // auto resource_num = resconf.GetInt64Value(server::CONFIG_RESOURCE_NUM, 2); + // pinned_memory = 1024 * 1024 * pinned_memory; + // temp_memory = 1024 * 1024 * temp_memory; + // knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(device_id, + // pinned_memory, + // temp_memory, + // resource_num); + // } + // } + // + // knowhere::FaissGpuResourceMgr::GetInstance().InitResource(); + // + // auto connections = config.GetChild(server::CONFIG_RESOURCE_CONNECTIONS).GetChildren(); + // if (connections.empty()) throw "connections config null exception"; + // for (auto &conn : connections) { + // auto &connect_name = conn.first; + // auto &connect_conf = conn.second; + // auto connect_speed = connect_conf.GetInt64Value(server::CONFIG_SPEED_CONNECTIONS); + // auto connect_endpoint = connect_conf.GetValue(server::CONFIG_ENDPOINT_CONNECTIONS); + // + // std::string delimiter = "==="; + // std::string left = connect_endpoint.substr(0, connect_endpoint.find(delimiter)); + // std::string right = connect_endpoint.substr(connect_endpoint.find(delimiter) + 3, + // connect_endpoint.length()); + // + // auto connection = Connection(connect_name, connect_speed); + // ResMgrInst::GetInstance()->Connect(left, right, connection); + // } + // } catch (const char *msg) { + // SERVER_LOG_ERROR << msg; + // // TODO: throw exception instead + // exit(-1); + //// throw std::exception(); + // } } void StartSchedulerService() { load_simple_config(); -// load_advance_config(); + // load_advance_config(); ResMgrInst::GetInstance()->Start(); SchedInst::GetInstance()->Start(); JobMgrInst::GetInstance()->Start(); @@ -170,6 +169,6 @@ StopSchedulerService() { ResMgrInst::GetInstance()->Stop(); } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/SchedInst.h b/cpp/src/scheduler/SchedInst.h index 4cca6ec5a9..9947e9f62b 100644 --- a/cpp/src/scheduler/SchedInst.h +++ b/cpp/src/scheduler/SchedInst.h @@ -17,12 +17,12 @@ #pragma once +#include "JobMgr.h" #include "ResourceMgr.h" #include "Scheduler.h" -#include "JobMgr.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -88,6 +88,6 @@ StartSchedulerService(); void StopSchedulerService(); -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Scheduler.cpp b/cpp/src/scheduler/Scheduler.cpp index 24f7bfe73b..5cffa99968 100644 --- a/cpp/src/scheduler/Scheduler.cpp +++ b/cpp/src/scheduler/Scheduler.cpp @@ -16,10 +16,10 @@ // under the License. #include "scheduler/Scheduler.h" +#include "Algorithm.h" +#include "action/Action.h" #include "cache/GpuCacheMgr.h" #include "event/LoadCompletedEvent.h" -#include "action/Action.h" -#include "Algorithm.h" #include @@ -27,9 +27,7 @@ namespace zilliz { namespace milvus { namespace scheduler { -Scheduler::Scheduler(ResourceMgrWPtr res_mgr) - : running_(false), - res_mgr_(std::move(res_mgr)) { +Scheduler::Scheduler(ResourceMgrWPtr res_mgr) : running_(false), res_mgr_(std::move(res_mgr)) { if (auto mgr = res_mgr_.lock()) { mgr->RegisterSubscriber(std::bind(&Scheduler::PostEvent, this, std::placeholders::_1)); } @@ -61,7 +59,7 @@ Scheduler::Stop() { } void -Scheduler::PostEvent(const EventPtr &event) { +Scheduler::PostEvent(const EventPtr& event) { { std::lock_guard lock(event_mutex_); event_queue_.push(event); @@ -78,9 +76,7 @@ void Scheduler::worker_function() { while (running_) { std::unique_lock lock(event_mutex_); - event_cv_.wait(lock, [this] { - return !event_queue_.empty(); - }); + event_cv_.wait(lock, [this] { return !event_queue_.empty(); }); auto event = event_queue_.front(); event_queue_.pop(); if (event == nullptr) { @@ -92,14 +88,14 @@ Scheduler::worker_function() { } void -Scheduler::Process(const EventPtr &event) { +Scheduler::Process(const EventPtr& event) { auto process_event = event_register_.at(static_cast(event->Type())); process_event(event); } // TODO: refactor the function void -Scheduler::OnLoadCompleted(const EventPtr &event) { +Scheduler::OnLoadCompleted(const EventPtr& event) { auto load_completed_event = std::static_pointer_cast(event); if (auto resource = event->resource_.lock()) { resource->WakeupExecutor(); @@ -118,31 +114,29 @@ Scheduler::OnLoadCompleted(const EventPtr &event) { Action::PushTaskToAllNeighbour(load_completed_event->task_table_item_->task, resource); break; } - default: { - break; - } + default: { break; } } } } void -Scheduler::OnStartUp(const EventPtr &event) { +Scheduler::OnStartUp(const EventPtr& event) { if (auto resource = event->resource_.lock()) { resource->WakeupLoader(); } } void -Scheduler::OnFinishTask(const EventPtr &event) { +Scheduler::OnFinishTask(const EventPtr& event) { } void -Scheduler::OnTaskTableUpdated(const EventPtr &event) { +Scheduler::OnTaskTableUpdated(const EventPtr& event) { if (auto resource = event->resource_.lock()) { resource->WakeupLoader(); } } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Scheduler.h b/cpp/src/scheduler/Scheduler.h index 073f7eeb0d..e72e40d1ba 100644 --- a/cpp/src/scheduler/Scheduler.h +++ b/cpp/src/scheduler/Scheduler.h @@ -18,14 +18,14 @@ #pragma once #include -#include #include -#include #include +#include +#include #include -#include "resource/Resource.h" #include "ResourceMgr.h" +#include "resource/Resource.h" #include "utils/Log.h" namespace zilliz { @@ -37,8 +37,8 @@ class Scheduler { public: explicit Scheduler(ResourceMgrWPtr res_mgr); - Scheduler(const Scheduler &) = delete; - Scheduler(Scheduler &&) = delete; + Scheduler(const Scheduler&) = delete; + Scheduler(Scheduler&&) = delete; /* * Start worker thread; @@ -56,7 +56,7 @@ class Scheduler { * Post event to scheduler event queue; */ void - PostEvent(const EventPtr &event); + PostEvent(const EventPtr& event); /* * Dump as string; @@ -74,7 +74,7 @@ class Scheduler { * Pull task from neighbours; */ void - OnStartUp(const EventPtr &event); + OnStartUp(const EventPtr& event); /* * Process finish task events; @@ -83,7 +83,7 @@ class Scheduler { * Pull task from neighbours; */ void - OnFinishTask(const EventPtr &event); + OnFinishTask(const EventPtr& event); /* * Process copy completed events; @@ -93,7 +93,7 @@ class Scheduler { * Pull task from neighbours; */ void - OnLoadCompleted(const EventPtr &event); + OnLoadCompleted(const EventPtr& event); /* * Process task table updated events, which happened on task_table->put; @@ -102,14 +102,14 @@ class Scheduler { * Push task to neighbours; */ void - OnTaskTableUpdated(const EventPtr &event); + OnTaskTableUpdated(const EventPtr& event); private: /* * Dispatch event to event handler; */ void - Process(const EventPtr &event); + Process(const EventPtr& event); /* * Called by worker_thread_; @@ -131,6 +131,6 @@ class Scheduler { using SchedulerPtr = std::shared_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/TaskCreator.cpp b/cpp/src/scheduler/TaskCreator.cpp index 8b0378c646..4f8979703f 100644 --- a/cpp/src/scheduler/TaskCreator.cpp +++ b/cpp/src/scheduler/TaskCreator.cpp @@ -24,7 +24,7 @@ namespace milvus { namespace scheduler { std::vector -TaskCreator::Create(const JobPtr &job) { +TaskCreator::Create(const JobPtr& job) { switch (job->type()) { case JobType::SEARCH: { return Create(std::static_pointer_cast(job)); @@ -40,9 +40,9 @@ TaskCreator::Create(const JobPtr &job) { } std::vector -TaskCreator::Create(const SearchJobPtr &job) { +TaskCreator::Create(const SearchJobPtr& job) { std::vector tasks; - for (auto &index_file : job->index_files()) { + for (auto& index_file : job->index_files()) { auto task = std::make_shared(index_file.second); task->label() = std::make_shared(); task->job_ = job; @@ -53,7 +53,7 @@ TaskCreator::Create(const SearchJobPtr &job) { } std::vector -TaskCreator::Create(const DeleteJobPtr &job) { +TaskCreator::Create(const DeleteJobPtr& job) { std::vector tasks; auto task = std::make_shared(job); task->label() = std::make_shared(); @@ -63,6 +63,6 @@ TaskCreator::Create(const DeleteJobPtr &job) { return tasks; } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/TaskCreator.h b/cpp/src/scheduler/TaskCreator.h index 81cb25010f..5ae6a0763a 100644 --- a/cpp/src/scheduler/TaskCreator.h +++ b/cpp/src/scheduler/TaskCreator.h @@ -16,23 +16,23 @@ // under the License. #pragma once -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include "job/DeleteJob.h" #include "job/Job.h" #include "job/SearchJob.h" -#include "job/DeleteJob.h" -#include "task/Task.h" -#include "task/SearchTask.h" #include "task/DeleteTask.h" +#include "task/SearchTask.h" +#include "task/Task.h" namespace zilliz { namespace milvus { @@ -41,16 +41,16 @@ namespace scheduler { class TaskCreator { public: static std::vector - Create(const JobPtr &job); + Create(const JobPtr& job); public: static std::vector - Create(const SearchJobPtr &job); + Create(const SearchJobPtr& job); static std::vector - Create(const DeleteJobPtr &job); + Create(const DeleteJobPtr& job); }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/TaskTable.cpp b/cpp/src/scheduler/TaskTable.cpp index a7343ee509..db06abd966 100644 --- a/cpp/src/scheduler/TaskTable.cpp +++ b/cpp/src/scheduler/TaskTable.cpp @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. - #include "scheduler/TaskTable.h" -#include "event/TaskTableUpdatedEvent.h" #include "Utils.h" +#include "event/TaskTableUpdatedEvent.h" -#include -#include #include +#include +#include namespace zilliz { namespace milvus { @@ -31,20 +30,29 @@ namespace scheduler { std::string ToString(TaskTableItemState state) { switch (state) { - case TaskTableItemState::INVALID: return "INVALID"; - case TaskTableItemState::START: return "START"; - case TaskTableItemState::LOADING: return "LOADING"; - case TaskTableItemState::LOADED: return "LOADED"; - case TaskTableItemState::EXECUTING: return "EXECUTING"; - case TaskTableItemState::EXECUTED: return "EXECUTED"; - case TaskTableItemState::MOVING: return "MOVING"; - case TaskTableItemState::MOVED: return "MOVED"; - default: return ""; + case TaskTableItemState::INVALID: + return "INVALID"; + case TaskTableItemState::START: + return "START"; + case TaskTableItemState::LOADING: + return "LOADING"; + case TaskTableItemState::LOADED: + return "LOADED"; + case TaskTableItemState::EXECUTING: + return "EXECUTING"; + case TaskTableItemState::EXECUTED: + return "EXECUTED"; + case TaskTableItemState::MOVING: + return "MOVING"; + case TaskTableItemState::MOVED: + return "MOVED"; + default: + return ""; } } std::string -ToString(const TaskTimestamp ×tamp) { +ToString(const TaskTimestamp& timestamp) { std::stringstream ss; ss << " subscriber) { @@ -117,7 +117,7 @@ class TaskTable { * Called by DBImpl; */ void - Put(std::vector &tasks); + Put(std::vector& tasks); /* * Return task table item reference; @@ -130,8 +130,8 @@ class TaskTable { * Remove sequence task which is DONE or MOVED from front; * Called by ? */ -// void -// Clear(); + // void + // Clear(); /* * Return true if task table empty, otherwise false; @@ -150,16 +150,17 @@ class TaskTable { } public: - TaskTableItemPtr & - operator[](uint64_t index) { + TaskTableItemPtr& operator[](uint64_t index) { return table_[index]; } - std::deque::iterator begin() { + std::deque::iterator + begin() { return table_.begin(); } - std::deque::iterator end() { + std::deque::iterator + end() { return table_.end(); } @@ -254,6 +255,6 @@ class TaskTable { uint64_t last_finish_ = -1; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Utils.cpp b/cpp/src/scheduler/Utils.cpp index bb19950ffe..b50fe4adb9 100644 --- a/cpp/src/scheduler/Utils.cpp +++ b/cpp/src/scheduler/Utils.cpp @@ -15,11 +15,10 @@ // specific language governing permissions and limitations // under the License. - #include "scheduler/Utils.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -40,6 +39,6 @@ get_num_gpu() { return n_devices; } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/Utils.h b/cpp/src/scheduler/Utils.h index c69028f0fa..eec5f8ba68 100644 --- a/cpp/src/scheduler/Utils.h +++ b/cpp/src/scheduler/Utils.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include namespace zilliz { @@ -28,6 +27,6 @@ get_current_timestamp(); uint64_t get_num_gpu(); -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/action/Action.h b/cpp/src/scheduler/action/Action.h index a5f67aa98d..b75dc4515f 100644 --- a/cpp/src/scheduler/action/Action.h +++ b/cpp/src/scheduler/action/Action.h @@ -17,8 +17,8 @@ #pragma once -#include "scheduler/resource/Resource.h" #include "scheduler/ResourceMgr.h" +#include "scheduler/resource/Resource.h" #include @@ -29,23 +29,22 @@ namespace scheduler { class Action { public: static void - PushTaskToNeighbourRandomly(const TaskPtr &task, const ResourcePtr &self); + PushTaskToNeighbourRandomly(const TaskPtr& task, const ResourcePtr& self); static void - PushTaskToAllNeighbour(const TaskPtr &task, const ResourcePtr &self); + PushTaskToAllNeighbour(const TaskPtr& task, const ResourcePtr& self); static void - PushTaskToResource(const TaskPtr &task, const ResourcePtr &dest); + PushTaskToResource(const TaskPtr& task, const ResourcePtr& dest); static void DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr resource, std::shared_ptr event); static void - SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, - ResourcePtr resource, + SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr resource, std::shared_ptr event); }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp index 909112eb62..c7514b1e2d 100644 --- a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp @@ -15,26 +15,25 @@ // specific language governing permissions and limitations // under the License. - #include #include #include "../Algorithm.h" -#include "src/cache/GpuCacheMgr.h" #include "Action.h" +#include "src/cache/GpuCacheMgr.h" namespace zilliz { namespace milvus { namespace scheduler { std::vector -get_neighbours(const ResourcePtr &self) { +get_neighbours(const ResourcePtr& self) { std::vector neighbours; - for (auto &neighbour_node : self->GetNeighbours()) { + for (auto& neighbour_node : self->GetNeighbours()) { auto node = neighbour_node.neighbour_node.lock(); if (not node) continue; auto resource = std::static_pointer_cast(node); -// if (not resource->HasExecutor()) continue; + // if (not resource->HasExecutor()) continue; neighbours.emplace_back(resource); } @@ -42,14 +41,14 @@ get_neighbours(const ResourcePtr &self) { } std::vector> -get_neighbours_with_connetion(const ResourcePtr &self) { +get_neighbours_with_connetion(const ResourcePtr& self) { std::vector> neighbours; - for (auto &neighbour_node : self->GetNeighbours()) { + for (auto& neighbour_node : self->GetNeighbours()) { auto node = neighbour_node.neighbour_node.lock(); if (not node) continue; auto resource = std::static_pointer_cast(node); -// if (not resource->HasExecutor()) continue; + // if (not resource->HasExecutor()) continue; Connection conn = neighbour_node.connection; neighbours.emplace_back(std::make_pair(resource, conn)); } @@ -57,13 +56,12 @@ get_neighbours_with_connetion(const ResourcePtr &self) { } void -Action::PushTaskToNeighbourRandomly(const TaskPtr &task, - const ResourcePtr &self) { +Action::PushTaskToNeighbourRandomly(const TaskPtr& task, const ResourcePtr& self) { auto neighbours = get_neighbours_with_connetion(self); if (not neighbours.empty()) { std::vector speeds; uint64_t total_speed = 0; - for (auto &neighbour : neighbours) { + for (auto& neighbour : neighbours) { uint64_t speed = neighbour.second.speed(); speeds.emplace_back(speed); total_speed += speed; @@ -83,33 +81,32 @@ Action::PushTaskToNeighbourRandomly(const TaskPtr &task, } } else { - //TODO: process + // TODO: process } } void -Action::PushTaskToAllNeighbour(const TaskPtr &task, const ResourcePtr &self) { +Action::PushTaskToAllNeighbour(const TaskPtr& task, const ResourcePtr& self) { auto neighbours = get_neighbours(self); - for (auto &neighbour : neighbours) { + for (auto& neighbour : neighbours) { neighbour->task_table().Put(task); } } void -Action::PushTaskToResource(const TaskPtr &task, const ResourcePtr &dest) { +Action::PushTaskToResource(const TaskPtr& task, const ResourcePtr& dest) { dest->task_table().Put(task); } void -Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr, - ResourcePtr resource, +Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr resource, std::shared_ptr event) { if (not resource->HasExecutor() && event->task_table_item_->Move()) { auto task = event->task_table_item_->task; auto search_task = std::static_pointer_cast(task); bool moved = false; - //to support test task, REFACTOR + // to support test task, REFACTOR if (auto index_engine = search_task->index_engine_) { auto location = index_engine->GetLocation(); @@ -131,8 +128,7 @@ Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr, } void -Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, - ResourcePtr resource, +Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr resource, std::shared_ptr event) { auto task = event->task_table_item_->task; if (resource->type() == ResourceType::DISK) { @@ -140,7 +136,7 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, auto compute_resources = res_mgr.lock()->GetComputeResources(); std::vector> paths; std::vector transport_costs; - for (auto &res : compute_resources) { + for (auto& res : compute_resources) { std::vector path; uint64_t transport_cost = ShortestPath(resource, res, res_mgr.lock(), path); transport_costs.push_back(transport_cost); @@ -155,8 +151,8 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, min_cost_idx = i; break; } - uint64_t cost = compute_resources[i]->TaskAvgCost() * compute_resources[i]->NumOfTaskToExec() - + transport_costs[i]; + uint64_t cost = + compute_resources[i]->TaskAvgCost() * compute_resources[i]->NumOfTaskToExec() + transport_costs[i]; if (min_cost > cost) { min_cost = cost; min_cost_idx = i; @@ -178,6 +174,6 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, } } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/Event.h b/cpp/src/scheduler/event/Event.h index 860c60c5b7..6d55820248 100644 --- a/cpp/src/scheduler/event/Event.h +++ b/cpp/src/scheduler/event/Event.h @@ -18,28 +18,20 @@ #pragma once #include -#include #include +#include namespace zilliz { namespace milvus { namespace scheduler { -enum class EventType { - START_UP, - LOAD_COMPLETED, - FINISH_TASK, - TASK_TABLE_UPDATED -}; +enum class EventType { START_UP, LOAD_COMPLETED, FINISH_TASK, TASK_TABLE_UPDATED }; class Resource; class Event { public: - explicit - Event(EventType type, std::weak_ptr resource) - : type_(type), - resource_(std::move(resource)) { + explicit Event(EventType type, std::weak_ptr resource) : type_(type), resource_(std::move(resource)) { } inline EventType @@ -50,7 +42,8 @@ class Event { virtual std::string Dump() const = 0; - friend std::ostream &operator<<(std::ostream &out, const Event &event); + friend std::ostream& + operator<<(std::ostream& out, const Event& event); public: EventType type_; @@ -59,6 +52,6 @@ class Event { using EventPtr = std::shared_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/EventDump.cpp b/cpp/src/scheduler/event/EventDump.cpp index a9ed751d88..6cc40ea28c 100644 --- a/cpp/src/scheduler/event/EventDump.cpp +++ b/cpp/src/scheduler/event/EventDump.cpp @@ -15,47 +15,46 @@ // specific language governing permissions and limitations // under the License. - #include "Event.h" -#include "StartUpEvent.h" -#include "LoadCompletedEvent.h" #include "FinishTaskEvent.h" +#include "LoadCompletedEvent.h" +#include "StartUpEvent.h" #include "TaskTableUpdatedEvent.h" namespace zilliz { namespace milvus { namespace scheduler { -std::ostream & -operator<<(std::ostream &out, const Event &event) { +std::ostream& +operator<<(std::ostream& out, const Event& event) { out << event.Dump(); return out; } -std::ostream & -operator<<(std::ostream &out, const StartUpEvent &event) { +std::ostream& +operator<<(std::ostream& out, const StartUpEvent& event) { out << event.Dump(); return out; } -std::ostream & -operator<<(std::ostream &out, const LoadCompletedEvent &event) { +std::ostream& +operator<<(std::ostream& out, const LoadCompletedEvent& event) { out << event.Dump(); return out; } -std::ostream & -operator<<(std::ostream &out, const FinishTaskEvent &event) { +std::ostream& +operator<<(std::ostream& out, const FinishTaskEvent& event) { out << event.Dump(); return out; } -std::ostream & -operator<<(std::ostream &out, const TaskTableUpdatedEvent &event) { +std::ostream& +operator<<(std::ostream& out, const TaskTableUpdatedEvent& event) { out << event.Dump(); return out; } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/FinishTaskEvent.h b/cpp/src/scheduler/event/FinishTaskEvent.h index f49acb16ad..b593abfe31 100644 --- a/cpp/src/scheduler/event/FinishTaskEvent.h +++ b/cpp/src/scheduler/event/FinishTaskEvent.h @@ -17,11 +17,12 @@ #pragma once +#include "scheduler/TaskTable.h" #include "scheduler/event/Event.h" #include -#include #include +#include namespace zilliz { namespace milvus { @@ -30,8 +31,7 @@ namespace scheduler { class FinishTaskEvent : public Event { public: FinishTaskEvent(std::weak_ptr resource, TaskTableItemPtr task_table_item) - : Event(EventType::FINISH_TASK, std::move(resource)), - task_table_item_(std::move(task_table_item)) { + : Event(EventType::FINISH_TASK, std::move(resource)), task_table_item_(std::move(task_table_item)) { } inline std::string @@ -39,12 +39,13 @@ class FinishTaskEvent : public Event { return ""; } - friend std::ostream &operator<<(std::ostream &out, const FinishTaskEvent &event); + friend std::ostream& + operator<<(std::ostream& out, const FinishTaskEvent& event); public: TaskTableItemPtr task_table_item_; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/LoadCompletedEvent.h b/cpp/src/scheduler/event/LoadCompletedEvent.h index 8d727f7435..fa1f34b8c1 100644 --- a/cpp/src/scheduler/event/LoadCompletedEvent.h +++ b/cpp/src/scheduler/event/LoadCompletedEvent.h @@ -17,12 +17,12 @@ #pragma once -#include "scheduler/event/Event.h" #include "scheduler/TaskTable.h" +#include "scheduler/event/Event.h" #include -#include #include +#include namespace zilliz { namespace milvus { @@ -31,8 +31,7 @@ namespace scheduler { class LoadCompletedEvent : public Event { public: LoadCompletedEvent(std::weak_ptr resource, TaskTableItemPtr task_table_item) - : Event(EventType::LOAD_COMPLETED, std::move(resource)), - task_table_item_(std::move(task_table_item)) { + : Event(EventType::LOAD_COMPLETED, std::move(resource)), task_table_item_(std::move(task_table_item)) { } inline std::string @@ -40,12 +39,13 @@ class LoadCompletedEvent : public Event { return ""; } - friend std::ostream &operator<<(std::ostream &out, const LoadCompletedEvent &event); + friend std::ostream& + operator<<(std::ostream& out, const LoadCompletedEvent& event); public: TaskTableItemPtr task_table_item_; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/StartUpEvent.h b/cpp/src/scheduler/event/StartUpEvent.h index 8e4ad120de..037bf0f4d0 100644 --- a/cpp/src/scheduler/event/StartUpEvent.h +++ b/cpp/src/scheduler/event/StartUpEvent.h @@ -20,8 +20,8 @@ #include "scheduler/event/Event.h" #include -#include #include +#include namespace zilliz { namespace milvus { @@ -29,8 +29,7 @@ namespace scheduler { class StartUpEvent : public Event { public: - explicit StartUpEvent(std::weak_ptr resource) - : Event(EventType::START_UP, std::move(resource)) { + explicit StartUpEvent(std::weak_ptr resource) : Event(EventType::START_UP, std::move(resource)) { } inline std::string @@ -38,9 +37,10 @@ class StartUpEvent : public Event { return ""; } - friend std::ostream &operator<<(std::ostream &out, const StartUpEvent &event); + friend std::ostream& + operator<<(std::ostream& out, const StartUpEvent& event); }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/event/TaskTableUpdatedEvent.h b/cpp/src/scheduler/event/TaskTableUpdatedEvent.h index ec579b31be..814b25f056 100644 --- a/cpp/src/scheduler/event/TaskTableUpdatedEvent.h +++ b/cpp/src/scheduler/event/TaskTableUpdatedEvent.h @@ -20,8 +20,8 @@ #include "Event.h" #include -#include #include +#include namespace zilliz { namespace milvus { @@ -38,9 +38,10 @@ class TaskTableUpdatedEvent : public Event { return ""; } - friend std::ostream &operator<<(std::ostream &out, const TaskTableUpdatedEvent &event); + friend std::ostream& + operator<<(std::ostream& out, const TaskTableUpdatedEvent& event); }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/DeleteJob.cpp b/cpp/src/scheduler/job/DeleteJob.cpp index 9d917751c6..5fcbb40513 100644 --- a/cpp/src/scheduler/job/DeleteJob.cpp +++ b/cpp/src/scheduler/job/DeleteJob.cpp @@ -23,10 +23,7 @@ namespace zilliz { namespace milvus { namespace scheduler { -DeleteJob::DeleteJob(JobId id, - std::string table_id, - engine::meta::MetaPtr meta_ptr, - uint64_t num_resource) +DeleteJob::DeleteJob(JobId id, std::string table_id, engine::meta::MetaPtr meta_ptr, uint64_t num_resource) : Job(id, JobType::DELETE), table_id_(std::move(table_id)), meta_ptr_(std::move(meta_ptr)), @@ -36,9 +33,7 @@ DeleteJob::DeleteJob(JobId id, void DeleteJob::WaitAndDelete() { std::unique_lock lock(mutex_); - cv_.wait(lock, [&] { - return done_resource == num_resource_; - }); + cv_.wait(lock, [&] { return done_resource == num_resource_; }); meta_ptr_->DeleteTableFiles(table_id_); } @@ -51,6 +46,6 @@ DeleteJob::ResourceDone() { cv_.notify_one(); } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/DeleteJob.h b/cpp/src/scheduler/job/DeleteJob.h index 7d8b20e47c..6c2f1e40c7 100644 --- a/cpp/src/scheduler/job/DeleteJob.h +++ b/cpp/src/scheduler/job/DeleteJob.h @@ -16,16 +16,16 @@ // under the License. #pragma once -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include #include "Job.h" #include "db/meta/Meta.h" @@ -36,10 +36,7 @@ namespace scheduler { class DeleteJob : public Job { public: - DeleteJob(JobId id, - std::string table_id, - engine::meta::MetaPtr meta_ptr, - uint64_t num_resource); + DeleteJob(JobId id, std::string table_id, engine::meta::MetaPtr meta_ptr, uint64_t num_resource); public: void @@ -71,6 +68,6 @@ class DeleteJob : public Job { using DeleteJobPtr = std::shared_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/Job.h b/cpp/src/scheduler/job/Job.h index c646a4f034..ccfa3518f9 100644 --- a/cpp/src/scheduler/job/Job.h +++ b/cpp/src/scheduler/job/Job.h @@ -16,16 +16,16 @@ // under the License. #pragma once -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include namespace zilliz { namespace milvus { @@ -64,6 +64,6 @@ class Job { using JobPtr = std::shared_ptr; using JobWPtr = std::weak_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/SearchJob.cpp b/cpp/src/scheduler/job/SearchJob.cpp index dee7125fed..b4d3ef84c2 100644 --- a/cpp/src/scheduler/job/SearchJob.cpp +++ b/cpp/src/scheduler/job/SearchJob.cpp @@ -22,20 +22,13 @@ namespace zilliz { namespace milvus { namespace scheduler { -SearchJob::SearchJob(zilliz::milvus::scheduler::JobId id, - uint64_t topk, - uint64_t nq, - uint64_t nprobe, - const float *vectors) - : Job(id, JobType::SEARCH), - topk_(topk), - nq_(nq), - nprobe_(nprobe), - vectors_(vectors) { +SearchJob::SearchJob(zilliz::milvus::scheduler::JobId id, uint64_t topk, uint64_t nq, uint64_t nprobe, + const float* vectors) + : Job(id, JobType::SEARCH), topk_(topk), nq_(nq), nprobe_(nprobe), vectors_(vectors) { } bool -SearchJob::AddIndexFile(const TableFileSchemaPtr &index_file) { +SearchJob::AddIndexFile(const TableFileSchemaPtr& index_file) { std::unique_lock lock(mutex_); if (index_file == nullptr || index_files_.find(index_file->id_) != index_files_.end()) { return false; @@ -50,9 +43,7 @@ SearchJob::AddIndexFile(const TableFileSchemaPtr &index_file) { void SearchJob::WaitResult() { std::unique_lock lock(mutex_); - cv_.wait(lock, [this] { - return index_files_.empty(); - }); + cv_.wait(lock, [this] { return index_files_.empty(); }); SERVER_LOG_DEBUG << "SearchJob " << id() << " all done"; } @@ -64,16 +55,16 @@ SearchJob::SearchDone(size_t index_id) { SERVER_LOG_DEBUG << "SearchJob " << id() << " finish index file: " << index_id; } -ResultSet & +ResultSet& SearchJob::GetResult() { return result_; } -Status & +Status& SearchJob::GetStatus() { return status_; } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/job/SearchJob.h b/cpp/src/scheduler/job/SearchJob.h index 7bb7fbefbf..7ffb0cd201 100644 --- a/cpp/src/scheduler/job/SearchJob.h +++ b/cpp/src/scheduler/job/SearchJob.h @@ -16,17 +16,17 @@ // under the License. #pragma once -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include #include +#include +#include +#include +#include +#include #include +#include #include "Job.h" #include "db/meta/MetaTypes.h" @@ -43,11 +43,11 @@ using ResultSet = std::vector; class SearchJob : public Job { public: - SearchJob(JobId id, uint64_t topk, uint64_t nq, uint64_t nprobe, const float *vectors); + SearchJob(JobId id, uint64_t topk, uint64_t nq, uint64_t nprobe, const float* vectors); public: bool - AddIndexFile(const TableFileSchemaPtr &index_file); + AddIndexFile(const TableFileSchemaPtr& index_file); void WaitResult(); @@ -55,10 +55,10 @@ class SearchJob : public Job { void SearchDone(size_t index_id); - ResultSet & + ResultSet& GetResult(); - Status & + Status& GetStatus(); public: @@ -77,12 +77,12 @@ class SearchJob : public Job { return nprobe_; } - const float * + const float* vectors() const { return vectors_; } - Id2IndexMap & + Id2IndexMap& index_files() { return index_files_; } @@ -92,7 +92,7 @@ class SearchJob : public Job { uint64_t nq_ = 0; uint64_t nprobe_ = 0; // TODO: smart pointer - const float *vectors_ = nullptr; + const float* vectors_ = nullptr; Id2IndexMap index_files_; // TODO: column-base better ? @@ -105,6 +105,6 @@ class SearchJob : public Job { using SearchJobPtr = std::shared_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Connection.h b/cpp/src/scheduler/resource/Connection.h index cf18b6c9a2..ac2ac81bf5 100644 --- a/cpp/src/scheduler/resource/Connection.h +++ b/cpp/src/scheduler/resource/Connection.h @@ -17,8 +17,8 @@ #pragma once -#include #include +#include #include namespace zilliz { @@ -28,11 +28,10 @@ namespace scheduler { class Connection { public: // TODO: update construct function, speed: double->uint64_t - Connection(std::string name, double speed) - : name_(std::move(name)), speed_(speed) { + Connection(std::string name, double speed) : name_(std::move(name)), speed_(speed) { } - const std::string & + const std::string& name() const { return name_; } @@ -60,6 +59,6 @@ class Connection { uint64_t speed_; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/CpuResource.cpp b/cpp/src/scheduler/resource/CpuResource.cpp index 5859dfd0cd..d1b0e2b8c3 100644 --- a/cpp/src/scheduler/resource/CpuResource.cpp +++ b/cpp/src/scheduler/resource/CpuResource.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include "scheduler/resource/CpuResource.h" #include @@ -24,8 +23,8 @@ namespace zilliz { namespace milvus { namespace scheduler { -std::ostream & -operator<<(std::ostream &out, const CpuResource &resource) { +std::ostream& +operator<<(std::ostream& out, const CpuResource& resource) { out << resource.Dump(); return out; } @@ -44,6 +43,6 @@ CpuResource::Process(TaskPtr task) { task->Execute(); } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/CpuResource.h b/cpp/src/scheduler/resource/CpuResource.h index 2226523fdf..562a306373 100644 --- a/cpp/src/scheduler/resource/CpuResource.h +++ b/cpp/src/scheduler/resource/CpuResource.h @@ -27,15 +27,15 @@ namespace scheduler { class CpuResource : public Resource { public: - explicit - CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); + explicit CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); inline std::string Dump() const override { return ""; } - friend std::ostream &operator<<(std::ostream &out, const CpuResource &resource); + friend std::ostream& + operator<<(std::ostream& out, const CpuResource& resource); protected: void @@ -45,6 +45,6 @@ class CpuResource : public Resource { Process(TaskPtr task) override; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/DiskResource.cpp b/cpp/src/scheduler/resource/DiskResource.cpp index eee2424cc1..08c03d2ef2 100644 --- a/cpp/src/scheduler/resource/DiskResource.cpp +++ b/cpp/src/scheduler/resource/DiskResource.cpp @@ -24,8 +24,8 @@ namespace zilliz { namespace milvus { namespace scheduler { -std::ostream & -operator<<(std::ostream &out, const DiskResource &resource) { +std::ostream& +operator<<(std::ostream& out, const DiskResource& resource) { out << resource.Dump(); return out; } @@ -42,6 +42,6 @@ void DiskResource::Process(TaskPtr task) { } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/DiskResource.h b/cpp/src/scheduler/resource/DiskResource.h index a7caf5c662..d86de49934 100644 --- a/cpp/src/scheduler/resource/DiskResource.h +++ b/cpp/src/scheduler/resource/DiskResource.h @@ -27,15 +27,15 @@ namespace scheduler { class DiskResource : public Resource { public: - explicit - DiskResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); + explicit DiskResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); inline std::string Dump() const override { return ""; } - friend std::ostream &operator<<(std::ostream &out, const DiskResource &resource); + friend std::ostream& + operator<<(std::ostream& out, const DiskResource& resource); protected: void @@ -45,6 +45,6 @@ class DiskResource : public Resource { Process(TaskPtr task) override; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/GpuResource.cpp b/cpp/src/scheduler/resource/GpuResource.cpp index 3c7abc0b29..a28b00bbe5 100644 --- a/cpp/src/scheduler/resource/GpuResource.cpp +++ b/cpp/src/scheduler/resource/GpuResource.cpp @@ -15,15 +15,14 @@ // specific language governing permissions and limitations // under the License. - #include "scheduler/resource/GpuResource.h" namespace zilliz { namespace milvus { namespace scheduler { -std::ostream & -operator<<(std::ostream &out, const GpuResource &resource) { +std::ostream& +operator<<(std::ostream& out, const GpuResource& resource) { out << resource.Dump(); return out; } @@ -42,6 +41,6 @@ GpuResource::Process(TaskPtr task) { task->Execute(); } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/GpuResource.h b/cpp/src/scheduler/resource/GpuResource.h index 9f19b07464..33a197aed0 100644 --- a/cpp/src/scheduler/resource/GpuResource.h +++ b/cpp/src/scheduler/resource/GpuResource.h @@ -28,15 +28,15 @@ namespace scheduler { class GpuResource : public Resource { public: - explicit - GpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); + explicit GpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); inline std::string Dump() const override { return ""; } - friend std::ostream &operator<<(std::ostream &out, const GpuResource &resource); + friend std::ostream& + operator<<(std::ostream& out, const GpuResource& resource); protected: void @@ -46,6 +46,6 @@ class GpuResource : public Resource { Process(TaskPtr task) override; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Node.cpp b/cpp/src/scheduler/resource/Node.cpp index cf652b8ba0..39333cc072 100644 --- a/cpp/src/scheduler/resource/Node.cpp +++ b/cpp/src/scheduler/resource/Node.cpp @@ -33,7 +33,7 @@ std::vector Node::GetNeighbours() { std::lock_guard lk(mutex_); std::vector ret; - for (auto &e : neighbours_) { + for (auto& e : neighbours_) { ret.push_back(e.second); } return ret; @@ -43,7 +43,7 @@ std::string Node::Dump() { std::stringstream ss; ss << "::neighbours:" << std::endl; - for (auto &neighbour : neighbours_) { + for (auto& neighbour : neighbours_) { ss << "\t" << std::endl; } @@ -51,7 +51,7 @@ Node::Dump() { } void -Node::AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &connection) { +Node::AddNeighbour(const NeighbourNodePtr& neighbour_node, Connection& connection) { std::lock_guard lk(mutex_); if (auto s = neighbour_node.lock()) { neighbours_.emplace(std::make_pair(s->id_, Neighbour(neighbour_node, connection))); @@ -59,6 +59,6 @@ Node::AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &connectio // else do nothing, consider it.. } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Node.h b/cpp/src/scheduler/resource/Node.h index 00337a7009..0713dafa2d 100644 --- a/cpp/src/scheduler/resource/Node.h +++ b/cpp/src/scheduler/resource/Node.h @@ -17,13 +17,13 @@ #pragma once -#include -#include #include +#include #include +#include -#include "scheduler/TaskTable.h" #include "Connection.h" +#include "scheduler/TaskTable.h" namespace zilliz { namespace milvus { @@ -34,8 +34,7 @@ class Node; using NeighbourNodePtr = std::weak_ptr; struct Neighbour { - Neighbour(NeighbourNodePtr nei, Connection conn) - : neighbour_node(nei), connection(conn) { + Neighbour(NeighbourNodePtr nei, Connection conn) : neighbour_node(nei), connection(conn) { } NeighbourNodePtr neighbour_node; @@ -48,7 +47,7 @@ class Node { Node(); void - AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &connection); + AddNeighbour(const NeighbourNodePtr& neighbour_node, Connection& connection); std::vector GetNeighbours(); @@ -66,6 +65,6 @@ class Node { using NodePtr = std::shared_ptr; using NodeWPtr = std::weak_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Resource.cpp b/cpp/src/scheduler/resource/Resource.cpp index 4eb71b8ac5..8273d69d33 100644 --- a/cpp/src/scheduler/resource/Resource.cpp +++ b/cpp/src/scheduler/resource/Resource.cpp @@ -25,17 +25,13 @@ namespace zilliz { namespace milvus { namespace scheduler { -std::ostream & -operator<<(std::ostream &out, const Resource &resource) { +std::ostream& +operator<<(std::ostream& out, const Resource& resource) { out << resource.Dump(); return out; } -Resource::Resource(std::string name, - ResourceType type, - uint64_t device_id, - bool enable_loader, - bool enable_executor) +Resource::Resource(std::string name, ResourceType type, uint64_t device_id, bool enable_loader, bool enable_executor) : name_(std::move(name)), type_(type), device_id_(device_id), @@ -95,7 +91,7 @@ Resource::WakeupExecutor() { uint64_t Resource::NumOfTaskToExec() { uint64_t count = 0; - for (auto &task : task_table_) { + for (auto& task : task_table_) { if (task->state == TaskTableItemState::LOADED) ++count; } return count; @@ -106,8 +102,7 @@ Resource::pick_task_load() { auto indexes = task_table_.PickToLoad(10); for (auto index : indexes) { // try to set one task loading, then return - if (task_table_.Load(index)) - return task_table_.Get(index); + if (task_table_.Load(index)) return task_table_.Get(index); // else try next } return nullptr; @@ -118,8 +113,7 @@ Resource::pick_task_execute() { auto indexes = task_table_.PickToExecute(3); for (auto index : indexes) { // try to set one task executing, then return - if (task_table_.Execute(index)) - return task_table_.Get(index); + if (task_table_.Execute(index)) return task_table_.Get(index); // else try next } return nullptr; @@ -129,9 +123,7 @@ void Resource::loader_function() { while (running_) { std::unique_lock lock(load_mutex_); - load_cv_.wait(lock, [&] { - return load_flag_; - }); + load_cv_.wait(lock, [&] { return load_flag_; }); load_flag_ = false; lock.unlock(); while (true) { @@ -157,9 +149,7 @@ Resource::executor_function() { } while (running_) { std::unique_lock lock(exec_mutex_); - exec_cv_.wait(lock, [&] { - return exec_flag_; - }); + exec_cv_.wait(lock, [&] { return exec_flag_; }); exec_flag_ = false; lock.unlock(); while (true) { @@ -183,6 +173,6 @@ Resource::executor_function() { } } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/Resource.h b/cpp/src/scheduler/resource/Resource.h index 1c18b1a2b2..f6ba88c2c3 100644 --- a/cpp/src/scheduler/resource/Resource.h +++ b/cpp/src/scheduler/resource/Resource.h @@ -17,20 +17,20 @@ #pragma once -#include -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include -#include "../event/Event.h" -#include "../event/StartUpEvent.h" -#include "../event/LoadCompletedEvent.h" -#include "../event/FinishTaskEvent.h" -#include "../event/TaskTableUpdatedEvent.h" #include "../TaskTable.h" +#include "../event/Event.h" +#include "../event/FinishTaskEvent.h" +#include "../event/LoadCompletedEvent.h" +#include "../event/StartUpEvent.h" +#include "../event/TaskTableUpdatedEvent.h" #include "../task/Task.h" #include "Connection.h" #include "Node.h" @@ -99,7 +99,7 @@ class Resource : public Node, public std::enable_shared_from_this { return device_id_; } - TaskTable & + TaskTable& task_table() { return task_table_; } @@ -130,14 +130,11 @@ class Resource : public Node, public std::enable_shared_from_this { return total_task_; } - friend std::ostream &operator<<(std::ostream &out, const Resource &resource); + friend std::ostream& + operator<<(std::ostream& out, const Resource& resource); protected: - Resource(std::string name, - ResourceType type, - uint64_t device_id, - bool enable_loader, - bool enable_executor); + Resource(std::string name, ResourceType type, uint64_t device_id, bool enable_loader, bool enable_executor); /* * Implementation by inherit class; @@ -212,6 +209,6 @@ class Resource : public Node, public std::enable_shared_from_this { using ResourcePtr = std::shared_ptr; using ResourceWPtr = std::weak_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/TestResource.cpp b/cpp/src/scheduler/resource/TestResource.cpp index 25560cf7ee..dd16117236 100644 --- a/cpp/src/scheduler/resource/TestResource.cpp +++ b/cpp/src/scheduler/resource/TestResource.cpp @@ -23,8 +23,8 @@ namespace zilliz { namespace milvus { namespace scheduler { -std::ostream & -operator<<(std::ostream &out, const TestResource &resource) { +std::ostream& +operator<<(std::ostream& out, const TestResource& resource) { out << resource.Dump(); return out; } @@ -43,6 +43,6 @@ TestResource::Process(TaskPtr task) { task->Execute(); } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/resource/TestResource.h b/cpp/src/scheduler/resource/TestResource.h index ac83a42c60..45dbe67500 100644 --- a/cpp/src/scheduler/resource/TestResource.h +++ b/cpp/src/scheduler/resource/TestResource.h @@ -19,8 +19,8 @@ #include "Resource.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -28,15 +28,15 @@ namespace scheduler { class TestResource : public Resource { public: - explicit - TestResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); + explicit TestResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); inline std::string Dump() const override { return ""; } - friend std::ostream &operator<<(std::ostream &out, const TestResource &resource); + friend std::ostream& + operator<<(std::ostream& out, const TestResource& resource); protected: void @@ -46,6 +46,6 @@ class TestResource : public Resource { Process(TaskPtr task) override; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/DeleteTask.cpp b/cpp/src/scheduler/task/DeleteTask.cpp index 52579d67c6..c6cc33ac88 100644 --- a/cpp/src/scheduler/task/DeleteTask.cpp +++ b/cpp/src/scheduler/task/DeleteTask.cpp @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. - #include "scheduler/task/DeleteTask.h" namespace zilliz { namespace milvus { namespace scheduler { -XDeleteTask::XDeleteTask(const scheduler::DeleteJobPtr &delete_job) +XDeleteTask::XDeleteTask(const scheduler::DeleteJobPtr& delete_job) : Task(TaskType::DeleteTask), delete_job_(delete_job) { } @@ -35,6 +34,6 @@ XDeleteTask::Execute() { delete_job_->ResourceDone(); } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/DeleteTask.h b/cpp/src/scheduler/task/DeleteTask.h index 608960e7c8..65e8724940 100644 --- a/cpp/src/scheduler/task/DeleteTask.h +++ b/cpp/src/scheduler/task/DeleteTask.h @@ -17,8 +17,8 @@ #pragma once -#include "scheduler/job/DeleteJob.h" #include "Task.h" +#include "scheduler/job/DeleteJob.h" namespace zilliz { namespace milvus { @@ -26,7 +26,7 @@ namespace scheduler { class XDeleteTask : public Task { public: - explicit XDeleteTask(const scheduler::DeleteJobPtr &job); + explicit XDeleteTask(const scheduler::DeleteJobPtr& job); void Load(LoadType type, uint8_t device_id) override; @@ -38,6 +38,6 @@ class XDeleteTask : public Task { scheduler::DeleteJobPtr delete_job_; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/Path.h b/cpp/src/scheduler/task/Path.h index 672dfff1b9..2be2a7fc0d 100644 --- a/cpp/src/scheduler/task/Path.h +++ b/cpp/src/scheduler/task/Path.h @@ -17,8 +17,8 @@ #pragma once -#include #include +#include namespace zilliz { namespace milvus { @@ -28,11 +28,11 @@ class Path { public: Path() = default; - Path(std::vector &path, uint64_t index) : path_(path), index_(index) { + Path(std::vector& path, uint64_t index) : path_(path), index_(index) { } void - push_back(const std::string &str) { + push_back(const std::string& str) { path_.push_back(str); } @@ -61,16 +61,17 @@ class Path { } public: - std::string & - operator[](uint64_t index) { + std::string& operator[](uint64_t index) { return path_[index]; } - std::vector::iterator begin() { + std::vector::iterator + begin() { return path_.begin(); } - std::vector::iterator end() { + std::vector::iterator + end() { return path_.end(); } @@ -79,6 +80,6 @@ class Path { uint64_t index_ = 0; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/SearchTask.cpp b/cpp/src/scheduler/task/SearchTask.cpp index 0c205fcafa..0255699915 100644 --- a/cpp/src/scheduler/task/SearchTask.cpp +++ b/cpp/src/scheduler/task/SearchTask.cpp @@ -16,15 +16,15 @@ // under the License. #include "scheduler/task/SearchTask.h" -#include "scheduler/job/SearchJob.h" -#include "metrics/Metrics.h" #include "db/engine/EngineFactory.h" -#include "utils/TimeRecorder.h" +#include "metrics/Metrics.h" +#include "scheduler/job/SearchJob.h" #include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include #include #include -#include namespace zilliz { namespace milvus { @@ -35,8 +35,8 @@ static constexpr size_t PARALLEL_REDUCE_BATCH = 1000; std::mutex XSearchTask::merge_mutex_; -//bool -//NeedParallelReduce(uint64_t nq, uint64_t topk) { +// bool +// NeedParallelReduce(uint64_t nq, uint64_t topk) { // server::ServerConfig &config = server::ServerConfig::GetInstance(); // server::ConfigNode &db_config = config.GetConfig(server::CONFIG_DB); // bool need_parallel = db_config.GetBoolValue(server::CONFIG_DB_PARALLEL_REDUCE, false); @@ -47,8 +47,8 @@ std::mutex XSearchTask::merge_mutex_; // return nq * topk >= PARALLEL_REDUCE_THRESHOLD; //} // -//void -//ParallelReduce(std::function &reduce_function, size_t max_index) { +// void +// ParallelReduce(std::function &reduce_function, size_t max_index) { // size_t reduce_batch = PARALLEL_REDUCE_BATCH; // // auto thread_count = std::thread::hardware_concurrency() - 1; //not all core do this work @@ -96,14 +96,10 @@ CollectFileMetrics(int file_type, size_t file_size) { } } -XSearchTask::XSearchTask(TableFileSchemaPtr file) - : Task(TaskType::SearchTask), file_(file) { +XSearchTask::XSearchTask(TableFileSchemaPtr file) : Task(TaskType::SearchTask), file_(file) { if (file_) { - index_engine_ = EngineFactory::Build(file_->dimension_, - file_->location_, - (EngineType) file_->engine_type_, - (MetricType) file_->metric_type_, - file_->nlist_); + index_engine_ = EngineFactory::Build(file_->dimension_, file_->location_, (EngineType)file_->engine_type_, + (MetricType)file_->metric_type_, file_->nlist_); } } @@ -128,8 +124,8 @@ XSearchTask::Load(LoadType type, uint8_t device_id) { error_msg = "Wrong load type"; stat = Status(SERVER_UNEXPECTED_ERROR, error_msg); } - } catch (std::exception &ex) { - //typical error: out of disk space or permition denied + } catch (std::exception& ex) { + // typical error: out of disk space or permition denied error_msg = "Failed to load index file: " + std::string(ex.what()); stat = Status(SERVER_UNEXPECTED_ERROR, error_msg); } @@ -155,19 +151,20 @@ XSearchTask::Load(LoadType type, uint8_t device_id) { size_t file_size = index_engine_->PhysicalSize(); - std::string info = "Load file id:" + std::to_string(file_->id_) + " file type:" + std::to_string(file_->file_type_) - + " size:" + std::to_string(file_size) + " bytes from location: " + file_->location_ + " totally cost"; + std::string info = "Load file id:" + std::to_string(file_->id_) + " file type:" + + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) + + " bytes from location: " + file_->location_ + " totally cost"; double span = rc.ElapseFromBegin(info); -// for (auto &context : search_contexts_) { -// context->AccumLoadCost(span); -// } + // for (auto &context : search_contexts_) { + // context->AccumLoadCost(span); + // } CollectFileMetrics(file_->file_type_, file_size); - //step 2: return search task for later execution + // step 2: return search task for later execution index_id_ = file_->id_; index_type_ = file_->file_type_; -// search_contexts_.swap(search_contexts_); + // search_contexts_.swap(search_contexts_); } void @@ -176,8 +173,8 @@ XSearchTask::Execute() { return; } -// ENGINE_LOG_DEBUG << "Searching in file id:" << index_id_ << " with " -// << search_contexts_.size() << " tasks"; + // ENGINE_LOG_DEBUG << "Searching in file id:" << index_id_ << " with " + // << search_contexts_.size() << " tasks"; TimeRecorder rc("DoSearch file id:" + std::to_string(index_id_)); @@ -188,45 +185,43 @@ XSearchTask::Execute() { if (auto job = job_.lock()) { auto search_job = std::static_pointer_cast(job); - //step 1: allocate memory + // step 1: allocate memory uint64_t nq = search_job->nq(); uint64_t topk = search_job->topk(); uint64_t nprobe = search_job->nprobe(); - const float *vectors = search_job->vectors(); + const float* vectors = search_job->vectors(); output_ids.resize(topk * nq); output_distance.resize(topk * nq); - std::string hdr = "job " + std::to_string(search_job->id()) + - " nq " + std::to_string(nq) + - " topk " + std::to_string(topk); + std::string hdr = + "job " + std::to_string(search_job->id()) + " nq " + std::to_string(nq) + " topk " + std::to_string(topk); try { - //step 2: search + // step 2: search index_engine_->Search(nq, vectors, topk, nprobe, output_distance.data(), output_ids.data()); double span = rc.RecordSection(hdr + ", do search"); -// search_job->AccumSearchCost(span); + // search_job->AccumSearchCost(span); - - //step 3: cluster result + // step 3: cluster result scheduler::ResultSet result_set; auto spec_k = index_engine_->Count() < topk ? index_engine_->Count() : topk; XSearchTask::ClusterResult(output_ids, output_distance, nq, spec_k, result_set); span = rc.RecordSection(hdr + ", cluster result"); -// search_job->AccumReduceCost(span); + // search_job->AccumReduceCost(span); // step 4: pick up topk result XSearchTask::TopkResult(result_set, topk, metric_l2, search_job->GetResult()); span = rc.RecordSection(hdr + ", reduce topk"); -// search_job->AccumReduceCost(span); - } catch (std::exception &ex) { + // search_job->AccumReduceCost(span); + } catch (std::exception& ex) { ENGINE_LOG_ERROR << "SearchTask encounter exception: " << ex.what(); -// search_job->IndexSearchDone(index_id_);//mark as done avoid dead lock, even search failed + // search_job->IndexSearchDone(index_id_);//mark as done avoid dead lock, even search failed } - //step 5: notify to send result to client + // step 5: notify to send result to client search_job->SearchDone(index_id_); } @@ -237,14 +232,11 @@ XSearchTask::Execute() { } Status -XSearchTask::ClusterResult(const std::vector &output_ids, - const std::vector &output_distance, - uint64_t nq, - uint64_t topk, - scheduler::ResultSet &result_set) { +XSearchTask::ClusterResult(const std::vector& output_ids, const std::vector& output_distance, + uint64_t nq, uint64_t topk, scheduler::ResultSet& result_set) { if (output_ids.size() < nq * topk || output_distance.size() < nq * topk) { - std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) + - " distance array size: " + std::to_string(output_distance.size()); + std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) + " distance array size: " + + std::to_string(output_distance.size()); ENGINE_LOG_ERROR << msg; return Status(DB_ERROR, msg); } @@ -267,21 +259,19 @@ XSearchTask::ClusterResult(const std::vector &output_ids, } }; -// if (NeedParallelReduce(nq, topk)) { -// ParallelReduce(reduce_worker, nq); -// } else { + // if (NeedParallelReduce(nq, topk)) { + // ParallelReduce(reduce_worker, nq); + // } else { reduce_worker(0, nq); -// } + // } return Status::OK(); } Status -XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, - scheduler::Id2DistanceMap &distance_target, - uint64_t topk, - bool ascending) { - //Note: the score_src and score_target are already arranged by score in ascending order +XSearchTask::MergeResult(scheduler::Id2DistanceMap& distance_src, scheduler::Id2DistanceMap& distance_target, + uint64_t topk, bool ascending) { + // Note: the score_src and score_target are already arranged by score in ascending order if (distance_src.empty()) { ENGINE_LOG_WARNING << "Empty distance source array"; return Status::OK(); @@ -299,8 +289,8 @@ XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, distance_merged.reserve(topk); size_t src_index = 0, target_index = 0; while (true) { - //all score_src items are merged, if score_merged.size() still less than topk - //move items from score_target to score_merged until score_merged.size() equal topk + // all score_src items are merged, if score_merged.size() still less than topk + // move items from score_target to score_merged until score_merged.size() equal topk if (src_index >= src_count) { for (size_t i = target_index; i < target_count && distance_merged.size() < topk; ++i) { distance_merged.push_back(distance_target[i]); @@ -308,8 +298,8 @@ XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, break; } - //all score_target items are merged, if score_merged.size() still less than topk - //move items from score_src to score_merged until score_merged.size() equal topk + // all score_target items are merged, if score_merged.size() still less than topk + // move items from score_src to score_merged until score_merged.size() equal topk if (target_index >= target_count) { for (size_t i = src_index; i < src_count && distance_merged.size() < topk; ++i) { distance_merged.push_back(distance_src[i]); @@ -317,11 +307,11 @@ XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, break; } - //compare score, + // compare score, // if ascending = true, put smallest score to score_merged one by one // else, put largest score to score_merged one by one - auto &src_pair = distance_src[src_index]; - auto &target_pair = distance_target[target_index]; + auto& src_pair = distance_src[src_index]; + auto& target_pair = distance_target[target_index]; if (ascending) { if (src_pair.second > target_pair.second) { distance_merged.push_back(target_pair); @@ -340,7 +330,7 @@ XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, } } - //score_merged.size() already equal topk + // score_merged.size() already equal topk if (distance_merged.size() >= topk) { break; } @@ -352,10 +342,8 @@ XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, } Status -XSearchTask::TopkResult(scheduler::ResultSet &result_src, - uint64_t topk, - bool ascending, - scheduler::ResultSet &result_target) { +XSearchTask::TopkResult(scheduler::ResultSet& result_src, uint64_t topk, bool ascending, + scheduler::ResultSet& result_target) { if (result_target.empty()) { result_target.swap(result_src); return Status::OK(); @@ -369,21 +357,21 @@ XSearchTask::TopkResult(scheduler::ResultSet &result_src, std::function ReduceWorker = [&](size_t from_index, size_t to_index) { for (size_t i = from_index; i < to_index; i++) { - scheduler::Id2DistanceMap &score_src = result_src[i]; - scheduler::Id2DistanceMap &score_target = result_target[i]; + scheduler::Id2DistanceMap& score_src = result_src[i]; + scheduler::Id2DistanceMap& score_target = result_target[i]; XSearchTask::MergeResult(score_src, score_target, topk, ascending); } }; -// if (NeedParallelReduce(result_src.size(), topk)) { -// ParallelReduce(ReduceWorker, result_src.size()); -// } else { + // if (NeedParallelReduce(result_src.size(), topk)) { + // ParallelReduce(ReduceWorker, result_src.size()); + // } else { ReduceWorker(0, result_src.size()); -// } + // } return Status::OK(); } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/SearchTask.h b/cpp/src/scheduler/task/SearchTask.h index 7c19ba20f9..872187a077 100644 --- a/cpp/src/scheduler/task/SearchTask.h +++ b/cpp/src/scheduler/task/SearchTask.h @@ -18,8 +18,8 @@ #pragma once #include "Task.h" -#include "scheduler/job/SearchJob.h" #include "scheduler/Definition.h" +#include "scheduler/job/SearchJob.h" #include @@ -39,21 +39,16 @@ class XSearchTask : public Task { Execute() override; public: - static Status ClusterResult(const std::vector &output_ids, - const std::vector &output_distence, - uint64_t nq, - uint64_t topk, - scheduler::ResultSet &result_set); + static Status + ClusterResult(const std::vector& output_ids, const std::vector& output_distence, uint64_t nq, + uint64_t topk, scheduler::ResultSet& result_set); - static Status MergeResult(scheduler::Id2DistanceMap &distance_src, - scheduler::Id2DistanceMap &distance_target, - uint64_t topk, - bool ascending); + static Status + MergeResult(scheduler::Id2DistanceMap& distance_src, scheduler::Id2DistanceMap& distance_target, uint64_t topk, + bool ascending); - static Status TopkResult(scheduler::ResultSet &result_src, - uint64_t topk, - bool ascending, - scheduler::ResultSet &result_target); + static Status + TopkResult(scheduler::ResultSet& result_src, uint64_t topk, bool ascending, scheduler::ResultSet& result_target); public: TableFileSchemaPtr file_; @@ -66,6 +61,6 @@ class XSearchTask : public Task { static std::mutex merge_mutex_; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/Task.h b/cpp/src/scheduler/task/Task.h index 3600c10f03..da5fb3b3af 100644 --- a/cpp/src/scheduler/task/Task.h +++ b/cpp/src/scheduler/task/Task.h @@ -17,13 +17,13 @@ #pragma once -#include "scheduler/tasklabel/TaskLabel.h" -#include "scheduler/job/Job.h" -#include "utils/Status.h" #include "Path.h" +#include "scheduler/job/Job.h" +#include "scheduler/tasklabel/TaskLabel.h" +#include "utils/Status.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -63,7 +63,7 @@ class Task { /* * Transport path; */ - inline Path & + inline Path& path() { return task_path_; } @@ -71,7 +71,7 @@ class Task { /* * Getter and Setter; */ - inline TaskLabelPtr & + inline TaskLabelPtr& label() { return label_; } @@ -85,12 +85,12 @@ class Task { public: Path task_path_; -// std::vector search_contexts_; + // std::vector search_contexts_; scheduler::JobWPtr job_; TaskType type_; TaskLabelPtr label_ = nullptr; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/TestTask.cpp b/cpp/src/scheduler/task/TestTask.cpp index fc66e56269..aba7343467 100644 --- a/cpp/src/scheduler/task/TestTask.cpp +++ b/cpp/src/scheduler/task/TestTask.cpp @@ -22,7 +22,7 @@ namespace zilliz { namespace milvus { namespace scheduler { -TestTask::TestTask(TableFileSchemaPtr &file) : XSearchTask(file) { +TestTask::TestTask(TableFileSchemaPtr& file) : XSearchTask(file) { } void @@ -43,11 +43,9 @@ TestTask::Execute() { void TestTask::Wait() { std::unique_lock lock(mutex_); - cv_.wait(lock, [&] { - return done_; - }); + cv_.wait(lock, [&] { return done_; }); } -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/task/TestTask.h b/cpp/src/scheduler/task/TestTask.h index 7051080861..5d96b888aa 100644 --- a/cpp/src/scheduler/task/TestTask.h +++ b/cpp/src/scheduler/task/TestTask.h @@ -25,7 +25,7 @@ namespace scheduler { class TestTask : public XSearchTask { public: - explicit TestTask(TableFileSchemaPtr &file); + explicit TestTask(TableFileSchemaPtr& file); public: void @@ -46,6 +46,6 @@ class TestTask : public XSearchTask { std::condition_variable cv_; }; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/tasklabel/BroadcastLabel.h b/cpp/src/scheduler/tasklabel/BroadcastLabel.h index 6fca107864..ed68d3ccc1 100644 --- a/cpp/src/scheduler/tasklabel/BroadcastLabel.h +++ b/cpp/src/scheduler/tasklabel/BroadcastLabel.h @@ -33,7 +33,6 @@ class BroadcastLabel : public TaskLabel { using BroadcastLabelPtr = std::shared_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz - +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/tasklabel/DefaultLabel.h b/cpp/src/scheduler/tasklabel/DefaultLabel.h index 7943c4f7c1..fe7019f655 100644 --- a/cpp/src/scheduler/tasklabel/DefaultLabel.h +++ b/cpp/src/scheduler/tasklabel/DefaultLabel.h @@ -33,6 +33,6 @@ class DefaultLabel : public TaskLabel { using DefaultLabelPtr = std::shared_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/tasklabel/SpecResLabel.h b/cpp/src/scheduler/tasklabel/SpecResLabel.h index cfc5aa9400..b5bea81b39 100644 --- a/cpp/src/scheduler/tasklabel/SpecResLabel.h +++ b/cpp/src/scheduler/tasklabel/SpecResLabel.h @@ -19,8 +19,8 @@ #include "TaskLabel.h" -#include #include +#include class Resource; @@ -32,16 +32,16 @@ namespace scheduler { class SpecResLabel : public TaskLabel { public: - explicit SpecResLabel(const ResourceWPtr &resource) + explicit SpecResLabel(const ResourceWPtr& resource) : TaskLabel(TaskLabelType::SPECIFIED_RESOURCE), resource_(resource) { } - inline ResourceWPtr & + inline ResourceWPtr& resource() { return resource_; } - inline std::string & + inline std::string& resource_name() { return resource_name_; } @@ -53,6 +53,6 @@ class SpecResLabel : public TaskLabel { using SpecResLabelPtr = std::shared_ptr(); -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/scheduler/tasklabel/TaskLabel.h b/cpp/src/scheduler/tasklabel/TaskLabel.h index d8d404e094..57f486ea89 100644 --- a/cpp/src/scheduler/tasklabel/TaskLabel.h +++ b/cpp/src/scheduler/tasklabel/TaskLabel.h @@ -24,9 +24,9 @@ namespace milvus { namespace scheduler { enum class TaskLabelType { - DEFAULT, // means can be executed in any resource - SPECIFIED_RESOURCE, // means must executing in special resource - BROADCAST, // means all enable-executor resource must execute task + DEFAULT, // means can be executed in any resource + SPECIFIED_RESOURCE, // means must executing in special resource + BROADCAST, // means all enable-executor resource must execute task }; class TaskLabel { @@ -46,6 +46,6 @@ class TaskLabel { using TaskLabelPtr = std::shared_ptr; -} // namespace scheduler -} // namespace milvus -} // namespace zilliz +} // namespace scheduler +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/sdk/examples/grpcsimple/main.cpp b/cpp/src/sdk/examples/grpcsimple/main.cpp index 166707259a..76c60ac1f8 100644 --- a/cpp/src/sdk/examples/grpcsimple/main.cpp +++ b/cpp/src/sdk/examples/grpcsimple/main.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include #include #include @@ -24,10 +23,10 @@ #include "src/ClientTest.h" void -print_help(const std::string &app_name); +print_help(const std::string& app_name); int -main(int argc, char *argv[]) { +main(int argc, char* argv[]) { printf("Client start...\n"); std::string app_name = basename(argv[0]); @@ -44,19 +43,20 @@ main(int argc, char *argv[]) { while ((value = getopt_long(argc, argv, "s:p:h", long_options, &option_index)) != -1) { switch (value) { case 's': { - char *address_ptr = strdup(optarg); + char* address_ptr = strdup(optarg); address = address_ptr; free(address_ptr); break; } case 'p': { - char *port_ptr = strdup(optarg); + char* port_ptr = strdup(optarg); port = port_ptr; free(port_ptr); break; } case 'h': - default:print_help(app_name); + default: + print_help(app_name); return EXIT_SUCCESS; } } @@ -69,7 +69,7 @@ main(int argc, char *argv[]) { } void -print_help(const std::string &app_name) { +print_help(const std::string& app_name) { printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str()); printf(" Options:\n"); printf(" -s --server Server address, default 127.0.0.1\n"); diff --git a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp index e29bcc30f3..45ea707975 100644 --- a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp @@ -19,14 +19,14 @@ #include "MilvusApi.h" #include "cache/CpuCacheMgr.h" -#include #include -#include -#include #include +#include +#include #include -#include +#include #include +#include //#define SET_VECTOR_IDS; @@ -40,14 +40,14 @@ constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; constexpr int64_t BATCH_ROW_COUNT = 100000; constexpr int64_t NQ = 5; constexpr int64_t TOP_K = 10; -constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different +constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different constexpr int64_t ADD_VECTOR_LOOP = 1; constexpr int64_t SECONDS_EACH_HOUR = 3600; #define BLOCK_SPLITER std::cout << "===========================================" << std::endl; void -PrintTableSchema(const milvus::TableSchema &tb_schema) { +PrintTableSchema(const milvus::TableSchema& tb_schema) { BLOCK_SPLITER std::cout << "Table name: " << tb_schema.table_name << std::endl; std::cout << "Table dimension: " << tb_schema.dimension << std::endl; @@ -55,19 +55,18 @@ PrintTableSchema(const milvus::TableSchema &tb_schema) { } void -PrintSearchResult(const std::vector> &search_record_array, - const std::vector &topk_query_result_array) { +PrintSearchResult(const std::vector>& search_record_array, + const std::vector& topk_query_result_array) { BLOCK_SPLITER std::cout << "Returned result count: " << topk_query_result_array.size() << std::endl; int32_t index = 0; - for (auto &result : topk_query_result_array) { + for (auto& result : topk_query_result_array) { auto search_id = search_record_array[index].first; index++; - std::cout << "No." << std::to_string(index) << " vector " << std::to_string(search_id) - << " top " << std::to_string(result.query_result_arrays.size()) - << " search result:" << std::endl; - for (auto &item : result.query_result_arrays) { + std::cout << "No." << std::to_string(index) << " vector " << std::to_string(search_id) << " top " + << std::to_string(result.query_result_arrays.size()) << " search result:" << std::endl; + for (auto& item : result.query_result_arrays) { std::cout << "\t" << std::to_string(item.id) << "\tdistance:" << std::to_string(item.distance); std::cout << std::endl; } @@ -84,9 +83,9 @@ CurrentTime() { tm t; gmtime_r(&tt, &t); - std::string str = std::to_string(t.tm_year + 1900) + "_" + std::to_string(t.tm_mon + 1) - + "_" + std::to_string(t.tm_mday) + "_" + std::to_string(t.tm_hour) - + "_" + std::to_string(t.tm_min) + "_" + std::to_string(t.tm_sec); + std::string str = std::to_string(t.tm_year + 1900) + "_" + std::to_string(t.tm_mon + 1) + "_" + + std::to_string(t.tm_mday) + "_" + std::to_string(t.tm_hour) + "_" + std::to_string(t.tm_min) + + "_" + std::to_string(t.tm_sec); return str; } @@ -100,8 +99,8 @@ CurrentTmDate(int64_t offset_day = 0) { tm t; gmtime_r(&tt, &t); - std::string str = std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1) - + "-" + std::to_string(t.tm_mday); + std::string str = + std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1) + "-" + std::to_string(t.tm_mday); return str; } @@ -124,8 +123,7 @@ BuildTableSchema() { } void -BuildVectors(int64_t from, int64_t to, - std::vector &vector_record_array) { +BuildVectors(int64_t from, int64_t to, std::vector& vector_record_array) { if (to <= from) { return; } @@ -135,7 +133,7 @@ BuildVectors(int64_t from, int64_t to, milvus::RowRecord record; record.data.resize(TABLE_DIMENSION); for (int64_t i = 0; i < TABLE_DIMENSION; i++) { - record.data[i] = (float) (k % (i + 1)); + record.data[i] = (float)(k % (i + 1)); } vector_record_array.emplace_back(record); @@ -150,8 +148,7 @@ Sleep(int seconds) { class TimeRecorder { public: - explicit TimeRecorder(const std::string &title) - : title_(title) { + explicit TimeRecorder(const std::string& title) : title_(title) { start_ = std::chrono::system_clock::now(); } @@ -167,16 +164,15 @@ class TimeRecorder { }; void -CheckResult(const std::vector> &search_record_array, - const std::vector &topk_query_result_array) { +CheckResult(const std::vector>& search_record_array, + const std::vector& topk_query_result_array) { BLOCK_SPLITER int64_t index = 0; - for (auto &result : topk_query_result_array) { + for (auto& result : topk_query_result_array) { auto result_id = result.query_result_arrays[0].id; auto search_id = search_record_array[index++].first; if (result_id != search_id) { - std::cout << "The top 1 result is wrong: " << result_id - << " vs. " << search_id << std::endl; + std::cout << "The top 1 result is wrong: " << result_id << " vs. " << search_id << std::endl; } else { std::cout << "Check result sucessfully" << std::endl; } @@ -186,8 +182,7 @@ CheckResult(const std::vector> &search_rec void DoSearch(std::shared_ptr conn, - const std::vector> &search_record_array, - const std::string &phase_name) { + const std::vector>& search_record_array, const std::string& phase_name) { std::vector query_range_array; milvus::Range rg; rg.start_value = CurrentTmDate(); @@ -195,7 +190,7 @@ DoSearch(std::shared_ptr conn, query_range_array.emplace_back(rg); std::vector record_array; - for (auto &pair : search_record_array) { + for (auto& pair : search_record_array) { record_array.push_back(pair.second); } @@ -214,24 +209,24 @@ DoSearch(std::shared_ptr conn, PrintSearchResult(search_record_array, topk_query_result_array); CheckResult(search_record_array, topk_query_result_array); } -} // namespace +} // namespace void -ClientTest::Test(const std::string &address, const std::string &port) { +ClientTest::Test(const std::string& address, const std::string& port) { std::shared_ptr conn = milvus::Connection::Create(); - {//connect server + { // connect server milvus::ConnectParam param = {address, port}; milvus::Status stat = conn->Connect(param); std::cout << "Connect function call status: " << stat.message() << std::endl; } - {//server version + { // server version std::string version = conn->ServerVersion(); std::cout << "Server version: " << version << std::endl; } - {//sdk version + { // sdk version std::string version = conn->ClientVersion(); std::cout << "SDK version: " << version << std::endl; } @@ -241,15 +236,15 @@ ClientTest::Test(const std::string &address, const std::string &port) { milvus::Status stat = conn->ShowTables(tables); std::cout << "ShowTables function call status: " << stat.message() << std::endl; std::cout << "All tables: " << std::endl; - for (auto &table : tables) { + for (auto& table : tables) { int64_t row_count = 0; -// conn->DropTable(table); + // conn->DropTable(table); stat = conn->CountTable(table, row_count); std::cout << "\t" << table << "(" << row_count << " rows)" << std::endl; } } - {//create table + { // create table milvus::TableSchema tb_schema = BuildTableSchema(); milvus::Status stat = conn->CreateTable(tb_schema); std::cout << "CreateTable function call status: " << stat.message() << std::endl; @@ -261,7 +256,7 @@ ClientTest::Test(const std::string &address, const std::string &port) { } } - {//describe table + { // describe table milvus::TableSchema tb_schema; milvus::Status stat = conn->DescribeTable(TABLE_NAME, tb_schema); std::cout << "DescribeTable function call status: " << stat.message() << std::endl; @@ -269,21 +264,21 @@ ClientTest::Test(const std::string &address, const std::string &port) { } std::vector> search_record_array; - {//insert vectors - for (int i = 0; i < ADD_VECTOR_LOOP; i++) {//add vectors + { // insert vectors + for (int i = 0; i < ADD_VECTOR_LOOP; i++) { // add vectors std::vector record_array; int64_t begin_index = i * BATCH_ROW_COUNT; BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array); #ifdef SET_VECTOR_IDS record_ids.resize(ADD_VECTOR_LOOP * BATCH_ROW_COUNT); - for (auto j = begin_index; j record_ids; - //generate user defined ids + // generate user defined ids for (int k = 0; k < BATCH_ROW_COUNT; k++) { record_ids.push_back(i * BATCH_ROW_COUNT + k); } @@ -299,22 +294,21 @@ ClientTest::Test(const std::string &address, const std::string &port) { std::cout << "Returned id array count: " << record_ids.size() << std::endl; if (search_record_array.size() < NQ) { - search_record_array.push_back( - std::make_pair(record_ids[SEARCH_TARGET], record_array[SEARCH_TARGET])); + search_record_array.push_back(std::make_pair(record_ids[SEARCH_TARGET], record_array[SEARCH_TARGET])); } } } - {//search vectors without index + { // search vectors without index Sleep(2); int64_t row_count = 0; milvus::Status stat = conn->CountTable(TABLE_NAME, row_count); std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; -// DoSearch(conn, search_record_array, "Search without index"); + // DoSearch(conn, search_record_array, "Search without index"); } - {//wait unit build index finish + { // wait unit build index finish std::cout << "Wait until create all index done" << std::endl; milvus::IndexParam index; index.table_name = TABLE_NAME; @@ -328,19 +322,19 @@ ClientTest::Test(const std::string &address, const std::string &port) { std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; } - {//preload table + { // preload table milvus::Status stat = conn->PreloadTable(TABLE_NAME); std::cout << "PreloadTable function call status: " << stat.message() << std::endl; } - {//search vectors after build index finish + { // search vectors after build index finish for (uint64_t i = 0; i < 5; ++i) { DoSearch(conn, search_record_array, "Search after build index finish"); } -// std::cout << conn->DumpTaskTables() << std::endl; + // std::cout << conn->DumpTaskTables() << std::endl; } - {//delete index + { // delete index milvus::Status stat = conn->DropIndex(TABLE_NAME); std::cout << "DropIndex function call status: " << stat.message() << std::endl; @@ -349,7 +343,7 @@ ClientTest::Test(const std::string &address, const std::string &port) { std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; } - {//delete by range + { // delete by range milvus::Range rg; rg.start_value = CurrentTmDate(-2); rg.end_value = CurrentTmDate(-3); @@ -358,17 +352,18 @@ ClientTest::Test(const std::string &address, const std::string &port) { std::cout << "DeleteByRange function call status: " << stat.message() << std::endl; } - {//delete table -// Status stat = conn->DropTable(TABLE_NAME); -// std::cout << "DeleteTable function call status: " << stat.message() << std::endl; + { + // delete table + // Status stat = conn->DropTable(TABLE_NAME); + // std::cout << "DeleteTable function call status: " << stat.message() << std::endl; } - {//server status + { // server status std::string status = conn->ServerStatus(); std::cout << "Server status before disconnect: " << status << std::endl; } milvus::Connection::Destroy(conn); - {//server status + { // server status std::string status = conn->ServerStatus(); std::cout << "Server status after disconnect: " << status << std::endl; } diff --git a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h index 165c36a180..b028b63f44 100644 --- a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h +++ b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.h @@ -21,5 +21,6 @@ class ClientTest { public: - void Test(const std::string &address, const std::string &port); + void + Test(const std::string& address, const std::string& port); }; diff --git a/cpp/src/sdk/grpc/ClientProxy.cpp b/cpp/src/sdk/grpc/ClientProxy.cpp index f4cd1e9a38..8ebd577e3a 100644 --- a/cpp/src/sdk/grpc/ClientProxy.cpp +++ b/cpp/src/sdk/grpc/ClientProxy.cpp @@ -20,14 +20,14 @@ #include "grpc/gen-milvus/milvus.grpc.pb.h" #include -#include #include +#include //#define GRPC_MULTIPLE_THREAD; namespace milvus { bool -UriCheck(const std::string &uri) { +UriCheck(const std::string& uri) { size_t index = uri.find_first_of(':', 0); if (index == std::string::npos) { return false; @@ -37,7 +37,7 @@ UriCheck(const std::string &uri) { } Status -ClientProxy::Connect(const ConnectParam ¶m) { +ClientProxy::Connect(const ConnectParam& param) { std::string uri = param.ip_address + ":" + param.port; channel_ = ::grpc::CreateChannel(uri, ::grpc::InsecureChannelCredentials()); @@ -53,7 +53,7 @@ ClientProxy::Connect(const ConnectParam ¶m) { } Status -ClientProxy::Connect(const std::string &uri) { +ClientProxy::Connect(const std::string& uri) { if (!UriCheck(uri)) { return Status(StatusCode::InvalidAgument, "Invalid uri"); } @@ -71,7 +71,7 @@ ClientProxy::Connected() const { try { std::string info; return client_ptr_->Cmd(info, ""); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::NotConnected, "connection lost: " + std::string(ex.what())); } } @@ -83,7 +83,7 @@ ClientProxy::Disconnect() { connected_ = false; channel_.reset(); return status; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "failed to disconnect: " + std::string(ex.what())); } } @@ -94,22 +94,22 @@ ClientProxy::ClientVersion() const { } Status -ClientProxy::CreateTable(const TableSchema ¶m) { +ClientProxy::CreateTable(const TableSchema& param) { try { ::milvus::grpc::TableSchema schema; schema.set_table_name(param.table_name); schema.set_dimension(param.dimension); schema.set_index_file_size(param.index_file_size); - schema.set_metric_type((int32_t) param.metric_type); + schema.set_metric_type((int32_t)param.metric_type); return client_ptr_->CreateTable(schema); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "failed to create table: " + std::string(ex.what())); } } bool -ClientProxy::HasTable(const std::string &table_name) { +ClientProxy::HasTable(const std::string& table_name) { Status status = Status::OK(); ::milvus::grpc::TableName grpc_table_name; grpc_table_name.set_table_name(table_name); @@ -118,34 +118,33 @@ ClientProxy::HasTable(const std::string &table_name) { } Status -ClientProxy::DropTable(const std::string &table_name) { +ClientProxy::DropTable(const std::string& table_name) { try { ::milvus::grpc::TableName grpc_table_name; grpc_table_name.set_table_name(table_name); return client_ptr_->DropTable(grpc_table_name); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "failed to drop table: " + std::string(ex.what())); } } Status -ClientProxy::CreateIndex(const IndexParam &index_param) { +ClientProxy::CreateIndex(const IndexParam& index_param) { try { - //TODO: add index params + // TODO: add index params ::milvus::grpc::IndexParam grpc_index_param; grpc_index_param.set_table_name(index_param.table_name); - grpc_index_param.mutable_index()->set_index_type((int32_t) index_param.index_type); + grpc_index_param.mutable_index()->set_index_type((int32_t)index_param.index_type); grpc_index_param.mutable_index()->set_nlist(index_param.nlist); return client_ptr_->CreateIndex(grpc_index_param); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "failed to build index: " + std::string(ex.what())); } } Status -ClientProxy::Insert(const std::string &table_name, - const std::vector &record_array, - std::vector &id_array) { +ClientProxy::Insert(const std::string& table_name, const std::vector& record_array, + std::vector& id_array) { Status status = Status::OK(); try { //////////////////////////////////////////////////////////////////////////// @@ -154,20 +153,17 @@ ClientProxy::Insert(const std::string &table_name, int thread_count = 10; std::shared_ptr<::milvus::grpc::InsertInfos> insert_info_array( - new ::milvus::grpc::InsertInfos[thread_count], - std::default_delete<::milvus::grpc::InsertInfos[]>() ); + new ::milvus::grpc::InsertInfos[thread_count], std::default_delete<::milvus::grpc::InsertInfos[]>()); - std::shared_ptr<::milvus::grpc::VectorIds> vector_ids_array( - new ::milvus::grpc::VectorIds[thread_count], - std::default_delete<::milvus::grpc::VectorIds[]>() ); + std::shared_ptr<::milvus::grpc::VectorIds> vector_ids_array(new ::milvus::grpc::VectorIds[thread_count], + std::default_delete<::milvus::grpc::VectorIds[]>()); int64_t record_count = record_array.size() / thread_count; for (size_t i = 0; i < thread_count; i++) { insert_info_array.get()[i].set_table_name(table_name); for (size_t j = i * record_count; j < record_count * (i + 1); j++) { - ::milvus::grpc::RowRecord *grpc_record = - insert_info_array.get()[i].add_row_record_array(); + ::milvus::grpc::RowRecord* grpc_record = insert_info_array.get()[i].add_row_record_array(); for (size_t k = 0; k < record_array[j].data.size(); k++) { grpc_record->add_vector_data(record_array[j].data[k]); } @@ -177,16 +173,13 @@ ClientProxy::Insert(const std::string &table_name, std::cout << "*****************************************************\n"; auto start = std::chrono::high_resolution_clock::now(); for (size_t j = 0; j < thread_count; j++) { - threads.push_back( - std::thread(&GrpcClient::InsertVector, client_ptr_, - std::ref(vector_ids_array.get()[j]), std::ref(insert_info_array.get()[j]), - std::ref(status))); + threads.push_back(std::thread(&GrpcClient::InsertVector, client_ptr_, std::ref(vector_ids_array.get()[j]), + std::ref(insert_info_array.get()[j]), std::ref(status))); } std::for_each(threads.begin(), threads.end(), std::mem_fn(&std::thread::join)); auto finish = std::chrono::high_resolution_clock::now(); - std::cout << - "InsertVector cost: " << std::chrono::duration_cast>(finish - start).count() - << "s\n"; + std::cout << "InsertVector cost: " + << std::chrono::duration_cast>(finish - start).count() << "s\n"; std::cout << "*****************************************************\n"; for (size_t i = 0; i < thread_count; i++) { @@ -198,14 +191,14 @@ ClientProxy::Insert(const std::string &table_name, ::milvus::grpc::InsertParam insert_param; insert_param.set_table_name(table_name); - for (auto &record : record_array) { - ::milvus::grpc::RowRecord *grpc_record = insert_param.add_row_record_array(); + for (auto& record : record_array) { + ::milvus::grpc::RowRecord* grpc_record = insert_param.add_row_record_array(); for (size_t i = 0; i < record.data.size(); i++) { grpc_record->add_vector_data(record.data[i]); } } - //Single thread + // Single thread ::milvus::grpc::VectorIds vector_ids; if (!id_array.empty()) { for (auto i = 0; i < id_array.size(); i++) { @@ -219,7 +212,7 @@ ClientProxy::Insert(const std::string &table_name, } } #endif - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "fail to add vector: " + std::string(ex.what())); } @@ -227,37 +220,34 @@ ClientProxy::Insert(const std::string &table_name, } Status -ClientProxy::Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) { +ClientProxy::Search(const std::string& table_name, const std::vector& query_record_array, + const std::vector& query_range_array, int64_t topk, int64_t nprobe, + std::vector& topk_query_result_array) { try { - //step 1: convert vectors data + // step 1: convert vectors data ::milvus::grpc::SearchParam search_param; search_param.set_table_name(table_name); search_param.set_topk(topk); search_param.set_nprobe(nprobe); - for (auto &record : query_record_array) { - ::milvus::grpc::RowRecord *row_record = search_param.add_query_record_array(); - for (auto &rec : record.data) { + for (auto& record : query_record_array) { + ::milvus::grpc::RowRecord* row_record = search_param.add_query_record_array(); + for (auto& rec : record.data) { row_record->add_vector_data(rec); } } - //step 2: convert range array - for (auto &range : query_range_array) { - ::milvus::grpc::Range *grpc_range = search_param.add_query_range_array(); + // step 2: convert range array + for (auto& range : query_range_array) { + ::milvus::grpc::Range* grpc_range = search_param.add_query_range_array(); grpc_range->set_start_value(range.start_value); grpc_range->set_end_value(range.end_value); } - //step 3: search vectors + // step 3: search vectors ::milvus::grpc::TopKQueryResultList topk_query_result_list; Status status = client_ptr_->Search(topk_query_result_list, search_param); - //step 4: convert result array + // step 4: convert result array for (uint64_t i = 0; i < topk_query_result_list.topk_query_result_size(); ++i) { TopKQueryResult result; for (uint64_t j = 0; j < topk_query_result_list.topk_query_result(i).query_result_arrays_size(); ++j) { @@ -270,13 +260,13 @@ ClientProxy::Search(const std::string &table_name, topk_query_result_array.emplace_back(result); } return status; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "fail to search vectors: " + std::string(ex.what())); } } Status -ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_schema) { +ClientProxy::DescribeTable(const std::string& table_name, TableSchema& table_schema) { try { ::milvus::grpc::TableSchema grpc_schema; @@ -285,27 +275,27 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch table_schema.table_name = grpc_schema.table_name(); table_schema.dimension = grpc_schema.dimension(); table_schema.index_file_size = grpc_schema.index_file_size(); - table_schema.metric_type = (MetricType) grpc_schema.metric_type(); + table_schema.metric_type = (MetricType)grpc_schema.metric_type(); return status; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "fail to describe table: " + std::string(ex.what())); } } Status -ClientProxy::CountTable(const std::string &table_name, int64_t &row_count) { +ClientProxy::CountTable(const std::string& table_name, int64_t& row_count) { try { Status status; row_count = client_ptr_->CountTable(table_name, status); return status; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "fail to show tables: " + std::string(ex.what())); } } Status -ClientProxy::ShowTables(std::vector &table_array) { +ClientProxy::ShowTables(std::vector& table_array) { try { Status status; milvus::grpc::TableNameList table_name_list; @@ -316,7 +306,7 @@ ClientProxy::ShowTables(std::vector &table_array) { table_array[i] = table_name_list.table_names(i); } return status; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "fail to show tables: " + std::string(ex.what())); } } @@ -328,7 +318,7 @@ ClientProxy::ServerVersion() const { std::string version; Status status = client_ptr_->Cmd(version, "version"); return version; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return ""; } } @@ -343,7 +333,7 @@ ClientProxy::ServerStatus() const { std::string dummy; Status status = client_ptr_->Cmd(dummy, ""); return "server alive"; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return "connection lost"; } } @@ -358,62 +348,62 @@ ClientProxy::DumpTaskTables() const { std::string dummy; Status status = client_ptr_->Cmd(dummy, "tasktable"); return dummy; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return "connection lost"; } } Status -ClientProxy::DeleteByRange(milvus::Range &range, const std::string &table_name) { +ClientProxy::DeleteByRange(milvus::Range& range, const std::string& table_name) { try { ::milvus::grpc::DeleteByRangeParam delete_by_range_param; delete_by_range_param.set_table_name(table_name); delete_by_range_param.mutable_range()->set_start_value(range.start_value); delete_by_range_param.mutable_range()->set_end_value(range.end_value); return client_ptr_->DeleteByRange(delete_by_range_param); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "fail to delete by range: " + std::string(ex.what())); } } Status -ClientProxy::PreloadTable(const std::string &table_name) const { +ClientProxy::PreloadTable(const std::string& table_name) const { try { ::milvus::grpc::TableName grpc_table_name; grpc_table_name.set_table_name(table_name); Status status = client_ptr_->PreloadTable(grpc_table_name); return status; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "fail to preload tables: " + std::string(ex.what())); } } Status -ClientProxy::DescribeIndex(const std::string &table_name, IndexParam &index_param) const { +ClientProxy::DescribeIndex(const std::string& table_name, IndexParam& index_param) const { try { ::milvus::grpc::TableName grpc_table_name; grpc_table_name.set_table_name(table_name); ::milvus::grpc::IndexParam grpc_index_param; Status status = client_ptr_->DescribeIndex(grpc_table_name, grpc_index_param); - index_param.index_type = (IndexType) (grpc_index_param.mutable_index()->index_type()); + index_param.index_type = (IndexType)(grpc_index_param.mutable_index()->index_type()); index_param.nlist = grpc_index_param.mutable_index()->nlist(); return status; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "fail to describe index: " + std::string(ex.what())); } } Status -ClientProxy::DropIndex(const std::string &table_name) const { +ClientProxy::DropIndex(const std::string& table_name) const { try { ::milvus::grpc::TableName grpc_table_name; grpc_table_name.set_table_name(table_name); Status status = client_ptr_->DropIndex(grpc_table_name); return status; - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "fail to drop index: " + std::string(ex.what())); } } -} // namespace milvus +} // namespace milvus diff --git a/cpp/src/sdk/grpc/ClientProxy.h b/cpp/src/sdk/grpc/ClientProxy.h index 044ce44847..dbeacc1380 100644 --- a/cpp/src/sdk/grpc/ClientProxy.h +++ b/cpp/src/sdk/grpc/ClientProxy.h @@ -17,12 +17,12 @@ #pragma once -#include "MilvusApi.h" #include "GrpcClient.h" +#include "MilvusApi.h" -#include -#include #include +#include +#include namespace milvus { @@ -30,10 +30,10 @@ class ClientProxy : public Connection { public: // Implementations of the Connection interface Status - Connect(const ConnectParam ¶m) override; + Connect(const ConnectParam& param) override; Status - Connect(const std::string &uri) override; + Connect(const std::string& uri) override; Status Connected() const override; @@ -42,38 +42,34 @@ class ClientProxy : public Connection { Disconnect() override; Status - CreateTable(const TableSchema ¶m) override; + CreateTable(const TableSchema& param) override; bool - HasTable(const std::string &table_name) override; + HasTable(const std::string& table_name) override; Status - DropTable(const std::string &table_name) override; + DropTable(const std::string& table_name) override; Status - CreateIndex(const IndexParam &index_param) override; + CreateIndex(const IndexParam& index_param) override; Status - Insert(const std::string &table_name, - const std::vector &record_array, - std::vector &id_array) override; + Insert(const std::string& table_name, const std::vector& record_array, + std::vector& id_array) override; Status - Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) override; + Search(const std::string& table_name, const std::vector& query_record_array, + const std::vector& query_range_array, int64_t topk, int64_t nprobe, + std::vector& topk_query_result_array) override; Status - DescribeTable(const std::string &table_name, TableSchema &table_schema) override; + DescribeTable(const std::string& table_name, TableSchema& table_schema) override; Status - CountTable(const std::string &table_name, int64_t &row_count) override; + CountTable(const std::string& table_name, int64_t& row_count) override; Status - ShowTables(std::vector &table_array) override; + ShowTables(std::vector& table_array) override; std::string ClientVersion() const override; @@ -88,17 +84,16 @@ class ClientProxy : public Connection { DumpTaskTables() const override; Status - DeleteByRange(Range &range, - const std::string &table_name) override; + DeleteByRange(Range& range, const std::string& table_name) override; Status - PreloadTable(const std::string &table_name) const override; + PreloadTable(const std::string& table_name) const override; Status - DescribeIndex(const std::string &table_name, IndexParam &index_param) const override; + DescribeIndex(const std::string& table_name, IndexParam& index_param) const override; Status - DropIndex(const std::string &table_name) const override; + DropIndex(const std::string& table_name) const override; private: std::shared_ptr<::grpc::Channel> channel_; @@ -108,4 +103,4 @@ class ClientProxy : public Connection { bool connected_ = false; }; -} // namespace milvus +} // namespace milvus diff --git a/cpp/src/sdk/grpc/GrpcClient.cpp b/cpp/src/sdk/grpc/GrpcClient.cpp index df86f0a3e7..5c27c3b73f 100644 --- a/cpp/src/sdk/grpc/GrpcClient.cpp +++ b/cpp/src/sdk/grpc/GrpcClient.cpp @@ -23,9 +23,9 @@ #include #include -#include -#include #include +#include +#include using grpc::Channel; using grpc::ClientContext; @@ -35,14 +35,14 @@ using grpc::ClientWriter; using grpc::Status; namespace milvus { -GrpcClient::GrpcClient(std::shared_ptr<::grpc::Channel> &channel) +GrpcClient::GrpcClient(std::shared_ptr<::grpc::Channel>& channel) : stub_(::milvus::grpc::MilvusService::NewStub(channel)) { } GrpcClient::~GrpcClient() = default; Status -GrpcClient::CreateTable(const ::milvus::grpc::TableSchema &table_schema) { +GrpcClient::CreateTable(const ::milvus::grpc::TableSchema& table_schema) { ClientContext context; grpc::Status response; ::grpc::Status grpc_status = stub_->CreateTable(&context, table_schema, &response); @@ -60,8 +60,7 @@ GrpcClient::CreateTable(const ::milvus::grpc::TableSchema &table_schema) { } bool -GrpcClient::HasTable(const ::milvus::grpc::TableName &table_name, - Status &status) { +GrpcClient::HasTable(const ::milvus::grpc::TableName& table_name, Status& status) { ClientContext context; ::milvus::grpc::BoolReply response; ::grpc::Status grpc_status = stub_->HasTable(&context, table_name, &response); @@ -79,7 +78,7 @@ GrpcClient::HasTable(const ::milvus::grpc::TableName &table_name, } Status -GrpcClient::DropTable(const ::milvus::grpc::TableName &table_name) { +GrpcClient::DropTable(const ::milvus::grpc::TableName& table_name) { ClientContext context; grpc::Status response; ::grpc::Status grpc_status = stub_->DropTable(&context, table_name, &response); @@ -97,7 +96,7 @@ GrpcClient::DropTable(const ::milvus::grpc::TableName &table_name) { } Status -GrpcClient::CreateIndex(const ::milvus::grpc::IndexParam &index_param) { +GrpcClient::CreateIndex(const ::milvus::grpc::IndexParam& index_param) { ClientContext context; grpc::Status response; ::grpc::Status grpc_status = stub_->CreateIndex(&context, index_param, &response); @@ -115,9 +114,8 @@ GrpcClient::CreateIndex(const ::milvus::grpc::IndexParam &index_param) { } void -GrpcClient::Insert(::milvus::grpc::VectorIds &vector_ids, - const ::milvus::grpc::InsertParam &insert_param, - Status &status) { +GrpcClient::Insert(::milvus::grpc::VectorIds& vector_ids, const ::milvus::grpc::InsertParam& insert_param, + Status& status) { ClientContext context; ::grpc::Status grpc_status = stub_->Insert(&context, insert_param, &vector_ids); @@ -136,8 +134,8 @@ GrpcClient::Insert(::milvus::grpc::VectorIds &vector_ids, } Status -GrpcClient::Search(::milvus::grpc::TopKQueryResultList &topk_query_result_list, - const ::milvus::grpc::SearchParam &search_param) { +GrpcClient::Search(::milvus::grpc::TopKQueryResultList& topk_query_result_list, + const ::milvus::grpc::SearchParam& search_param) { ::milvus::grpc::TopKQueryResult query_result; ClientContext context; ::grpc::Status grpc_status = stub_->Search(&context, search_param, &topk_query_result_list); @@ -149,16 +147,14 @@ GrpcClient::Search(::milvus::grpc::TopKQueryResultList &topk_query_result_list, } if (topk_query_result_list.status().error_code() != grpc::SUCCESS) { std::cerr << topk_query_result_list.status().reason() << std::endl; - return Status(StatusCode::ServerFailed, - topk_query_result_list.status().reason()); + return Status(StatusCode::ServerFailed, topk_query_result_list.status().reason()); } return Status::OK(); } Status -GrpcClient::DescribeTable(::milvus::grpc::TableSchema &grpc_schema, - const std::string &table_name) { +GrpcClient::DescribeTable(::milvus::grpc::TableSchema& grpc_schema, const std::string& table_name) { ClientContext context; ::milvus::grpc::TableName grpc_tablename; grpc_tablename.set_table_name(table_name); @@ -172,15 +168,14 @@ GrpcClient::DescribeTable(::milvus::grpc::TableSchema &grpc_schema, if (grpc_schema.status().error_code() != grpc::SUCCESS) { std::cerr << grpc_schema.status().reason() << std::endl; - return Status(StatusCode::ServerFailed, - grpc_schema.status().reason()); + return Status(StatusCode::ServerFailed, grpc_schema.status().reason()); } return Status::OK(); } int64_t -GrpcClient::CountTable(const std::string &table_name, Status &status) { +GrpcClient::CountTable(const std::string& table_name, Status& status) { ClientContext context; ::milvus::grpc::TableRowCount response; ::milvus::grpc::TableName grpc_tablename; @@ -204,7 +199,7 @@ GrpcClient::CountTable(const std::string &table_name, Status &status) { } Status -GrpcClient::ShowTables(milvus::grpc::TableNameList &table_name_list) { +GrpcClient::ShowTables(milvus::grpc::TableNameList& table_name_list) { ClientContext context; ::milvus::grpc::Command command; ::grpc::Status grpc_status = stub_->ShowTables(&context, command, &table_name_list); @@ -217,16 +212,14 @@ GrpcClient::ShowTables(milvus::grpc::TableNameList &table_name_list) { if (table_name_list.status().error_code() != grpc::SUCCESS) { std::cerr << table_name_list.status().reason() << std::endl; - return Status(StatusCode::ServerFailed, - table_name_list.status().reason()); + return Status(StatusCode::ServerFailed, table_name_list.status().reason()); } return Status::OK(); } Status -GrpcClient::Cmd(std::string &result, - const std::string &cmd) { +GrpcClient::Cmd(std::string& result, const std::string& cmd) { ClientContext context; ::milvus::grpc::StringReply response; ::milvus::grpc::Command command; @@ -248,7 +241,7 @@ GrpcClient::Cmd(std::string &result, } Status -GrpcClient::PreloadTable(milvus::grpc::TableName &table_name) { +GrpcClient::PreloadTable(milvus::grpc::TableName& table_name) { ClientContext context; ::milvus::grpc::Status response; ::grpc::Status grpc_status = stub_->PreloadTable(&context, table_name, &response); @@ -266,7 +259,7 @@ GrpcClient::PreloadTable(milvus::grpc::TableName &table_name) { } Status -GrpcClient::DeleteByRange(grpc::DeleteByRangeParam &delete_by_range_param) { +GrpcClient::DeleteByRange(grpc::DeleteByRangeParam& delete_by_range_param) { ClientContext context; ::milvus::grpc::Status response; ::grpc::Status grpc_status = stub_->DeleteByRange(&context, delete_by_range_param, &response); @@ -290,7 +283,7 @@ GrpcClient::Disconnect() { } Status -GrpcClient::DescribeIndex(grpc::TableName &table_name, grpc::IndexParam &index_param) { +GrpcClient::DescribeIndex(grpc::TableName& table_name, grpc::IndexParam& index_param) { ClientContext context; ::grpc::Status grpc_status = stub_->DescribeIndex(&context, table_name, &index_param); @@ -307,7 +300,7 @@ GrpcClient::DescribeIndex(grpc::TableName &table_name, grpc::IndexParam &index_p } Status -GrpcClient::DropIndex(grpc::TableName &table_name) { +GrpcClient::DropIndex(grpc::TableName& table_name) { ClientContext context; ::milvus::grpc::Status response; ::grpc::Status grpc_status = stub_->DropIndex(&context, table_name, &response); @@ -324,4 +317,4 @@ GrpcClient::DropIndex(grpc::TableName &table_name) { return Status::OK(); } -} // namespace milvus +} // namespace milvus diff --git a/cpp/src/sdk/grpc/GrpcClient.h b/cpp/src/sdk/grpc/GrpcClient.h index 8f81e83ae8..d2e6ae5095 100644 --- a/cpp/src/sdk/grpc/GrpcClient.h +++ b/cpp/src/sdk/grpc/GrpcClient.h @@ -37,56 +37,51 @@ namespace milvus { class GrpcClient { public: - explicit GrpcClient(std::shared_ptr<::grpc::Channel> &channel); + explicit GrpcClient(std::shared_ptr<::grpc::Channel>& channel); - virtual - ~GrpcClient(); + virtual ~GrpcClient(); Status - CreateTable(const grpc::TableSchema &table_schema); + CreateTable(const grpc::TableSchema& table_schema); bool - HasTable(const grpc::TableName &table_name, Status &status); + HasTable(const grpc::TableName& table_name, Status& status); Status - DropTable(const grpc::TableName &table_name); + DropTable(const grpc::TableName& table_name); Status - CreateIndex(const grpc::IndexParam &index_param); + CreateIndex(const grpc::IndexParam& index_param); void - Insert(grpc::VectorIds &vector_ids, - const grpc::InsertParam &insert_param, - Status &status); + Insert(grpc::VectorIds& vector_ids, const grpc::InsertParam& insert_param, Status& status); Status - Search(::milvus::grpc::TopKQueryResultList &topk_query_result_list, - const grpc::SearchParam &search_param); + Search(::milvus::grpc::TopKQueryResultList& topk_query_result_list, const grpc::SearchParam& search_param); Status - DescribeTable(grpc::TableSchema &grpc_schema, - const std::string &table_name); + DescribeTable(grpc::TableSchema& grpc_schema, const std::string& table_name); int64_t - CountTable(const std::string &table_name, Status &status); + CountTable(const std::string& table_name, Status& status); Status - ShowTables(milvus::grpc::TableNameList &table_name_list); + ShowTables(milvus::grpc::TableNameList& table_name_list); Status - Cmd(std::string &result, const std::string &cmd); + Cmd(std::string& result, const std::string& cmd); Status - DeleteByRange(grpc::DeleteByRangeParam &delete_by_range_param); + DeleteByRange(grpc::DeleteByRangeParam& delete_by_range_param); Status - PreloadTable(grpc::TableName &table_name); + PreloadTable(grpc::TableName& table_name); Status - DescribeIndex(grpc::TableName &table_name, grpc::IndexParam &index_param); + DescribeIndex(grpc::TableName& table_name, grpc::IndexParam& index_param); Status - DropIndex(grpc::TableName &table_name); + DropIndex(grpc::TableName& table_name); Status Disconnect(); @@ -95,4 +90,4 @@ class GrpcClient { std::unique_ptr stub_; }; -} // namespace milvus +} // namespace milvus diff --git a/cpp/src/sdk/include/MilvusApi.h b/cpp/src/sdk/include/MilvusApi.h index e6025fd52e..9425ef3be3 100644 --- a/cpp/src/sdk/include/MilvusApi.h +++ b/cpp/src/sdk/include/MilvusApi.h @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "Status.h" +#include #include #include -#include /** \brief Milvus SDK namespace */ @@ -48,18 +47,18 @@ enum class MetricType { * @brief Connect API parameter */ struct ConnectParam { - std::string ip_address; ///< Server IP address - std::string port; ///< Server PORT + std::string ip_address; ///< Server IP address + std::string port; ///< Server PORT }; /** * @brief Table Schema */ struct TableSchema { - std::string table_name; ///< Table name - int64_t dimension = 0; ///< Vector dimension, must be a positive value - int64_t index_file_size = 0; ///< Index file size, must be a positive value - MetricType metric_type = MetricType::L2; ///< Index metric type + std::string table_name; ///< Table name + int64_t dimension = 0; ///< Vector dimension, must be a positive value + int64_t index_file_size = 0; ///< Index file size, must be a positive value + MetricType metric_type = MetricType::L2; ///< Index metric type }; /** @@ -67,30 +66,30 @@ struct TableSchema { * for DATE partition, the format is like: 'year-month-day' */ struct Range { - std::string start_value; ///< Range start - std::string end_value; ///< Range stop + std::string start_value; ///< Range start + std::string end_value; ///< Range stop }; /** * @brief Record inserted */ struct RowRecord { - std::vector data; ///< Vector raw data + std::vector data; ///< Vector raw data }; /** * @brief Query result */ struct QueryResult { - int64_t id; ///< Output result - double distance; ///< Vector similarity distance + int64_t id; ///< Output result + double distance; ///< Vector similarity distance }; /** * @brief TopK query result */ struct TopKQueryResult { - std::vector query_result_arrays; ///< TopK query result + std::vector query_result_arrays; ///< TopK query result }; /** @@ -129,7 +128,7 @@ class Connection { */ static Status - Destroy(std::shared_ptr &connection_ptr); + Destroy(std::shared_ptr& connection_ptr); /** * @brief Connect @@ -143,7 +142,7 @@ class Connection { */ virtual Status - Connect(const ConnectParam ¶m) = 0; + Connect(const ConnectParam& param) = 0; /** * @brief Connect @@ -156,7 +155,7 @@ class Connection { * @return Indicate if connect is successful */ virtual Status - Connect(const std::string &uri) = 0; + Connect(const std::string& uri) = 0; /** * @brief connected @@ -188,7 +187,7 @@ class Connection { * @return Indicate if table is created successfully */ virtual Status - CreateTable(const TableSchema ¶m) = 0; + CreateTable(const TableSchema& param) = 0; /** * @brief Test table existence method @@ -200,7 +199,7 @@ class Connection { * @return Indicate if table is cexist */ virtual bool - HasTable(const std::string &table_name) = 0; + HasTable(const std::string& table_name) = 0; /** * @brief Delete table method @@ -212,7 +211,7 @@ class Connection { * @return Indicate if table is delete successfully. */ virtual Status - DropTable(const std::string &table_name) = 0; + DropTable(const std::string& table_name) = 0; /** * @brief Create index method @@ -228,7 +227,7 @@ class Connection { * @return Indicate if build index successfully. */ virtual Status - CreateIndex(const IndexParam &index_param) = 0; + CreateIndex(const IndexParam& index_param) = 0; /** * @brief Add vector to table @@ -242,9 +241,8 @@ class Connection { * @return Indicate if vector array are inserted successfully */ virtual Status - Insert(const std::string &table_name, - const std::vector &record_array, - std::vector &id_array) = 0; + Insert(const std::string& table_name, const std::vector& record_array, + std::vector& id_array) = 0; /** * @brief Search vector @@ -260,12 +258,9 @@ class Connection { * @return Indicate if query is successful. */ virtual Status - Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) = 0; + Search(const std::string& table_name, const std::vector& query_record_array, + const std::vector& query_range_array, int64_t topk, int64_t nprobe, + std::vector& topk_query_result_array) = 0; /** * @brief Show table description @@ -278,7 +273,7 @@ class Connection { * @return Indicate if this operation is successful. */ virtual Status - DescribeTable(const std::string &table_name, TableSchema &table_schema) = 0; + DescribeTable(const std::string& table_name, TableSchema& table_schema) = 0; /** * @brief Get table row count @@ -291,8 +286,7 @@ class Connection { * @return Indicate if this operation is successful. */ virtual Status - CountTable(const std::string &table_name, - int64_t &row_count) = 0; + CountTable(const std::string& table_name, int64_t& row_count) = 0; /** * @brief Show all tables in database @@ -304,7 +298,7 @@ class Connection { * @return Indicate if this operation is successful. */ virtual Status - ShowTables(std::vector &table_array) = 0; + ShowTables(std::vector& table_array) = 0; /** * @brief Give the client version @@ -350,8 +344,7 @@ class Connection { * @return Indicate if this operation is successful. */ virtual Status - DeleteByRange(Range &range, - const std::string &table_name) = 0; + DeleteByRange(Range& range, const std::string& table_name) = 0; /** * @brief preload table @@ -363,7 +356,7 @@ class Connection { * @return Indicate if this operation is successful. */ virtual Status - PreloadTable(const std::string &table_name) const = 0; + PreloadTable(const std::string& table_name) const = 0; /** * @brief describe index @@ -375,7 +368,7 @@ class Connection { * @return index informations and indicate if this operation is successful. */ virtual Status - DescribeIndex(const std::string &table_name, IndexParam &index_param) const = 0; + DescribeIndex(const std::string& table_name, IndexParam& index_param) const = 0; /** * @brief drop index @@ -387,7 +380,7 @@ class Connection { * @return Indicate if this operation is successful. */ virtual Status - DropIndex(const std::string &table_name) const = 0; + DropIndex(const std::string& table_name) const = 0; }; -} // namespace milvus +} // namespace milvus diff --git a/cpp/src/sdk/include/Status.h b/cpp/src/sdk/include/Status.h index 670d9662b4..e9b267c5f7 100644 --- a/cpp/src/sdk/include/Status.h +++ b/cpp/src/sdk/include/Status.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include @@ -46,19 +45,19 @@ enum class StatusCode { */ class Status { public: - Status(StatusCode code, const std::string &msg); + Status(StatusCode code, const std::string& msg); Status(); ~Status(); - Status(const Status &s); + Status(const Status& s); - Status & - operator=(const Status &s); + Status& + operator=(const Status& s); - Status(Status &&s); + Status(Status&& s); - Status & - operator=(Status &&s); + Status& + operator=(Status&& s); static Status OK() { @@ -72,7 +71,7 @@ class Status { StatusCode code() const { - return (state_ == nullptr) ? StatusCode::OK : *(StatusCode *) (state_); + return (state_ == nullptr) ? StatusCode::OK : *(StatusCode*)(state_); } std::string @@ -80,13 +79,13 @@ class Status { private: inline void - CopyFrom(const Status &s); + CopyFrom(const Status& s); inline void - MoveFrom(Status &s); + MoveFrom(Status& s); private: - const char *state_ = nullptr; -}; // Status + const char* state_ = nullptr; +}; // Status -} // namespace milvus +} // namespace milvus diff --git a/cpp/src/sdk/interface/ConnectionImpl.cpp b/cpp/src/sdk/interface/ConnectionImpl.cpp index 5a01cd1a79..7034ce4a4d 100644 --- a/cpp/src/sdk/interface/ConnectionImpl.cpp +++ b/cpp/src/sdk/interface/ConnectionImpl.cpp @@ -25,7 +25,7 @@ Connection::Create() { } Status -Connection::Destroy(std::shared_ptr &connection_ptr) { +Connection::Destroy(std::shared_ptr& connection_ptr) { if (connection_ptr != nullptr) { return connection_ptr->Disconnect(); } @@ -38,12 +38,12 @@ ConnectionImpl::ConnectionImpl() { } Status -ConnectionImpl::Connect(const ConnectParam ¶m) { +ConnectionImpl::Connect(const ConnectParam& param) { return client_proxy_->Connect(param); } Status -ConnectionImpl::Connect(const std::string &uri) { +ConnectionImpl::Connect(const std::string& uri) { return client_proxy_->Connect(uri); } @@ -63,55 +63,51 @@ ConnectionImpl::ClientVersion() const { } Status -ConnectionImpl::CreateTable(const TableSchema ¶m) { +ConnectionImpl::CreateTable(const TableSchema& param) { return client_proxy_->CreateTable(param); } bool -ConnectionImpl::HasTable(const std::string &table_name) { +ConnectionImpl::HasTable(const std::string& table_name) { return client_proxy_->HasTable(table_name); } Status -ConnectionImpl::DropTable(const std::string &table_name) { +ConnectionImpl::DropTable(const std::string& table_name) { return client_proxy_->DropTable(table_name); } Status -ConnectionImpl::CreateIndex(const IndexParam &index_param) { +ConnectionImpl::CreateIndex(const IndexParam& index_param) { return client_proxy_->CreateIndex(index_param); } Status -ConnectionImpl::Insert(const std::string &table_name, - const std::vector &record_array, - std::vector &id_array) { +ConnectionImpl::Insert(const std::string& table_name, const std::vector& record_array, + std::vector& id_array) { return client_proxy_->Insert(table_name, record_array, id_array); } Status -ConnectionImpl::Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) { - return client_proxy_->Search(table_name, query_record_array, query_range_array, topk, - nprobe, topk_query_result_array); +ConnectionImpl::Search(const std::string& table_name, const std::vector& query_record_array, + const std::vector& query_range_array, int64_t topk, int64_t nprobe, + std::vector& topk_query_result_array) { + return client_proxy_->Search(table_name, query_record_array, query_range_array, topk, nprobe, + topk_query_result_array); } Status -ConnectionImpl::DescribeTable(const std::string &table_name, TableSchema &table_schema) { +ConnectionImpl::DescribeTable(const std::string& table_name, TableSchema& table_schema) { return client_proxy_->DescribeTable(table_name, table_schema); } Status -ConnectionImpl::CountTable(const std::string &table_name, int64_t &row_count) { +ConnectionImpl::CountTable(const std::string& table_name, int64_t& row_count) { return client_proxy_->CountTable(table_name, row_count); } Status -ConnectionImpl::ShowTables(std::vector &table_array) { +ConnectionImpl::ShowTables(std::vector& table_array) { return client_proxy_->ShowTables(table_array); } @@ -131,24 +127,23 @@ ConnectionImpl::DumpTaskTables() const { } Status -ConnectionImpl::DeleteByRange(Range &range, - const std::string &table_name) { +ConnectionImpl::DeleteByRange(Range& range, const std::string& table_name) { return client_proxy_->DeleteByRange(range, table_name); } Status -ConnectionImpl::PreloadTable(const std::string &table_name) const { +ConnectionImpl::PreloadTable(const std::string& table_name) const { return client_proxy_->PreloadTable(table_name); } Status -ConnectionImpl::DescribeIndex(const std::string &table_name, IndexParam &index_param) const { +ConnectionImpl::DescribeIndex(const std::string& table_name, IndexParam& index_param) const { return client_proxy_->DescribeIndex(table_name, index_param); } Status -ConnectionImpl::DropIndex(const std::string &table_name) const { +ConnectionImpl::DropIndex(const std::string& table_name) const { return client_proxy_->DropIndex(table_name); } -} // namespace milvus +} // namespace milvus diff --git a/cpp/src/sdk/interface/ConnectionImpl.h b/cpp/src/sdk/interface/ConnectionImpl.h index 7d5e090882..6bc3432bc4 100644 --- a/cpp/src/sdk/interface/ConnectionImpl.h +++ b/cpp/src/sdk/interface/ConnectionImpl.h @@ -20,9 +20,9 @@ #include "MilvusApi.h" #include "sdk/grpc/ClientProxy.h" -#include #include #include +#include namespace milvus { @@ -32,10 +32,10 @@ class ConnectionImpl : public Connection { // Implementations of the Connection interface Status - Connect(const ConnectParam ¶m) override; + Connect(const ConnectParam& param) override; Status - Connect(const std::string &uri) override; + Connect(const std::string& uri) override; Status Connected() const override; @@ -44,38 +44,34 @@ class ConnectionImpl : public Connection { Disconnect() override; Status - CreateTable(const TableSchema ¶m) override; + CreateTable(const TableSchema& param) override; bool - HasTable(const std::string &table_name) override; + HasTable(const std::string& table_name) override; Status - DropTable(const std::string &table_name) override; + DropTable(const std::string& table_name) override; Status - CreateIndex(const IndexParam &index_param) override; + CreateIndex(const IndexParam& index_param) override; Status - Insert(const std::string &table_name, - const std::vector &record_array, - std::vector &id_array) override; + Insert(const std::string& table_name, const std::vector& record_array, + std::vector& id_array) override; Status - Search(const std::string &table_name, - const std::vector &query_record_array, - const std::vector &query_range_array, - int64_t topk, - int64_t nprobe, - std::vector &topk_query_result_array) override; + Search(const std::string& table_name, const std::vector& query_record_array, + const std::vector& query_range_array, int64_t topk, int64_t nprobe, + std::vector& topk_query_result_array) override; Status - DescribeTable(const std::string &table_name, TableSchema &table_schema) override; + DescribeTable(const std::string& table_name, TableSchema& table_schema) override; Status - CountTable(const std::string &table_name, int64_t &row_count) override; + CountTable(const std::string& table_name, int64_t& row_count) override; Status - ShowTables(std::vector &table_array) override; + ShowTables(std::vector& table_array) override; std::string ClientVersion() const override; @@ -90,20 +86,19 @@ class ConnectionImpl : public Connection { DumpTaskTables() const override; Status - DeleteByRange(Range &range, - const std::string &table_name) override; + DeleteByRange(Range& range, const std::string& table_name) override; Status - PreloadTable(const std::string &table_name) const override; + PreloadTable(const std::string& table_name) const override; Status - DescribeIndex(const std::string &table_name, IndexParam &index_param) const override; + DescribeIndex(const std::string& table_name, IndexParam& index_param) const override; Status - DropIndex(const std::string &table_name) const override; + DropIndex(const std::string& table_name) const override; private: std::shared_ptr client_proxy_; }; -} // namespace milvus +} // namespace milvus diff --git a/cpp/src/sdk/interface/Status.cpp b/cpp/src/sdk/interface/Status.cpp index a8780f2ddd..cd46c8c245 100644 --- a/cpp/src/sdk/interface/Status.cpp +++ b/cpp/src/sdk/interface/Status.cpp @@ -23,12 +23,12 @@ namespace milvus { constexpr int CODE_WIDTH = sizeof(StatusCode); -Status::Status(StatusCode code, const std::string &msg) { - //4 bytes store code - //4 bytes store message length - //the left bytes store message string - const uint32_t length = (uint32_t) msg.size(); - char *result = new char[length + sizeof(length) + CODE_WIDTH]; +Status::Status(StatusCode code, const std::string& msg) { + // 4 bytes store code + // 4 bytes store message length + // the left bytes store message string + const uint32_t length = (uint32_t)msg.size(); + char* result = new char[length + sizeof(length) + CODE_WIDTH]; memcpy(result, &code, CODE_WIDTH); memcpy(result + CODE_WIDTH, &length, sizeof(length)); memcpy(result + sizeof(length) + CODE_WIDTH, msg.data(), length); @@ -36,38 +36,35 @@ Status::Status(StatusCode code, const std::string &msg) { state_ = result; } -Status::Status() - : state_(nullptr) { +Status::Status() : state_(nullptr) { } Status::~Status() { delete state_; } -Status::Status(const Status &s) - : state_(nullptr) { +Status::Status(const Status& s) : state_(nullptr) { CopyFrom(s); } -Status & -Status::operator=(const Status &s) { +Status& +Status::operator=(const Status& s) { CopyFrom(s); return *this; } -Status::Status(Status &&s) - : state_(nullptr) { +Status::Status(Status&& s) : state_(nullptr) { MoveFrom(s); } -Status & -Status::operator=(Status &&s) { +Status& +Status::operator=(Status&& s) { MoveFrom(s); return *this; } void -Status::CopyFrom(const Status &s) { +Status::CopyFrom(const Status& s) { delete state_; state_ = nullptr; if (s.state_ == nullptr) { @@ -78,11 +75,11 @@ Status::CopyFrom(const Status &s) { memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length)); int buff_len = length + sizeof(length) + CODE_WIDTH; state_ = new char[buff_len]; - memcpy((void *) state_, (void *) s.state_, buff_len); + memcpy((void*)state_, (void*)s.state_, buff_len); } void -Status::MoveFrom(Status &s) { +Status::MoveFrom(Status& s) { delete state_; state_ = s.state_; s.state_ = nullptr; @@ -104,6 +101,4 @@ Status::message() const { return msg; } -} // namespace milvus - - +} // namespace milvus diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 0ad122fcad..788f792feb 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -17,14 +17,14 @@ #include "server/Config.h" +#include #include #include #include -#include -#include #include -#include +#include #include +#include #include "config/ConfigMgr.h" #include "utils/CommonUtil.h" @@ -36,14 +36,14 @@ namespace server { constexpr uint64_t GB = 1UL << 30; -Config & +Config& Config::GetInstance() { static Config config_inst; return config_inst; } Status -Config::LoadConfigFile(const std::string &filename) { +Config::LoadConfigFile(const std::string& filename) { if (filename.empty()) { std::cerr << "ERROR: need specify config file" << std::endl; exit(1); @@ -56,14 +56,13 @@ Config::LoadConfigFile(const std::string &filename) { } try { - ConfigMgr *mgr = const_cast(ConfigMgr::GetInstance()); + ConfigMgr* mgr = const_cast(ConfigMgr::GetInstance()); ErrorCode err = mgr->LoadConfigFile(filename); if (err != 0) { std::cerr << "Server failed to load config file: " << filename << std::endl; exit(1); } - } - catch (YAML::Exception &e) { + } catch (YAML::Exception& e) { std::cerr << "Server failed to load config file: " << filename << std::endl; exit(1); } @@ -256,7 +255,7 @@ Config::ResetDefaultConfig() { } void -Config::PrintConfigSection(const std::string &config_node_name) { +Config::PrintConfigSection(const std::string& config_node_name) { std::cout << std::endl; std::cout << config_node_name << ":" << std::endl; if (config_map_.find(config_node_name) != config_map_.end()) { @@ -278,7 +277,7 @@ Config::PrintAll() { //////////////////////////////////////////////////////////////////////////////// Status -Config::CheckServerConfigAddress(const std::string &value) { +Config::CheckServerConfigAddress(const std::string& value) { if (!ValidationUtil::ValidateIpAddress(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid server config address: " + value); } @@ -286,7 +285,7 @@ Config::CheckServerConfigAddress(const std::string &value) { } Status -Config::CheckServerConfigPort(const std::string &value) { +Config::CheckServerConfigPort(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid server config port: " + value); } else { @@ -299,10 +298,8 @@ Config::CheckServerConfigPort(const std::string &value) { } Status -Config::CheckServerConfigDeployMode(const std::string &value) { - if (value != "single" && - value != "cluster_readonly" && - value != "cluster_writable") { +Config::CheckServerConfigDeployMode(const std::string& value) { + if (value != "single" && value != "cluster_readonly" && value != "cluster_writable") { return Status(SERVER_INVALID_ARGUMENT, "Invalid server config mode [single, cluster_readonly, cluster_writable]: " + value); } @@ -310,7 +307,7 @@ Config::CheckServerConfigDeployMode(const std::string &value) { } Status -Config::CheckServerConfigTimeZone(const std::string &value) { +Config::CheckServerConfigTimeZone(const std::string& value) { if (value.length() <= 3) { return Status(SERVER_INVALID_ARGUMENT, "Invalid server config time_zone: " + value); } else { @@ -328,7 +325,7 @@ Config::CheckServerConfigTimeZone(const std::string &value) { } Status -Config::CheckDBConfigPrimaryPath(const std::string &value) { +Config::CheckDBConfigPrimaryPath(const std::string& value) { if (value.empty()) { return Status(SERVER_INVALID_ARGUMENT, "DB config primary_path empty"); } @@ -336,12 +333,12 @@ Config::CheckDBConfigPrimaryPath(const std::string &value) { } Status -Config::CheckDBConfigSecondaryPath(const std::string &value) { +Config::CheckDBConfigSecondaryPath(const std::string& value) { return Status::OK(); } Status -Config::CheckDBConfigBackendUrl(const std::string &value) { +Config::CheckDBConfigBackendUrl(const std::string& value) { if (!ValidationUtil::ValidateDbURI(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config backend_url: " + value); } @@ -349,7 +346,7 @@ Config::CheckDBConfigBackendUrl(const std::string &value) { } Status -Config::CheckDBConfigArchiveDiskThreshold(const std::string &value) { +Config::CheckDBConfigArchiveDiskThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config archive_disk_threshold: " + value); } @@ -357,7 +354,7 @@ Config::CheckDBConfigArchiveDiskThreshold(const std::string &value) { } Status -Config::CheckDBConfigArchiveDaysThreshold(const std::string &value) { +Config::CheckDBConfigArchiveDaysThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config archive_days_threshold: " + value); } @@ -365,7 +362,7 @@ Config::CheckDBConfigArchiveDaysThreshold(const std::string &value) { } Status -Config::CheckDBConfigInsertBufferSize(const std::string &value) { +Config::CheckDBConfigInsertBufferSize(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config insert_buffer_size: " + value); } else { @@ -380,7 +377,7 @@ Config::CheckDBConfigInsertBufferSize(const std::string &value) { } Status -Config::CheckDBConfigBuildIndexGPU(const std::string &value) { +Config::CheckDBConfigBuildIndexGPU(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config build_index_gpu: " + value); } else { @@ -393,7 +390,7 @@ Config::CheckDBConfigBuildIndexGPU(const std::string &value) { } Status -Config::CheckMetricConfigEnableMonitor(const std::string &value) { +Config::CheckMetricConfigEnableMonitor(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config auto_bootup: " + value); } @@ -401,7 +398,7 @@ Config::CheckMetricConfigEnableMonitor(const std::string &value) { } Status -Config::CheckMetricConfigCollector(const std::string &value) { +Config::CheckMetricConfigCollector(const std::string& value) { if (value != "prometheus") { return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config collector: " + value); } @@ -409,7 +406,7 @@ Config::CheckMetricConfigCollector(const std::string &value) { } Status -Config::CheckMetricConfigPrometheusPort(const std::string &value) { +Config::CheckMetricConfigPrometheusPort(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config prometheus_port: " + value); } @@ -417,7 +414,7 @@ Config::CheckMetricConfigPrometheusPort(const std::string &value) { } Status -Config::CheckCacheConfigCpuCacheCapacity(const std::string &value) { +Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_cache_capacity: " + value); } else { @@ -426,7 +423,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string &value) { CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (cpu_cache_capacity >= total_mem) { return Status(SERVER_INVALID_ARGUMENT, "Cache config cpu_cache_capacity exceed system memory: " + value); - } else if (cpu_cache_capacity > (double) total_mem * 0.9) { + } else if (cpu_cache_capacity > (double)total_mem * 0.9) { std::cerr << "Warning: cpu_cache_capacity value is too big" << std::endl; } @@ -442,7 +439,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string &value) { } Status -Config::CheckCacheConfigCpuCacheThreshold(const std::string &value) { +Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_cache_threshold: " + value); } else { @@ -455,7 +452,7 @@ Config::CheckCacheConfigCpuCacheThreshold(const std::string &value) { } Status -Config::CheckCacheConfigGpuCacheCapacity(const std::string &value) { +Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_cache_capacity: " + value); } else { @@ -470,7 +467,7 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string &value) { } else if (gpu_cache_capacity >= gpu_memory) { return Status(SERVER_INVALID_ARGUMENT, "Cache config gpu_cache_capacity exceed GPU memory: " + std::to_string(gpu_memory)); - } else if (gpu_cache_capacity > (double) gpu_memory * 0.9) { + } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { std::cerr << "Warning: gpu_cache_capacity value is too big" << std::endl; } } @@ -478,7 +475,7 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string &value) { } Status -Config::CheckCacheConfigGpuCacheThreshold(const std::string &value) { +Config::CheckCacheConfigGpuCacheThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_cache_threshold: " + value); } else { @@ -491,7 +488,7 @@ Config::CheckCacheConfigGpuCacheThreshold(const std::string &value) { } Status -Config::CheckCacheConfigCacheInsertData(const std::string &value) { +Config::CheckCacheConfigCacheInsertData(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cache_insert_data: " + value); } @@ -499,7 +496,7 @@ Config::CheckCacheConfigCacheInsertData(const std::string &value) { } Status -Config::CheckEngineConfigUseBlasThreshold(const std::string &value) { +Config::CheckEngineConfigUseBlasThreshold(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config use_blas_threshold: " + value); } @@ -507,7 +504,7 @@ Config::CheckEngineConfigUseBlasThreshold(const std::string &value) { } Status -Config::CheckEngineConfigOmpThreadNum(const std::string &value) { +Config::CheckEngineConfigOmpThreadNum(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config omp_thread_num: " + value); } else { @@ -521,7 +518,7 @@ Config::CheckEngineConfigOmpThreadNum(const std::string &value) { } Status -Config::CheckResourceConfigMode(const std::string &value) { +Config::CheckResourceConfigMode(const std::string& value) { if (value != "simple") { return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config mode: " + value); } @@ -529,7 +526,7 @@ Config::CheckResourceConfigMode(const std::string &value) { } Status -Config::CheckResourceConfigPool(const std::vector &value) { +Config::CheckResourceConfigPool(const std::vector& value) { if (value.empty()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config pool"); } @@ -537,17 +534,15 @@ Config::CheckResourceConfigPool(const std::vector &value) { } //////////////////////////////////////////////////////////////////////////////// -ConfigNode & -Config::GetConfigNode(const std::string &name) { - ConfigMgr *mgr = ConfigMgr::GetInstance(); - ConfigNode &root_node = mgr->GetRootNode(); +ConfigNode& +Config::GetConfigNode(const std::string& name) { + ConfigMgr* mgr = ConfigMgr::GetInstance(); + ConfigNode& root_node = mgr->GetRootNode(); return root_node.GetChild(name); } Status -Config::GetConfigValueInMem(const std::string &parent_key, - const std::string &child_key, - std::string &value) { +Config::GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value) { std::lock_guard lock(mutex_); if (config_map_.find(parent_key) != config_map_.end() && config_map_[parent_key].find(child_key) != config_map_[parent_key].end()) { @@ -559,9 +554,7 @@ Config::GetConfigValueInMem(const std::string &parent_key, } void -Config::SetConfigValueInMem(const std::string &parent_key, - const std::string &child_key, - const std::string &value) { +Config::SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, const std::string& value) { std::lock_guard lock(mutex_); config_map_[parent_key][child_key] = value; } @@ -572,8 +565,7 @@ std::string Config::GetServerConfigStrAddress() { std::string value; if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_ADDRESS, - CONFIG_SERVER_ADDRESS_DEFAULT); + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); } return value; @@ -583,8 +575,7 @@ std::string Config::GetServerConfigStrPort() { std::string value; if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_PORT, - CONFIG_SERVER_PORT_DEFAULT); + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_PORT, CONFIG_SERVER_PORT_DEFAULT); SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); } return value; @@ -594,8 +585,7 @@ std::string Config::GetServerConfigStrDeployMode() { std::string value; if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_DEPLOY_MODE, - CONFIG_SERVER_DEPLOY_MODE_DEFAULT); + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_DEPLOY_MODE, CONFIG_SERVER_DEPLOY_MODE_DEFAULT); SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); } return value; @@ -605,8 +595,7 @@ std::string Config::GetServerConfigStrTimeZone() { std::string value; if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_TIME_ZONE, - CONFIG_SERVER_TIME_ZONE_DEFAULT); + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); } return value; @@ -618,8 +607,7 @@ std::string Config::GetDBConfigStrPrimaryPath() { std::string value; if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PRIMARY_PATH, - CONFIG_DB_PRIMARY_PATH_DEFAULT); + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PRIMARY_PATH, CONFIG_DB_PRIMARY_PATH_DEFAULT); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); } return value; @@ -629,8 +617,7 @@ std::string Config::GetDBConfigStrSecondaryPath() { std::string value; if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SECONDARY_PATH, - CONFIG_DB_SECONDARY_PATH_DEFAULT); + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SECONDARY_PATH, CONFIG_DB_SECONDARY_PATH_DEFAULT); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); } return value; @@ -640,8 +627,7 @@ std::string Config::GetDBConfigStrBackendUrl() { std::string value; if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BACKEND_URL, - CONFIG_DB_BACKEND_URL_DEFAULT); + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BACKEND_URL, CONFIG_DB_BACKEND_URL_DEFAULT); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); } return value; @@ -673,8 +659,7 @@ std::string Config::GetDBConfigStrInsertBufferSize() { std::string value; if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_INSERT_BUFFER_SIZE, - CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); } return value; @@ -684,8 +669,7 @@ std::string Config::GetDBConfigStrBuildIndexGPU() { std::string value; if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUILD_INDEX_GPU, - CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); } return value; @@ -697,8 +681,8 @@ std::string Config::GetMetricConfigStrEnableMonitor() { std::string value; if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value).ok()) { - value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_ENABLE_MONITOR, - CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); + value = + GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_ENABLE_MONITOR, CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value); } return value; @@ -708,8 +692,7 @@ std::string Config::GetMetricConfigStrCollector() { std::string value; if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value).ok()) { - value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_COLLECTOR, - CONFIG_METRIC_COLLECTOR_DEFAULT); + value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_COLLECTOR, CONFIG_METRIC_COLLECTOR_DEFAULT); SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); } return value; @@ -719,8 +702,8 @@ std::string Config::GetMetricConfigStrPrometheusPort() { std::string value; if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value).ok()) { - value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, - CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); + value = + GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); } return value; @@ -732,8 +715,8 @@ std::string Config::GetCacheConfigStrCpuCacheCapacity() { std::string value; if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value).ok()) { - value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_CACHE_CAPACITY, - CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT); + value = GetConfigNode(CONFIG_CACHE) + .GetValue(CONFIG_CACHE_CPU_CACHE_CAPACITY, CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT); SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value); } return value; @@ -743,8 +726,8 @@ std::string Config::GetCacheConfigStrCpuCacheThreshold() { std::string value; if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value).ok()) { - value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_CACHE_THRESHOLD, - CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT); + value = GetConfigNode(CONFIG_CACHE) + .GetValue(CONFIG_CACHE_CPU_CACHE_THRESHOLD, CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT); SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); } return value; @@ -754,8 +737,8 @@ std::string Config::GetCacheConfigStrGpuCacheCapacity() { std::string value; if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value).ok()) { - value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_CACHE_CAPACITY, - CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); + value = GetConfigNode(CONFIG_CACHE) + .GetValue(CONFIG_CACHE_GPU_CACHE_CAPACITY, CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value); } return value; @@ -765,8 +748,8 @@ std::string Config::GetCacheConfigStrGpuCacheThreshold() { std::string value; if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value).ok()) { - value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_CACHE_THRESHOLD, - CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); + value = GetConfigNode(CONFIG_CACHE) + .GetValue(CONFIG_CACHE_GPU_CACHE_THRESHOLD, CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value); } return value; @@ -776,8 +759,8 @@ std::string Config::GetCacheConfigStrCacheInsertData() { std::string value; if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value).ok()) { - value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, - CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); + value = GetConfigNode(CONFIG_CACHE) + .GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); } return value; @@ -789,8 +772,8 @@ std::string Config::GetEngineConfigStrUseBlasThreshold() { std::string value; if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value).ok()) { - value = GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_USE_BLAS_THRESHOLD, - CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); + value = GetConfigNode(CONFIG_ENGINE) + .GetValue(CONFIG_ENGINE_USE_BLAS_THRESHOLD, CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value); } return value; @@ -800,8 +783,8 @@ std::string Config::GetEngineConfigStrOmpThreadNum() { std::string value; if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value).ok()) { - value = GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, - CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); + value = + GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); } return value; @@ -813,8 +796,7 @@ std::string Config::GetResourceConfigStrMode() { std::string value; if (!GetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value).ok()) { - value = GetConfigNode(CONFIG_RESOURCE).GetValue(CONFIG_RESOURCE_MODE, - CONFIG_RESOURCE_MODE_DEFAULT); + value = GetConfigNode(CONFIG_RESOURCE).GetValue(CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); } return value; @@ -822,49 +804,49 @@ Config::GetResourceConfigStrMode() { //////////////////////////////////////////////////////////////////////////////// Status -Config::GetServerConfigAddress(std::string &value) { +Config::GetServerConfigAddress(std::string& value) { value = GetServerConfigStrAddress(); return CheckServerConfigAddress(value); } Status -Config::GetServerConfigPort(std::string &value) { +Config::GetServerConfigPort(std::string& value) { value = GetServerConfigStrPort(); return CheckServerConfigPort(value); } Status -Config::GetServerConfigDeployMode(std::string &value) { +Config::GetServerConfigDeployMode(std::string& value) { value = GetServerConfigStrDeployMode(); return CheckServerConfigDeployMode(value); } Status -Config::GetServerConfigTimeZone(std::string &value) { +Config::GetServerConfigTimeZone(std::string& value) { value = GetServerConfigStrTimeZone(); return CheckServerConfigTimeZone(value); } Status -Config::GetDBConfigPrimaryPath(std::string &value) { +Config::GetDBConfigPrimaryPath(std::string& value) { value = GetDBConfigStrPrimaryPath(); return CheckDBConfigPrimaryPath(value); } Status -Config::GetDBConfigSecondaryPath(std::string &value) { +Config::GetDBConfigSecondaryPath(std::string& value) { value = GetDBConfigStrSecondaryPath(); return Status::OK(); } Status -Config::GetDBConfigBackendUrl(std::string &value) { +Config::GetDBConfigBackendUrl(std::string& value) { value = GetDBConfigStrBackendUrl(); return CheckDBConfigBackendUrl(value); } Status -Config::GetDBConfigArchiveDiskThreshold(int32_t &value) { +Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { std::string str = GetDBConfigStrArchiveDiskThreshold(); Status s = CheckDBConfigArchiveDiskThreshold(str); if (!s.ok()) return s; @@ -873,7 +855,7 @@ Config::GetDBConfigArchiveDiskThreshold(int32_t &value) { } Status -Config::GetDBConfigArchiveDaysThreshold(int32_t &value) { +Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { std::string str = GetDBConfigStrArchiveDaysThreshold(); Status s = CheckDBConfigArchiveDaysThreshold(str); if (!s.ok()) return s; @@ -882,7 +864,7 @@ Config::GetDBConfigArchiveDaysThreshold(int32_t &value) { } Status -Config::GetDBConfigInsertBufferSize(int32_t &value) { +Config::GetDBConfigInsertBufferSize(int32_t& value) { std::string str = GetDBConfigStrInsertBufferSize(); Status s = CheckDBConfigInsertBufferSize(str); if (!s.ok()) return s; @@ -891,7 +873,7 @@ Config::GetDBConfigInsertBufferSize(int32_t &value) { } Status -Config::GetDBConfigBuildIndexGPU(int32_t &value) { +Config::GetDBConfigBuildIndexGPU(int32_t& value) { std::string str = GetDBConfigStrBuildIndexGPU(); Status s = CheckDBConfigBuildIndexGPU(str); if (!s.ok()) return s; @@ -900,7 +882,7 @@ Config::GetDBConfigBuildIndexGPU(int32_t &value) { } Status -Config::GetMetricConfigEnableMonitor(bool &value) { +Config::GetMetricConfigEnableMonitor(bool& value) { std::string str = GetMetricConfigStrEnableMonitor(); Status s = CheckMetricConfigEnableMonitor(str); if (!s.ok()) return s; @@ -910,19 +892,19 @@ Config::GetMetricConfigEnableMonitor(bool &value) { } Status -Config::GetMetricConfigCollector(std::string &value) { +Config::GetMetricConfigCollector(std::string& value) { value = GetMetricConfigStrCollector(); return Status::OK(); } Status -Config::GetMetricConfigPrometheusPort(std::string &value) { +Config::GetMetricConfigPrometheusPort(std::string& value) { value = GetMetricConfigStrPrometheusPort(); return CheckMetricConfigPrometheusPort(value); } Status -Config::GetCacheConfigCpuCacheCapacity(int32_t &value) { +Config::GetCacheConfigCpuCacheCapacity(int32_t& value) { std::string str = GetCacheConfigStrCpuCacheCapacity(); Status s = CheckCacheConfigCpuCacheCapacity(str); if (!s.ok()) return s; @@ -931,7 +913,7 @@ Config::GetCacheConfigCpuCacheCapacity(int32_t &value) { } Status -Config::GetCacheConfigCpuCacheThreshold(float &value) { +Config::GetCacheConfigCpuCacheThreshold(float& value) { std::string str = GetCacheConfigStrCpuCacheThreshold(); Status s = CheckCacheConfigCpuCacheThreshold(str); if (!s.ok()) return s; @@ -940,7 +922,7 @@ Config::GetCacheConfigCpuCacheThreshold(float &value) { } Status -Config::GetCacheConfigGpuCacheCapacity(int32_t &value) { +Config::GetCacheConfigGpuCacheCapacity(int32_t& value) { std::string str = GetCacheConfigStrGpuCacheCapacity(); Status s = CheckCacheConfigGpuCacheCapacity(str); if (!s.ok()) return s; @@ -949,7 +931,7 @@ Config::GetCacheConfigGpuCacheCapacity(int32_t &value) { } Status -Config::GetCacheConfigGpuCacheThreshold(float &value) { +Config::GetCacheConfigGpuCacheThreshold(float& value) { std::string str = GetCacheConfigStrGpuCacheThreshold(); Status s = CheckCacheConfigGpuCacheThreshold(str); if (!s.ok()) return s; @@ -958,7 +940,7 @@ Config::GetCacheConfigGpuCacheThreshold(float &value) { } Status -Config::GetCacheConfigCacheInsertData(bool &value) { +Config::GetCacheConfigCacheInsertData(bool& value) { std::string str = GetCacheConfigStrCacheInsertData(); Status s = CheckCacheConfigCacheInsertData(str); if (!s.ok()) return s; @@ -968,7 +950,7 @@ Config::GetCacheConfigCacheInsertData(bool &value) { } Status -Config::GetEngineConfigUseBlasThreshold(int32_t &value) { +Config::GetEngineConfigUseBlasThreshold(int32_t& value) { std::string str = GetEngineConfigStrUseBlasThreshold(); Status s = CheckEngineConfigUseBlasThreshold(str); if (!s.ok()) return s; @@ -977,7 +959,7 @@ Config::GetEngineConfigUseBlasThreshold(int32_t &value) { } Status -Config::GetEngineConfigOmpThreadNum(int32_t &value) { +Config::GetEngineConfigOmpThreadNum(int32_t& value) { std::string str = GetEngineConfigStrOmpThreadNum(); Status s = CheckEngineConfigOmpThreadNum(str); if (!s.ok()) return s; @@ -986,13 +968,13 @@ Config::GetEngineConfigOmpThreadNum(int32_t &value) { } Status -Config::GetResourceConfigMode(std::string &value) { +Config::GetResourceConfigMode(std::string& value) { value = GetResourceConfigStrMode(); return CheckResourceConfigMode(value); } Status -Config::GetResourceConfigPool(std::vector &value) { +Config::GetResourceConfigPool(std::vector& value) { ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE); value = resource_config.GetSequence(CONFIG_RESOURCE_POOL); return CheckResourceConfigPool(value); @@ -1001,7 +983,7 @@ Config::GetResourceConfigPool(std::vector &value) { /////////////////////////////////////////////////////////////////////////////// /* server config */ Status -Config::SetServerConfigAddress(const std::string &value) { +Config::SetServerConfigAddress(const std::string& value) { Status s = CheckServerConfigAddress(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); @@ -1009,7 +991,7 @@ Config::SetServerConfigAddress(const std::string &value) { } Status -Config::SetServerConfigPort(const std::string &value) { +Config::SetServerConfigPort(const std::string& value) { Status s = CheckServerConfigPort(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); @@ -1017,7 +999,7 @@ Config::SetServerConfigPort(const std::string &value) { } Status -Config::SetServerConfigDeployMode(const std::string &value) { +Config::SetServerConfigDeployMode(const std::string& value) { Status s = CheckServerConfigDeployMode(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); @@ -1025,7 +1007,7 @@ Config::SetServerConfigDeployMode(const std::string &value) { } Status -Config::SetServerConfigTimeZone(const std::string &value) { +Config::SetServerConfigTimeZone(const std::string& value) { Status s = CheckServerConfigTimeZone(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); @@ -1034,7 +1016,7 @@ Config::SetServerConfigTimeZone(const std::string &value) { /* db config */ Status -Config::SetDBConfigPrimaryPath(const std::string &value) { +Config::SetDBConfigPrimaryPath(const std::string& value) { Status s = CheckDBConfigPrimaryPath(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); @@ -1042,7 +1024,7 @@ Config::SetDBConfigPrimaryPath(const std::string &value) { } Status -Config::SetDBConfigSecondaryPath(const std::string &value) { +Config::SetDBConfigSecondaryPath(const std::string& value) { Status s = CheckDBConfigSecondaryPath(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); @@ -1050,7 +1032,7 @@ Config::SetDBConfigSecondaryPath(const std::string &value) { } Status -Config::SetDBConfigBackendUrl(const std::string &value) { +Config::SetDBConfigBackendUrl(const std::string& value) { Status s = CheckDBConfigBackendUrl(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); @@ -1058,7 +1040,7 @@ Config::SetDBConfigBackendUrl(const std::string &value) { } Status -Config::SetDBConfigArchiveDiskThreshold(const std::string &value) { +Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { Status s = CheckDBConfigArchiveDiskThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); @@ -1066,7 +1048,7 @@ Config::SetDBConfigArchiveDiskThreshold(const std::string &value) { } Status -Config::SetDBConfigArchiveDaysThreshold(const std::string &value) { +Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { Status s = CheckDBConfigArchiveDaysThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); @@ -1074,7 +1056,7 @@ Config::SetDBConfigArchiveDaysThreshold(const std::string &value) { } Status -Config::SetDBConfigInsertBufferSize(const std::string &value) { +Config::SetDBConfigInsertBufferSize(const std::string& value) { Status s = CheckDBConfigInsertBufferSize(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); @@ -1082,7 +1064,7 @@ Config::SetDBConfigInsertBufferSize(const std::string &value) { } Status -Config::SetDBConfigBuildIndexGPU(const std::string &value) { +Config::SetDBConfigBuildIndexGPU(const std::string& value) { Status s = CheckDBConfigBuildIndexGPU(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); @@ -1091,7 +1073,7 @@ Config::SetDBConfigBuildIndexGPU(const std::string &value) { /* metric config */ Status -Config::SetMetricConfigEnableMonitor(const std::string &value) { +Config::SetMetricConfigEnableMonitor(const std::string& value) { Status s = CheckMetricConfigEnableMonitor(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_ENABLE_MONITOR, value); @@ -1099,7 +1081,7 @@ Config::SetMetricConfigEnableMonitor(const std::string &value) { } Status -Config::SetMetricConfigCollector(const std::string &value) { +Config::SetMetricConfigCollector(const std::string& value) { Status s = CheckMetricConfigCollector(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_COLLECTOR, value); @@ -1107,7 +1089,7 @@ Config::SetMetricConfigCollector(const std::string &value) { } Status -Config::SetMetricConfigPrometheusPort(const std::string &value) { +Config::SetMetricConfigPrometheusPort(const std::string& value) { Status s = CheckMetricConfigPrometheusPort(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_PROMETHEUS_PORT, value); @@ -1116,7 +1098,7 @@ Config::SetMetricConfigPrometheusPort(const std::string &value) { /* cache config */ Status -Config::SetCacheConfigCpuCacheCapacity(const std::string &value) { +Config::SetCacheConfigCpuCacheCapacity(const std::string& value) { Status s = CheckCacheConfigCpuCacheCapacity(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_CAPACITY, value); @@ -1124,7 +1106,7 @@ Config::SetCacheConfigCpuCacheCapacity(const std::string &value) { } Status -Config::SetCacheConfigCpuCacheThreshold(const std::string &value) { +Config::SetCacheConfigCpuCacheThreshold(const std::string& value) { Status s = CheckCacheConfigCpuCacheThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); @@ -1132,7 +1114,7 @@ Config::SetCacheConfigCpuCacheThreshold(const std::string &value) { } Status -Config::SetCacheConfigGpuCacheCapacity(const std::string &value) { +Config::SetCacheConfigGpuCacheCapacity(const std::string& value) { Status s = CheckCacheConfigGpuCacheCapacity(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_CAPACITY, value); @@ -1140,7 +1122,7 @@ Config::SetCacheConfigGpuCacheCapacity(const std::string &value) { } Status -Config::SetCacheConfigGpuCacheThreshold(const std::string &value) { +Config::SetCacheConfigGpuCacheThreshold(const std::string& value) { Status s = CheckCacheConfigGpuCacheThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value); @@ -1148,7 +1130,7 @@ Config::SetCacheConfigGpuCacheThreshold(const std::string &value) { } Status -Config::SetCacheConfigCacheInsertData(const std::string &value) { +Config::SetCacheConfigCacheInsertData(const std::string& value) { Status s = CheckCacheConfigCacheInsertData(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CACHE_INSERT_DATA, value); @@ -1157,7 +1139,7 @@ Config::SetCacheConfigCacheInsertData(const std::string &value) { /* engine config */ Status -Config::SetEngineConfigUseBlasThreshold(const std::string &value) { +Config::SetEngineConfigUseBlasThreshold(const std::string& value) { Status s = CheckEngineConfigUseBlasThreshold(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value); @@ -1165,7 +1147,7 @@ Config::SetEngineConfigUseBlasThreshold(const std::string &value) { } Status -Config::SetEngineConfigOmpThreadNum(const std::string &value) { +Config::SetEngineConfigOmpThreadNum(const std::string& value) { Status s = CheckEngineConfigOmpThreadNum(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_OMP_THREAD_NUM, value); @@ -1174,14 +1156,13 @@ Config::SetEngineConfigOmpThreadNum(const std::string &value) { /* resource config */ Status -Config::SetResourceConfigMode(const std::string &value) { +Config::SetResourceConfigMode(const std::string& value) { Status s = CheckResourceConfigMode(value); if (!s.ok()) return s; SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_MODE, value); return Status::OK(); } -} // namespace server -} // namespace milvus -} // namespace zilliz - +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index c2484a850a..4e3d563901 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -17,252 +17,346 @@ #pragma once -#include -#include -#include -#include #include +#include +#include +#include +#include -#include "utils/Status.h" #include "config/ConfigNode.h" +#include "utils/Status.h" namespace zilliz { namespace milvus { namespace server { /* server config */ -static const char *CONFIG_SERVER = "server_config"; -static const char *CONFIG_SERVER_ADDRESS = "address"; -static const char *CONFIG_SERVER_ADDRESS_DEFAULT = "127.0.0.1"; -static const char *CONFIG_SERVER_PORT = "port"; -static const char *CONFIG_SERVER_PORT_DEFAULT = "19530"; -static const char *CONFIG_SERVER_DEPLOY_MODE = "deploy_mode"; -static const char *CONFIG_SERVER_DEPLOY_MODE_DEFAULT = "single"; -static const char *CONFIG_SERVER_TIME_ZONE = "time_zone"; -static const char *CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; +static const char* CONFIG_SERVER = "server_config"; +static const char* CONFIG_SERVER_ADDRESS = "address"; +static const char* CONFIG_SERVER_ADDRESS_DEFAULT = "127.0.0.1"; +static const char* CONFIG_SERVER_PORT = "port"; +static const char* CONFIG_SERVER_PORT_DEFAULT = "19530"; +static const char* CONFIG_SERVER_DEPLOY_MODE = "deploy_mode"; +static const char* CONFIG_SERVER_DEPLOY_MODE_DEFAULT = "single"; +static const char* CONFIG_SERVER_TIME_ZONE = "time_zone"; +static const char* CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; /* db config */ -static const char *CONFIG_DB = "db_config"; -static const char *CONFIG_DB_PRIMARY_PATH = "primary_path"; -static const char *CONFIG_DB_PRIMARY_PATH_DEFAULT = "/tmp/milvus"; -static const char *CONFIG_DB_SECONDARY_PATH = "secondary_path"; -static const char *CONFIG_DB_SECONDARY_PATH_DEFAULT = ""; -static const char *CONFIG_DB_BACKEND_URL = "backend_url"; -static const char *CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/"; -static const char *CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold"; -static const char *CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0"; -static const char *CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold"; -static const char *CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; -static const char *CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; -static const char *CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4"; -static const char *CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu"; -static const char *CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0"; +static const char* CONFIG_DB = "db_config"; +static const char* CONFIG_DB_PRIMARY_PATH = "primary_path"; +static const char* CONFIG_DB_PRIMARY_PATH_DEFAULT = "/tmp/milvus"; +static const char* CONFIG_DB_SECONDARY_PATH = "secondary_path"; +static const char* CONFIG_DB_SECONDARY_PATH_DEFAULT = ""; +static const char* CONFIG_DB_BACKEND_URL = "backend_url"; +static const char* CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/"; +static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold"; +static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0"; +static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold"; +static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; +static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; +static const char* CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4"; +static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu"; +static const char* CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0"; /* cache config */ -static const char *CONFIG_CACHE = "cache_config"; -static const char *CONFIG_CACHE_CPU_CACHE_CAPACITY = "cpu_cache_capacity"; -static const char *CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT = "16"; -static const char *CONFIG_CACHE_GPU_CACHE_CAPACITY = "gpu_cache_capacity"; -static const char *CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT = "0"; -static const char *CONFIG_CACHE_CPU_CACHE_THRESHOLD = "cpu_mem_threshold"; -static const char *CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT = "0.85"; -static const char *CONFIG_CACHE_GPU_CACHE_THRESHOLD = "gpu_mem_threshold"; -static const char *CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT = "0.85"; -static const char *CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; -static const char *CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; +static const char* CONFIG_CACHE = "cache_config"; +static const char* CONFIG_CACHE_CPU_CACHE_CAPACITY = "cpu_cache_capacity"; +static const char* CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT = "16"; +static const char* CONFIG_CACHE_GPU_CACHE_CAPACITY = "gpu_cache_capacity"; +static const char* CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT = "0"; +static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD = "cpu_mem_threshold"; +static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT = "0.85"; +static const char* CONFIG_CACHE_GPU_CACHE_THRESHOLD = "gpu_mem_threshold"; +static const char* CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT = "0.85"; +static const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; +static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; /* metric config */ -static const char *CONFIG_METRIC = "metric_config"; -static const char *CONFIG_METRIC_ENABLE_MONITOR = "enable_monitor"; -static const char *CONFIG_METRIC_ENABLE_MONITOR_DEFAULT = "false"; -static const char *CONFIG_METRIC_COLLECTOR = "collector"; -static const char *CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus"; -static const char *CONFIG_METRIC_PROMETHEUS = "prometheus_config"; -static const char *CONFIG_METRIC_PROMETHEUS_PORT = "port"; -static const char *CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT = "8080"; +static const char* CONFIG_METRIC = "metric_config"; +static const char* CONFIG_METRIC_ENABLE_MONITOR = "enable_monitor"; +static const char* CONFIG_METRIC_ENABLE_MONITOR_DEFAULT = "false"; +static const char* CONFIG_METRIC_COLLECTOR = "collector"; +static const char* CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus"; +static const char* CONFIG_METRIC_PROMETHEUS = "prometheus_config"; +static const char* CONFIG_METRIC_PROMETHEUS_PORT = "port"; +static const char* CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT = "8080"; /* engine config */ -static const char *CONFIG_ENGINE = "engine_config"; -static const char *CONFIG_ENGINE_USE_BLAS_THRESHOLD = "use_blas_threshold"; -static const char *CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT = "20"; -static const char *CONFIG_ENGINE_OMP_THREAD_NUM = "omp_thread_num"; -static const char *CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; +static const char* CONFIG_ENGINE = "engine_config"; +static const char* CONFIG_ENGINE_USE_BLAS_THRESHOLD = "use_blas_threshold"; +static const char* CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT = "20"; +static const char* CONFIG_ENGINE_OMP_THREAD_NUM = "omp_thread_num"; +static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; /* resource config */ -static const char *CONFIG_RESOURCE = "resource_config"; -static const char *CONFIG_RESOURCE_MODE = "mode"; -static const char *CONFIG_RESOURCE_MODE_DEFAULT = "simple"; -static const char *CONFIG_RESOURCE_POOL = "resource_pool"; +static const char* CONFIG_RESOURCE = "resource_config"; +static const char* CONFIG_RESOURCE_MODE = "mode"; +static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; +static const char* CONFIG_RESOURCE_POOL = "resource_pool"; class Config { public: - static Config &GetInstance(); - Status LoadConfigFile(const std::string &filename); - Status ValidateConfig(); - Status ResetDefaultConfig(); - void PrintAll(); + static Config& + GetInstance(); + Status + LoadConfigFile(const std::string& filename); + Status + ValidateConfig(); + Status + ResetDefaultConfig(); + void + PrintAll(); private: - ConfigNode &GetConfigNode(const std::string &name); + ConfigNode& + GetConfigNode(const std::string& name); - Status GetConfigValueInMem(const std::string &parent_key, - const std::string &child_key, - std::string &value); + Status + GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value); - void SetConfigValueInMem(const std::string &parent_key, - const std::string &child_key, - const std::string &value); + void + SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, const std::string& value); - void PrintConfigSection(const std::string &config_node_name); + void + PrintConfigSection(const std::string& config_node_name); /////////////////////////////////////////////////////////////////////////// /* server config */ - Status CheckServerConfigAddress(const std::string &value); - Status CheckServerConfigPort(const std::string &value); - Status CheckServerConfigDeployMode(const std::string &value); - Status CheckServerConfigTimeZone(const std::string &value); + Status + CheckServerConfigAddress(const std::string& value); + Status + CheckServerConfigPort(const std::string& value); + Status + CheckServerConfigDeployMode(const std::string& value); + Status + CheckServerConfigTimeZone(const std::string& value); /* db config */ - Status CheckDBConfigPrimaryPath(const std::string &value); - Status CheckDBConfigSecondaryPath(const std::string &value); - Status CheckDBConfigBackendUrl(const std::string &value); - Status CheckDBConfigArchiveDiskThreshold(const std::string &value); - Status CheckDBConfigArchiveDaysThreshold(const std::string &value); - Status CheckDBConfigInsertBufferSize(const std::string &value); - Status CheckDBConfigBuildIndexGPU(const std::string &value); + Status + CheckDBConfigPrimaryPath(const std::string& value); + Status + CheckDBConfigSecondaryPath(const std::string& value); + Status + CheckDBConfigBackendUrl(const std::string& value); + Status + CheckDBConfigArchiveDiskThreshold(const std::string& value); + Status + CheckDBConfigArchiveDaysThreshold(const std::string& value); + Status + CheckDBConfigInsertBufferSize(const std::string& value); + Status + CheckDBConfigBuildIndexGPU(const std::string& value); /* metric config */ - Status CheckMetricConfigEnableMonitor(const std::string &value); - Status CheckMetricConfigCollector(const std::string &value); - Status CheckMetricConfigPrometheusPort(const std::string &value); + Status + CheckMetricConfigEnableMonitor(const std::string& value); + Status + CheckMetricConfigCollector(const std::string& value); + Status + CheckMetricConfigPrometheusPort(const std::string& value); /* cache config */ - Status CheckCacheConfigCpuCacheCapacity(const std::string &value); - Status CheckCacheConfigCpuCacheThreshold(const std::string &value); - Status CheckCacheConfigGpuCacheCapacity(const std::string &value); - Status CheckCacheConfigGpuCacheThreshold(const std::string &value); - Status CheckCacheConfigCacheInsertData(const std::string &value); + Status + CheckCacheConfigCpuCacheCapacity(const std::string& value); + Status + CheckCacheConfigCpuCacheThreshold(const std::string& value); + Status + CheckCacheConfigGpuCacheCapacity(const std::string& value); + Status + CheckCacheConfigGpuCacheThreshold(const std::string& value); + Status + CheckCacheConfigCacheInsertData(const std::string& value); /* engine config */ - Status CheckEngineConfigUseBlasThreshold(const std::string &value); - Status CheckEngineConfigOmpThreadNum(const std::string &value); + Status + CheckEngineConfigUseBlasThreshold(const std::string& value); + Status + CheckEngineConfigOmpThreadNum(const std::string& value); /* resource config */ - Status CheckResourceConfigMode(const std::string &value); - Status CheckResourceConfigPool(const std::vector &value); + Status + CheckResourceConfigMode(const std::string& value); + Status + CheckResourceConfigPool(const std::vector& value); /////////////////////////////////////////////////////////////////////////// /* server config */ - std::string GetServerConfigStrAddress(); - std::string GetServerConfigStrPort(); - std::string GetServerConfigStrDeployMode(); - std::string GetServerConfigStrTimeZone(); + std::string + GetServerConfigStrAddress(); + std::string + GetServerConfigStrPort(); + std::string + GetServerConfigStrDeployMode(); + std::string + GetServerConfigStrTimeZone(); /* db config */ - std::string GetDBConfigStrPrimaryPath(); - std::string GetDBConfigStrSecondaryPath(); - std::string GetDBConfigStrBackendUrl(); - std::string GetDBConfigStrArchiveDiskThreshold(); - std::string GetDBConfigStrArchiveDaysThreshold(); - std::string GetDBConfigStrInsertBufferSize(); - std::string GetDBConfigStrBuildIndexGPU(); + std::string + GetDBConfigStrPrimaryPath(); + std::string + GetDBConfigStrSecondaryPath(); + std::string + GetDBConfigStrBackendUrl(); + std::string + GetDBConfigStrArchiveDiskThreshold(); + std::string + GetDBConfigStrArchiveDaysThreshold(); + std::string + GetDBConfigStrInsertBufferSize(); + std::string + GetDBConfigStrBuildIndexGPU(); /* metric config */ - std::string GetMetricConfigStrEnableMonitor(); - std::string GetMetricConfigStrCollector(); - std::string GetMetricConfigStrPrometheusPort(); + std::string + GetMetricConfigStrEnableMonitor(); + std::string + GetMetricConfigStrCollector(); + std::string + GetMetricConfigStrPrometheusPort(); /* cache config */ - std::string GetCacheConfigStrCpuCacheCapacity(); - std::string GetCacheConfigStrCpuCacheThreshold(); - std::string GetCacheConfigStrGpuCacheCapacity(); - std::string GetCacheConfigStrGpuCacheThreshold(); - std::string GetCacheConfigStrCacheInsertData(); + std::string + GetCacheConfigStrCpuCacheCapacity(); + std::string + GetCacheConfigStrCpuCacheThreshold(); + std::string + GetCacheConfigStrGpuCacheCapacity(); + std::string + GetCacheConfigStrGpuCacheThreshold(); + std::string + GetCacheConfigStrCacheInsertData(); /* engine config */ - std::string GetEngineConfigStrUseBlasThreshold(); - std::string GetEngineConfigStrOmpThreadNum(); + std::string + GetEngineConfigStrUseBlasThreshold(); + std::string + GetEngineConfigStrOmpThreadNum(); /* resource config */ - std::string GetResourceConfigStrMode(); + std::string + GetResourceConfigStrMode(); public: /* server config */ - Status GetServerConfigAddress(std::string &value); - Status GetServerConfigPort(std::string &value); - Status GetServerConfigDeployMode(std::string &value); - Status GetServerConfigTimeZone(std::string &value); + Status + GetServerConfigAddress(std::string& value); + Status + GetServerConfigPort(std::string& value); + Status + GetServerConfigDeployMode(std::string& value); + Status + GetServerConfigTimeZone(std::string& value); /* db config */ - Status GetDBConfigPrimaryPath(std::string &value); - Status GetDBConfigSecondaryPath(std::string &value); - Status GetDBConfigBackendUrl(std::string &value); - Status GetDBConfigArchiveDiskThreshold(int32_t &value); - Status GetDBConfigArchiveDaysThreshold(int32_t &value); - Status GetDBConfigInsertBufferSize(int32_t &value); - Status GetDBConfigBuildIndexGPU(int32_t &value); + Status + GetDBConfigPrimaryPath(std::string& value); + Status + GetDBConfigSecondaryPath(std::string& value); + Status + GetDBConfigBackendUrl(std::string& value); + Status + GetDBConfigArchiveDiskThreshold(int32_t& value); + Status + GetDBConfigArchiveDaysThreshold(int32_t& value); + Status + GetDBConfigInsertBufferSize(int32_t& value); + Status + GetDBConfigBuildIndexGPU(int32_t& value); /* metric config */ - Status GetMetricConfigEnableMonitor(bool &value); - Status GetMetricConfigCollector(std::string &value); - Status GetMetricConfigPrometheusPort(std::string &value); + Status + GetMetricConfigEnableMonitor(bool& value); + Status + GetMetricConfigCollector(std::string& value); + Status + GetMetricConfigPrometheusPort(std::string& value); /* cache config */ - Status GetCacheConfigCpuCacheCapacity(int32_t &value); - Status GetCacheConfigCpuCacheThreshold(float &value); - Status GetCacheConfigGpuCacheCapacity(int32_t &value); - Status GetCacheConfigGpuCacheThreshold(float &value); - Status GetCacheConfigCacheInsertData(bool &value); + Status + GetCacheConfigCpuCacheCapacity(int32_t& value); + Status + GetCacheConfigCpuCacheThreshold(float& value); + Status + GetCacheConfigGpuCacheCapacity(int32_t& value); + Status + GetCacheConfigGpuCacheThreshold(float& value); + Status + GetCacheConfigCacheInsertData(bool& value); /* engine config */ - Status GetEngineConfigUseBlasThreshold(int32_t &value); - Status GetEngineConfigOmpThreadNum(int32_t &value); + Status + GetEngineConfigUseBlasThreshold(int32_t& value); + Status + GetEngineConfigOmpThreadNum(int32_t& value); /* resource config */ - Status GetResourceConfigMode(std::string &value); - Status GetResourceConfigPool(std::vector &value); + Status + GetResourceConfigMode(std::string& value); + Status + GetResourceConfigPool(std::vector& value); public: /* server config */ - Status SetServerConfigAddress(const std::string &value); - Status SetServerConfigPort(const std::string &value); - Status SetServerConfigDeployMode(const std::string &value); - Status SetServerConfigTimeZone(const std::string &value); + Status + SetServerConfigAddress(const std::string& value); + Status + SetServerConfigPort(const std::string& value); + Status + SetServerConfigDeployMode(const std::string& value); + Status + SetServerConfigTimeZone(const std::string& value); /* db config */ - Status SetDBConfigPrimaryPath(const std::string &value); - Status SetDBConfigSecondaryPath(const std::string &value); - Status SetDBConfigBackendUrl(const std::string &value); - Status SetDBConfigArchiveDiskThreshold(const std::string &value); - Status SetDBConfigArchiveDaysThreshold(const std::string &value); - Status SetDBConfigInsertBufferSize(const std::string &value); - Status SetDBConfigBuildIndexGPU(const std::string &value); + Status + SetDBConfigPrimaryPath(const std::string& value); + Status + SetDBConfigSecondaryPath(const std::string& value); + Status + SetDBConfigBackendUrl(const std::string& value); + Status + SetDBConfigArchiveDiskThreshold(const std::string& value); + Status + SetDBConfigArchiveDaysThreshold(const std::string& value); + Status + SetDBConfigInsertBufferSize(const std::string& value); + Status + SetDBConfigBuildIndexGPU(const std::string& value); /* metric config */ - Status SetMetricConfigEnableMonitor(const std::string &value); - Status SetMetricConfigCollector(const std::string &value); - Status SetMetricConfigPrometheusPort(const std::string &value); + Status + SetMetricConfigEnableMonitor(const std::string& value); + Status + SetMetricConfigCollector(const std::string& value); + Status + SetMetricConfigPrometheusPort(const std::string& value); /* cache config */ - Status SetCacheConfigCpuCacheCapacity(const std::string &value); - Status SetCacheConfigCpuCacheThreshold(const std::string &value); - Status SetCacheConfigGpuCacheCapacity(const std::string &value); - Status SetCacheConfigGpuCacheThreshold(const std::string &value); - Status SetCacheConfigCacheInsertData(const std::string &value); + Status + SetCacheConfigCpuCacheCapacity(const std::string& value); + Status + SetCacheConfigCpuCacheThreshold(const std::string& value); + Status + SetCacheConfigGpuCacheCapacity(const std::string& value); + Status + SetCacheConfigGpuCacheThreshold(const std::string& value); + Status + SetCacheConfigCacheInsertData(const std::string& value); /* engine config */ - Status SetEngineConfigUseBlasThreshold(const std::string &value); - Status SetEngineConfigOmpThreadNum(const std::string &value); + Status + SetEngineConfigUseBlasThreshold(const std::string& value); + Status + SetEngineConfigOmpThreadNum(const std::string& value); /* resource config */ - Status SetResourceConfigMode(const std::string &value); + Status + SetResourceConfigMode(const std::string& value); private: std::unordered_map> config_map_; std::mutex mutex_; }; -} // namespace server -} // namespace milvus -} // namespace zilliz - +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index d764570966..c0e02f24e4 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include "server/DBWrapper.h" #include "Config.h" #include "db/DBFactory.h" @@ -23,9 +22,9 @@ #include "utils/Log.h" #include "utils/StringHelpFunctions.h" -#include -#include #include +#include +#include namespace zilliz { namespace milvus { @@ -34,10 +33,11 @@ namespace server { DBWrapper::DBWrapper() { } -Status DBWrapper::StartService() { +Status +DBWrapper::StartService() { Config& config = Config::GetInstance(); Status s; - //db config + // db config engine::DBOptions opt; s = config.GetDBConfigBackendUrl(opt.meta_.backend_uri_); @@ -70,9 +70,8 @@ Status DBWrapper::StartService() { } else if (mode == "cluster_writable") { opt.mode_ = engine::DBOptions::MODE::CLUSTER_WRITABLE; } else { - std::cerr << - "ERROR: mode specified in server_config must be ['single', 'cluster_readonly', 'cluster_writable']" - << std::endl; + std::cerr << "ERROR: mode specified in server_config must be ['single', 'cluster_readonly', 'cluster_writable']" + << std::endl; kill(0, SIGUSR1); } @@ -86,18 +85,18 @@ Status DBWrapper::StartService() { } else { uint32_t sys_thread_cnt = 8; if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { - omp_thread = (int32_t)ceil(sys_thread_cnt*0.5); + omp_thread = (int32_t)ceil(sys_thread_cnt * 0.5); omp_set_num_threads(omp_thread); } } - //init faiss global variable + // init faiss global variable int32_t use_blas_threshold; s = config.GetEngineConfigUseBlasThreshold(use_blas_threshold); if (!s.ok()) return s; faiss::distance_compute_blas_threshold = use_blas_threshold; - //set archive config + // set archive config engine::ArchiveConf::CriteriaT criterial; int32_t disk, days; s = config.GetDBConfigArchiveDiskThreshold(disk); @@ -113,7 +112,7 @@ Status DBWrapper::StartService() { } opt.meta_.archive_conf_.SetCriterias(criterial); - //create db root folder + // create db root folder Status status = CommonUtil::CreateDirectory(opt.meta_.path_); if (!status.ok()) { std::cerr << "ERROR! Failed to create database root path: " << opt.meta_.path_ << std::endl; @@ -128,10 +127,10 @@ Status DBWrapper::StartService() { } } - //create db instance + // create db instance try { db_ = engine::DBFactory::Build(opt); - } catch(std::exception& ex) { + } catch (std::exception& ex) { std::cerr << "ERROR! Failed to open database: " << ex.what() << std::endl; kill(0, SIGUSR1); } @@ -141,7 +140,8 @@ Status DBWrapper::StartService() { return Status::OK(); } -Status DBWrapper::StopService() { +Status +DBWrapper::StopService() { if (db_) { db_->Stop(); } @@ -149,6 +149,6 @@ Status DBWrapper::StopService() { return Status::OK(); } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/DBWrapper.h b/cpp/src/server/DBWrapper.h index 4648f4b59d..5a035e44bc 100644 --- a/cpp/src/server/DBWrapper.h +++ b/cpp/src/server/DBWrapper.h @@ -17,8 +17,8 @@ #pragma once -#include "utils/Status.h" #include "db/DB.h" +#include "utils/Status.h" #include @@ -32,19 +32,24 @@ class DBWrapper { ~DBWrapper() = default; public: - static DBWrapper &GetInstance() { + static DBWrapper& + GetInstance() { static DBWrapper wrapper; return wrapper; } - static engine::DBPtr DB() { + static engine::DBPtr + DB() { return GetInstance().EngineDB(); } - Status StartService(); - Status StopService(); + Status + StartService(); + Status + StopService(); - engine::DBPtr EngineDB() { + engine::DBPtr + EngineDB() { return db_; } @@ -52,6 +57,6 @@ class DBWrapper { engine::DBPtr db_; }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index f0675fc84d..b86f386acc 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -17,41 +17,39 @@ #include "server/Server.h" -#include #include #include #include #include +#include //#include -#include #include +#include -#include "server/grpc_impl/GrpcServer.h" +#include "DBWrapper.h" +#include "metrics/Metrics.h" +#include "scheduler/SchedInst.h" #include "server/Config.h" +#include "server/grpc_impl/GrpcServer.h" #include "utils/Log.h" #include "utils/LogUtil.h" #include "utils/SignalUtil.h" #include "utils/TimeRecorder.h" -#include "metrics/Metrics.h" -#include "scheduler/SchedInst.h" #include "wrapper/KnowhereResource.h" -#include "DBWrapper.h" namespace zilliz { namespace milvus { namespace server { -Server & +Server& Server::GetInstance() { static Server server; return server; } void -Server::Init(int64_t daemonized, - const std::string &pid_filename, - const std::string &config_filename, - const std::string &log_config_file) { +Server::Init(int64_t daemonized, const std::string& pid_filename, const std::string& config_filename, + const std::string& log_config_file) { daemonized_ = daemonized; pid_filename_ = pid_filename; config_filename_ = config_filename; @@ -66,9 +64,9 @@ Server::Daemonize() { std::cout << "Milvus server run in daemonize mode"; -// std::string log_path(GetLogDirFullPath()); -// log_path += "zdb_server.(INFO/WARNNING/ERROR/CRITICAL)"; -// SERVER_LOG_INFO << "Log will be exported to: " + log_path); + // std::string log_path(GetLogDirFullPath()); + // log_path += "zdb_server.(INFO/WARNNING/ERROR/CRITICAL)"; + // SERVER_LOG_INFO << "Log will be exported to: " + log_path); pid_t pid = 0; @@ -160,7 +158,7 @@ Server::Start() { } /* log path is defined in Config file, so InitLog must be called after LoadConfig */ - Config &config = Config::GetInstance(); + Config& config = Config::GetInstance(); std::string time_zone; Status s = config.GetServerConfigTimeZone(time_zone); if (!s.ok()) { @@ -194,7 +192,7 @@ Server::Start() { StartService(); std::cout << "Milvus server start successfully." << std::endl; - } catch (std::exception &ex) { + } catch (std::exception& ex) { std::cerr << "Milvus server encounter exception: " << ex.what(); } } @@ -233,7 +231,7 @@ Server::Stop() { ErrorCode Server::LoadConfig() { - Config &config = Config::GetInstance(); + Config& config = Config::GetInstance(); Status s = config.LoadConfigFile(config_filename_); if (!s.ok()) { std::cerr << "Failed to load config file: " << config_filename_ << std::endl; @@ -264,6 +262,6 @@ Server::StopService() { engine::KnowhereResource::Finalize(); } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/Server.h b/cpp/src/server/Server.h index 9bd0ce13b5..b86945cef9 100644 --- a/cpp/src/server/Server.h +++ b/cpp/src/server/Server.h @@ -28,26 +28,32 @@ namespace server { class Server { public: - static Server &GetInstance(); + static Server& + GetInstance(); - void Init(int64_t daemonized, - const std::string &pid_filename, - const std::string &config_filename, - const std::string &log_config_file); + void + Init(int64_t daemonized, const std::string& pid_filename, const std::string& config_filename, + const std::string& log_config_file); - void Start(); - void Stop(); + void + Start(); + void + Stop(); private: Server() = default; ~Server() = default; - void Daemonize(); + void + Daemonize(); - ErrorCode LoadConfig(); + ErrorCode + LoadConfig(); - void StartService(); - void StopService(); + void + StartService(); + void + StopService(); private: int64_t daemonized_ = 0; @@ -57,6 +63,6 @@ class Server { std::string log_config_file_; }; // Server -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp b/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp index a64422aab7..6089df6f8e 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestHandler.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include "server/grpc_impl/GrpcRequestHandler.h" #include "server/grpc_impl/GrpcRequestTask.h" #include "utils/TimeRecorder.h" @@ -28,18 +27,16 @@ namespace server { namespace grpc { ::grpc::Status -GrpcRequestHandler::CreateTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableSchema *request, - ::milvus::grpc::Status *response) { +GrpcRequestHandler::CreateTable(::grpc::ServerContext* context, const ::milvus::grpc::TableSchema* request, + ::milvus::grpc::Status* response) { BaseTaskPtr task_ptr = CreateTableTask::Create(request); GrpcRequestScheduler::ExecTask(task_ptr, response); return ::grpc::Status::OK; } ::grpc::Status -GrpcRequestHandler::HasTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::BoolReply *response) { +GrpcRequestHandler::HasTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::BoolReply* response) { bool has_table = false; BaseTaskPtr task_ptr = HasTableTask::Create(request->table_name(), has_table); ::milvus::grpc::Status grpc_status; @@ -51,27 +48,24 @@ GrpcRequestHandler::HasTable(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::DropTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::Status *response) { +GrpcRequestHandler::DropTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::Status* response) { BaseTaskPtr task_ptr = DropTableTask::Create(request->table_name()); GrpcRequestScheduler::ExecTask(task_ptr, response); return ::grpc::Status::OK; } ::grpc::Status -GrpcRequestHandler::CreateIndex(::grpc::ServerContext *context, - const ::milvus::grpc::IndexParam *request, - ::milvus::grpc::Status *response) { +GrpcRequestHandler::CreateIndex(::grpc::ServerContext* context, const ::milvus::grpc::IndexParam* request, + ::milvus::grpc::Status* response) { BaseTaskPtr task_ptr = CreateIndexTask::Create(request); GrpcRequestScheduler::ExecTask(task_ptr, response); return ::grpc::Status::OK; } ::grpc::Status -GrpcRequestHandler::Insert(::grpc::ServerContext *context, - const ::milvus::grpc::InsertParam *request, - ::milvus::grpc::VectorIds *response) { +GrpcRequestHandler::Insert(::grpc::ServerContext* context, const ::milvus::grpc::InsertParam* request, + ::milvus::grpc::VectorIds* response) { BaseTaskPtr task_ptr = InsertTask::Create(request, response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -81,9 +75,8 @@ GrpcRequestHandler::Insert(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::Search(::grpc::ServerContext *context, - const ::milvus::grpc::SearchParam *request, - ::milvus::grpc::TopKQueryResultList *response) { +GrpcRequestHandler::Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request, + ::milvus::grpc::TopKQueryResultList* response) { std::vector file_id_array; BaseTaskPtr task_ptr = SearchTask::Create(request, file_id_array, response); ::milvus::grpc::Status grpc_status; @@ -94,14 +87,13 @@ GrpcRequestHandler::Search(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::SearchInFiles(::grpc::ServerContext *context, - const ::milvus::grpc::SearchInFilesParam *request, - ::milvus::grpc::TopKQueryResultList *response) { +GrpcRequestHandler::SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request, + ::milvus::grpc::TopKQueryResultList* response) { std::vector file_id_array; for (int i = 0; i < request->file_id_array_size(); i++) { file_id_array.push_back(request->file_id_array(i)); } - ::milvus::grpc::SearchInFilesParam *request_mutable = const_cast<::milvus::grpc::SearchInFilesParam *>(request); + ::milvus::grpc::SearchInFilesParam* request_mutable = const_cast<::milvus::grpc::SearchInFilesParam*>(request); BaseTaskPtr task_ptr = SearchTask::Create(request_mutable->mutable_search_param(), file_id_array, response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -111,9 +103,8 @@ GrpcRequestHandler::SearchInFiles(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::DescribeTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::TableSchema *response) { +GrpcRequestHandler::DescribeTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::TableSchema* response) { BaseTaskPtr task_ptr = DescribeTableTask::Create(request->table_name(), response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -123,9 +114,8 @@ GrpcRequestHandler::DescribeTable(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::CountTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::TableRowCount *response) { +GrpcRequestHandler::CountTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::TableRowCount* response) { int64_t row_count = 0; BaseTaskPtr task_ptr = CountTableTask::Create(request->table_name(), row_count); ::milvus::grpc::Status grpc_status; @@ -137,9 +127,8 @@ GrpcRequestHandler::CountTable(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::ShowTables(::grpc::ServerContext *context, - const ::milvus::grpc::Command *request, - ::milvus::grpc::TableNameList *response) { +GrpcRequestHandler::ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, + ::milvus::grpc::TableNameList* response) { BaseTaskPtr task_ptr = ShowTablesTask::Create(response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -149,9 +138,8 @@ GrpcRequestHandler::ShowTables(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::Cmd(::grpc::ServerContext *context, - const ::milvus::grpc::Command *request, - ::milvus::grpc::StringReply *response) { +GrpcRequestHandler::Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, + ::milvus::grpc::StringReply* response) { std::string result; BaseTaskPtr task_ptr = CmdTask::Create(request->cmd(), result); ::milvus::grpc::Status grpc_status; @@ -163,9 +151,8 @@ GrpcRequestHandler::Cmd(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::DeleteByRange(::grpc::ServerContext *context, - const ::milvus::grpc::DeleteByRangeParam *request, - ::milvus::grpc::Status *response) { +GrpcRequestHandler::DeleteByRange(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByRangeParam* request, + ::milvus::grpc::Status* response) { BaseTaskPtr task_ptr = DeleteByRangeTask::Create(request); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -175,9 +162,8 @@ GrpcRequestHandler::DeleteByRange(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::PreloadTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::Status *response) { +GrpcRequestHandler::PreloadTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::Status* response) { BaseTaskPtr task_ptr = PreloadTableTask::Create(request->table_name()); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -187,9 +173,8 @@ GrpcRequestHandler::PreloadTable(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::DescribeIndex(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::IndexParam *response) { +GrpcRequestHandler::DescribeIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::IndexParam* response) { BaseTaskPtr task_ptr = DescribeIndexTask::Create(request->table_name(), response); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -199,9 +184,8 @@ GrpcRequestHandler::DescribeIndex(::grpc::ServerContext *context, } ::grpc::Status -GrpcRequestHandler::DropIndex(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::Status *response) { +GrpcRequestHandler::DropIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::Status* response) { BaseTaskPtr task_ptr = DropIndexTask::Create(request->table_name()); ::milvus::grpc::Status grpc_status; GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); @@ -210,7 +194,7 @@ GrpcRequestHandler::DropIndex(::grpc::ServerContext *context, return ::grpc::Status::OK; } -} // namespace grpc -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestHandler.h b/cpp/src/server/grpc_impl/GrpcRequestHandler.h index 549e1d9d35..73f05572de 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestHandler.h +++ b/cpp/src/server/grpc_impl/GrpcRequestHandler.h @@ -45,8 +45,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param context */ ::grpc::Status - CreateTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableSchema *request, ::milvus::grpc::Status *response) override; + CreateTable(::grpc::ServerContext* context, const ::milvus::grpc::TableSchema* request, + ::milvus::grpc::Status* response) override; /** * @brief Test table existence method @@ -64,8 +64,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param context */ ::grpc::Status - HasTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, ::milvus::grpc::BoolReply *response) override; + HasTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::BoolReply* response) override; /** * @brief Drop table method @@ -83,8 +83,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param context */ ::grpc::Status - DropTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, ::milvus::grpc::Status *response) override; + DropTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::Status* response) override; /** * @brief build index by table method @@ -102,8 +102,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param context */ ::grpc::Status - CreateIndex(::grpc::ServerContext *context, - const ::milvus::grpc::IndexParam *request, ::milvus::grpc::Status *response) override; + CreateIndex(::grpc::ServerContext* context, const ::milvus::grpc::IndexParam* request, + ::milvus::grpc::Status* response) override; /** * @brief Insert vector array to table @@ -121,9 +121,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param response */ ::grpc::Status - Insert(::grpc::ServerContext *context, - const ::milvus::grpc::InsertParam *request, - ::milvus::grpc::VectorIds *response) override; + Insert(::grpc::ServerContext* context, const ::milvus::grpc::InsertParam* request, + ::milvus::grpc::VectorIds* response) override; /** * @brief Query vector @@ -146,9 +145,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param writer */ ::grpc::Status - Search(::grpc::ServerContext *context, - const ::milvus::grpc::SearchParam *request, - ::milvus::grpc::TopKQueryResultList *response) override; + Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request, + ::milvus::grpc::TopKQueryResultList* response) override; /** * @brief Internal use query interface @@ -171,9 +169,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param writer */ ::grpc::Status - SearchInFiles(::grpc::ServerContext *context, - const ::milvus::grpc::SearchInFilesParam *request, - ::milvus::grpc::TopKQueryResultList *response) override; + SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request, + ::milvus::grpc::TopKQueryResultList* response) override; /** * @brief Get table schema @@ -191,9 +188,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param response */ ::grpc::Status - DescribeTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::TableSchema *response) override; + DescribeTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::TableSchema* response) override; /** * @brief Get table row count @@ -211,9 +207,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param context */ ::grpc::Status - CountTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::TableRowCount *response) override; + CountTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::TableRowCount* response) override; /** * @brief List all tables in database @@ -231,9 +226,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param writer */ ::grpc::Status - ShowTables(::grpc::ServerContext *context, - const ::milvus::grpc::Command *request, - ::milvus::grpc::TableNameList *table_name_list) override; + ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, + ::milvus::grpc::TableNameList* table_name_list) override; /** * @brief Give the server status @@ -251,9 +245,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param response */ ::grpc::Status - Cmd(::grpc::ServerContext *context, - const ::milvus::grpc::Command *request, - ::milvus::grpc::StringReply *response) override; + Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, + ::milvus::grpc::StringReply* response) override; /** * @brief delete table by range @@ -270,9 +263,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param response */ ::grpc::Status - DeleteByRange(::grpc::ServerContext *context, - const ::milvus::grpc::DeleteByRangeParam *request, - ::milvus::grpc::Status *response) override; + DeleteByRange(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByRangeParam* request, + ::milvus::grpc::Status* response) override; /** * @brief preload table @@ -289,9 +281,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param response */ ::grpc::Status - PreloadTable(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::Status *response) override; + PreloadTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::Status* response) override; /** * @brief Describe index @@ -308,9 +299,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param response */ ::grpc::Status - DescribeIndex(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::IndexParam *response) override; + DescribeIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::IndexParam* response) override; /** * @brief Drop index @@ -327,13 +317,11 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { * @param response */ ::grpc::Status - DropIndex(::grpc::ServerContext *context, - const ::milvus::grpc::TableName *request, - ::milvus::grpc::Status *response) override; + DropIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::Status* response) override; }; -} // namespace grpc -} // namespace server -} // namespace milvus -} // namespace zilliz - +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp index 4c58195f37..fc302414c2 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp @@ -70,13 +70,11 @@ ErrorMap(ErrorCode code) { return ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR; } } -} // namespace +} // namespace //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -GrpcBaseTask::GrpcBaseTask(const std::string &task_group, bool async) - : task_group_(task_group), - async_(async), - done_(false) { +GrpcBaseTask::GrpcBaseTask(const std::string& task_group, bool async) + : task_group_(task_group), async_(async), done_(false) { } GrpcBaseTask::~GrpcBaseTask() { @@ -97,7 +95,7 @@ GrpcBaseTask::Done() { } Status -GrpcBaseTask::SetStatus(ErrorCode error_code, const std::string &error_msg) { +GrpcBaseTask::SetStatus(ErrorCode error_code, const std::string& error_msg) { status_ = Status(error_code, error_msg); SERVER_LOG_ERROR << error_msg; return status_; @@ -106,16 +104,13 @@ GrpcBaseTask::SetStatus(ErrorCode error_code, const std::string &error_msg) { Status GrpcBaseTask::WaitToFinish() { std::unique_lock lock(finish_mtx_); - finish_cond_.wait(lock, [this] { - return done_; - }); + finish_cond_.wait(lock, [this] { return done_; }); return status_; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -GrpcRequestScheduler::GrpcRequestScheduler() - : stopped_(false) { +GrpcRequestScheduler::GrpcRequestScheduler() : stopped_(false) { Start(); } @@ -124,17 +119,17 @@ GrpcRequestScheduler::~GrpcRequestScheduler() { } void -GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Status *grpc_status) { +GrpcRequestScheduler::ExecTask(BaseTaskPtr& task_ptr, ::milvus::grpc::Status* grpc_status) { if (task_ptr == nullptr) { return; } - GrpcRequestScheduler &scheduler = GrpcRequestScheduler::GetInstance(); + GrpcRequestScheduler& scheduler = GrpcRequestScheduler::GetInstance(); scheduler.ExecuteTask(task_ptr); if (!task_ptr->IsAsync()) { task_ptr->WaitToFinish(); - const Status &status = task_ptr->status(); + const Status& status = task_ptr->status(); if (!status.ok()) { grpc_status->set_reason(status.message()); grpc_status->set_error_code(ErrorMap(status.code())); @@ -168,8 +163,7 @@ GrpcRequestScheduler::Stop() { } for (auto iter : execute_threads_) { - if (iter == nullptr) - continue; + if (iter == nullptr) continue; iter->join(); } @@ -178,7 +172,7 @@ GrpcRequestScheduler::Stop() { } Status -GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr &task_ptr) { +GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr& task_ptr) { if (task_ptr == nullptr) { return Status::OK(); } @@ -190,10 +184,10 @@ GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr &task_ptr) { } if (task_ptr->IsAsync()) { - return Status::OK(); //async execution, caller need to call WaitToFinish at somewhere + return Status::OK(); // async execution, caller need to call WaitToFinish at somewhere } - return task_ptr->WaitToFinish();//sync execution + return task_ptr->WaitToFinish(); // sync execution } void @@ -206,7 +200,7 @@ GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) { BaseTaskPtr task = task_queue->Take(); if (task == nullptr) { SERVER_LOG_ERROR << "Take null from task queue, stop thread"; - break;//stop the thread + break; // stop the thread } try { @@ -214,14 +208,14 @@ GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) { if (!status.ok()) { SERVER_LOG_ERROR << "Task failed with code: " << status.ToString(); } - } catch (std::exception &ex) { + } catch (std::exception& ex) { SERVER_LOG_ERROR << "Task failed to execute: " << ex.what(); } } } Status -GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) { +GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr& task_ptr) { std::lock_guard lock(queue_mtx_); std::string group_name = task_ptr->TaskGroup(); @@ -232,7 +226,7 @@ GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) { queue->Put(task_ptr); task_groups_.insert(std::make_pair(group_name, queue)); - //start a thread + // start a thread ThreadPtr thread = std::make_shared(&GrpcRequestScheduler::TakeTaskToExecute, this, queue); execute_threads_.push_back(thread); SERVER_LOG_INFO << "Create new thread for task group: " << group_name; @@ -241,7 +235,7 @@ GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) { return Status::OK(); } -} // namespace grpc -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.h b/cpp/src/server/grpc_impl/GrpcRequestScheduler.h index df5357a4bb..b5a87c0bfa 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.h +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.h @@ -17,16 +17,16 @@ #pragma once -#include "utils/Status.h" -#include "utils/BlockingQueue.h" #include "grpc/gen-status/status.grpc.pb.h" #include "grpc/gen-status/status.pb.h" +#include "utils/BlockingQueue.h" +#include "utils/Status.h" #include -#include -#include #include #include +#include +#include namespace zilliz { namespace milvus { @@ -35,33 +35,41 @@ namespace grpc { class GrpcBaseTask { protected: - explicit GrpcBaseTask(const std::string &task_group, bool async = false); + explicit GrpcBaseTask(const std::string& task_group, bool async = false); virtual ~GrpcBaseTask(); public: - Status Execute(); + Status + Execute(); - void Done(); + void + Done(); - Status WaitToFinish(); + Status + WaitToFinish(); - std::string TaskGroup() const { + std::string + TaskGroup() const { return task_group_; } - const Status &status() const { + const Status& + status() const { return status_; } - bool IsAsync() const { + bool + IsAsync() const { return async_; } protected: - virtual Status OnExecute() = 0; + virtual Status + OnExecute() = 0; - Status SetStatus(ErrorCode error_code, const std::string &msg); + Status + SetStatus(ErrorCode error_code, const std::string& msg); protected: mutable std::mutex finish_mtx_; @@ -80,27 +88,34 @@ using ThreadPtr = std::shared_ptr; class GrpcRequestScheduler { public: - static GrpcRequestScheduler &GetInstance() { + static GrpcRequestScheduler& + GetInstance() { static GrpcRequestScheduler scheduler; return scheduler; } - void Start(); + void + Start(); - void Stop(); + void + Stop(); - Status ExecuteTask(const BaseTaskPtr &task_ptr); + Status + ExecuteTask(const BaseTaskPtr& task_ptr); - static void ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Status *grpc_status); + static void + ExecTask(BaseTaskPtr& task_ptr, ::milvus::grpc::Status* grpc_status); protected: GrpcRequestScheduler(); virtual ~GrpcRequestScheduler(); - void TakeTaskToExecute(TaskQueuePtr task_queue); + void + TakeTaskToExecute(TaskQueuePtr task_queue); - Status PutTaskToQueue(const BaseTaskPtr &task_ptr); + Status + PutTaskToQueue(const BaseTaskPtr& task_ptr); private: mutable std::mutex queue_mtx_; @@ -112,7 +127,7 @@ class GrpcRequestScheduler { bool stopped_; }; -} // namespace grpc -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index 5702b80bec..7ced2e69c1 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -19,29 +19,29 @@ #include #include -#include #include +#include //#include -#include "server/Server.h" +#include "../../../version.h" +#include "GrpcServer.h" +#include "db/Utils.h" +#include "scheduler/SchedInst.h" #include "server/DBWrapper.h" +#include "server/Server.h" #include "utils/CommonUtil.h" #include "utils/Log.h" #include "utils/TimeRecorder.h" #include "utils/ValidationUtil.h" -#include "GrpcServer.h" -#include "db/Utils.h" -#include "scheduler/SchedInst.h" -#include "../../../version.h" namespace zilliz { namespace milvus { namespace server { namespace grpc { -static const char *DQL_TASK_GROUP = "dql"; -static const char *DDL_DML_TASK_GROUP = "ddl_dml"; -static const char *PING_TASK_GROUP = "ping"; +static const char* DQL_TASK_GROUP = "dql"; +static const char* DDL_DML_TASK_GROUP = "ddl_dml"; +static const char* PING_TASK_GROUP = "ping"; using DB_META = zilliz::milvus::engine::meta::Meta; using DB_DATE = zilliz::milvus::engine::meta::DateT; @@ -82,10 +82,9 @@ IndexType(engine::EngineType type) { constexpr int64_t DAY_SECONDS = 24 * 60 * 60; Status -ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range> &range_array, - std::vector &dates) { +ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range>& range_array, std::vector& dates) { dates.clear(); - for (auto &range : range_array) { + for (auto& range : range_array) { time_t tt_start, tt_end; tm tm_start, tm_end; if (!CommonUtil::TimeStrToTime(range.start_value(), tt_start, tm_start)) { @@ -96,37 +95,34 @@ ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range> &range_array, return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); } - int64_t days = (tt_end > tt_start) ? (tt_end - tt_start) / DAY_SECONDS : (tt_start - tt_end) / - DAY_SECONDS; + int64_t days = (tt_end > tt_start) ? (tt_end - tt_start) / DAY_SECONDS : (tt_start - tt_end) / DAY_SECONDS; if (days == 0) { return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value() + " to " + range.end_value()); } - //range: [start_day, end_day) + // range: [start_day, end_day) for (int64_t i = 0; i < days; i++) { time_t tt_day = tt_start + DAY_SECONDS * i; tm tm_day; CommonUtil::ConvertTime(tt_day, tm_day); - int64_t date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 + - tm_day.tm_mday;//according to db logic + int64_t date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 + tm_day.tm_mday; // according to db logic dates.push_back(date); } } return Status::OK(); } -} // namespace +} // namespace //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CreateTableTask::CreateTableTask(const ::milvus::grpc::TableSchema *schema) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - schema_(schema) { +CreateTableTask::CreateTableTask(const ::milvus::grpc::TableSchema* schema) + : GrpcBaseTask(DDL_DML_TASK_GROUP), schema_(schema) { } BaseTaskPtr -CreateTableTask::Create(const ::milvus::grpc::TableSchema *schema) { +CreateTableTask::Create(const ::milvus::grpc::TableSchema* schema) { if (schema == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; @@ -139,7 +135,7 @@ CreateTableTask::OnExecute() { TimeRecorder rc("CreateTableTask"); try { - //step 1: check arguments + // step 1: check arguments auto status = ValidationUtil::ValidateTableName(schema_->table_name()); if (!status.ok()) { return status; @@ -160,23 +156,23 @@ CreateTableTask::OnExecute() { return status; } - //step 2: construct table schema + // step 2: construct table schema engine::meta::TableSchema table_info; table_info.table_id_ = schema_->table_name(); - table_info.dimension_ = (uint16_t) schema_->dimension(); + table_info.dimension_ = (uint16_t)schema_->dimension(); table_info.index_file_size_ = schema_->index_file_size(); table_info.metric_type_ = schema_->metric_type(); - //step 3: create table + // step 3: create table status = DBWrapper::DB()->CreateTable(table_info); if (!status.ok()) { - //table could exist + // table could exist if (status.code() == DB_ALREADY_EXIST) { return Status(SERVER_INVALID_TABLE_NAME, status.message()); } return status; } - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -186,14 +182,12 @@ CreateTableTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DescribeTableTask::DescribeTableTask(const std::string &table_name, ::milvus::grpc::TableSchema *schema) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name), - schema_(schema) { +DescribeTableTask::DescribeTableTask(const std::string& table_name, ::milvus::grpc::TableSchema* schema) + : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name), schema_(schema) { } BaseTaskPtr -DescribeTableTask::Create(const std::string &table_name, ::milvus::grpc::TableSchema *schema) { +DescribeTableTask::Create(const std::string& table_name, ::milvus::grpc::TableSchema* schema) { return std::shared_ptr(new DescribeTableTask(table_name, schema)); } @@ -202,13 +196,13 @@ DescribeTableTask::OnExecute() { TimeRecorder rc("DescribeTableTask"); try { - //step 1: check arguments + // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { return status; } - //step 2: get table info + // step 2: get table info engine::meta::TableSchema table_info; table_info.table_id_ = table_name_; status = DBWrapper::DB()->DescribeTable(table_info); @@ -220,7 +214,7 @@ DescribeTableTask::OnExecute() { schema_->set_dimension(table_info.dimension_); schema_->set_index_file_size(table_info.index_file_size_); schema_->set_metric_type(table_info.metric_type_); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -230,13 +224,12 @@ DescribeTableTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CreateIndexTask::CreateIndexTask(const ::milvus::grpc::IndexParam *index_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - index_param_(index_param) { +CreateIndexTask::CreateIndexTask(const ::milvus::grpc::IndexParam* index_param) + : GrpcBaseTask(DDL_DML_TASK_GROUP), index_param_(index_param) { } BaseTaskPtr -CreateIndexTask::Create(const ::milvus::grpc::IndexParam *index_param) { +CreateIndexTask::Create(const ::milvus::grpc::IndexParam* index_param) { if (index_param == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; @@ -249,7 +242,7 @@ CreateIndexTask::OnExecute() { try { TimeRecorder rc("CreateIndexTask"); - //step 1: check arguments + // step 1: check arguments std::string table_name_ = index_param_->table_name(); auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { @@ -266,7 +259,7 @@ CreateIndexTask::OnExecute() { return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); } - auto &grpc_index = index_param_->index(); + auto& grpc_index = index_param_->index(); status = ValidationUtil::ValidateTableIndexType(grpc_index.index_type()); if (!status.ok()) { return status; @@ -277,7 +270,7 @@ CreateIndexTask::OnExecute() { return status; } - //step 2: check table existence + // step 2: check table existence engine::TableIndex index; index.engine_type_ = grpc_index.index_type(); index.nlist_ = grpc_index.nlist(); @@ -287,7 +280,7 @@ CreateIndexTask::OnExecute() { } rc.ElapseFromBegin("totally cost"); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -295,14 +288,12 @@ CreateIndexTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -HasTableTask::HasTableTask(const std::string &table_name, bool &has_table) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name), - has_table_(has_table) { +HasTableTask::HasTableTask(const std::string& table_name, bool& has_table) + : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name), has_table_(has_table) { } BaseTaskPtr -HasTableTask::Create(const std::string &table_name, bool &has_table) { +HasTableTask::Create(const std::string& table_name, bool& has_table) { return std::shared_ptr(new HasTableTask(table_name, has_table)); } @@ -311,20 +302,20 @@ HasTableTask::OnExecute() { try { TimeRecorder rc("HasTableTask"); - //step 1: check arguments + // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { return status; } - //step 2: check table existence + // step 2: check table existence status = DBWrapper::DB()->HasTable(table_name_, has_table_); if (!status.ok()) { return status; } rc.ElapseFromBegin("totally cost"); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -332,13 +323,12 @@ HasTableTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DropTableTask::DropTableTask(const std::string &table_name) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name) { +DropTableTask::DropTableTask(const std::string& table_name) + : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name) { } BaseTaskPtr -DropTableTask::Create(const std::string &table_name) { +DropTableTask::Create(const std::string& table_name) { return std::shared_ptr(new DropTableTask(table_name)); } @@ -347,13 +337,13 @@ DropTableTask::OnExecute() { try { TimeRecorder rc("DropTableTask"); - //step 1: check arguments + // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { return status; } - //step 2: check table existence + // step 2: check table existence engine::meta::TableSchema table_info; table_info.table_id_ = table_name_; status = DBWrapper::DB()->DescribeTable(table_info); @@ -367,7 +357,7 @@ DropTableTask::OnExecute() { rc.ElapseFromBegin("check validation"); - //step 3: Drop table + // step 3: Drop table std::vector dates; status = DBWrapper::DB()->DeleteTable(table_name_, dates); if (!status.ok()) { @@ -375,7 +365,7 @@ DropTableTask::OnExecute() { } rc.ElapseFromBegin("total cost"); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -383,13 +373,12 @@ DropTableTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -ShowTablesTask::ShowTablesTask(::milvus::grpc::TableNameList *table_name_list) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_list_(table_name_list) { +ShowTablesTask::ShowTablesTask(::milvus::grpc::TableNameList* table_name_list) + : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_list_(table_name_list) { } BaseTaskPtr -ShowTablesTask::Create(::milvus::grpc::TableNameList *table_name_list) { +ShowTablesTask::Create(::milvus::grpc::TableNameList* table_name_list) { return std::shared_ptr(new ShowTablesTask(table_name_list)); } @@ -401,23 +390,19 @@ ShowTablesTask::OnExecute() { return statuts; } - for (auto &schema : schema_array) { + for (auto& schema : schema_array) { table_name_list_->add_table_names(schema.table_id_); } return Status::OK(); } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -InsertTask::InsertTask(const ::milvus::grpc::InsertParam *insert_param, - ::milvus::grpc::VectorIds *record_ids) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - insert_param_(insert_param), - record_ids_(record_ids) { +InsertTask::InsertTask(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids) + : GrpcBaseTask(DDL_DML_TASK_GROUP), insert_param_(insert_param), record_ids_(record_ids) { } BaseTaskPtr -InsertTask::Create(const ::milvus::grpc::InsertParam *insert_param, - ::milvus::grpc::VectorIds *record_ids) { +InsertTask::Create(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids) { if (insert_param == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; @@ -430,7 +415,7 @@ InsertTask::OnExecute() { try { TimeRecorder rc("InsertVectorTask"); - //step 1: check arguments + // step 1: check arguments auto status = ValidationUtil::ValidateTableName(insert_param_->table_name()); if (!status.ok()) { return status; @@ -441,34 +426,32 @@ InsertTask::OnExecute() { if (!record_ids_->vector_id_array().empty()) { if (record_ids_->vector_id_array().size() != insert_param_->row_record_array_size()) { - return Status(SERVER_ILLEGAL_VECTOR_ID, - "Size of vector ids is not equal to row record array size"); + return Status(SERVER_ILLEGAL_VECTOR_ID, "Size of vector ids is not equal to row record array size"); } } - //step 2: check table existence + // step 2: check table existence engine::meta::TableSchema table_info; table_info.table_id_ = insert_param_->table_name(); status = DBWrapper::DB()->DescribeTable(table_info); if (!status.ok()) { if (status.code() == DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, - "Table " + insert_param_->table_name() + " not exists"); + return Status(SERVER_TABLE_NOT_EXIST, "Table " + insert_param_->table_name() + " not exists"); } else { return status; } } - //step 3: check table flag - //all user provide id, or all internal id + // step 3: check table flag + // all user provide id, or all internal id bool user_provide_ids = !insert_param_->row_id_array().empty(); - //user already provided id before, all insert action require user id + // user already provided id before, all insert action require user id if ((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) && !user_provide_ids) { return Status(SERVER_ILLEGAL_VECTOR_ID, "Table vector ids are user defined, please provide id for this batch"); } - //user didn't provided id before, no need to provide user id + // user didn't provided id before, no need to provide user id if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) && user_provide_ids) { return Status(SERVER_ILLEGAL_VECTOR_ID, "Table vector ids are auto generated, no need to provide id for this batch"); @@ -477,12 +460,12 @@ InsertTask::OnExecute() { rc.RecordSection("check validation"); #ifdef MILVUS_ENABLE_PROFILING - std::string fname = "/tmp/insert_" + std::to_string(this->insert_param_->row_record_array_size()) - + ".profiling"; + std::string fname = + "/tmp/insert_" + std::to_string(this->insert_param_->row_record_array_size()) + ".profiling"; ProfilerStart(fname.c_str()); #endif - //step 4: prepare float data + // step 4: prepare float data std::vector vec_f(insert_param_->row_record_array_size() * table_info.dimension_, 0); // TODO: change to one dimension array in protobuf or use multiple-thread to copy the data @@ -493,25 +476,23 @@ InsertTask::OnExecute() { uint64_t vec_dim = insert_param_->row_record_array(i).vector_data().size(); if (vec_dim != table_info.dimension_) { ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; - std::string error_msg = "Invalid row record dimension: " + std::to_string(vec_dim) - + " vs. table dimension:" + - std::to_string(table_info.dimension_); + std::string error_msg = "Invalid row record dimension: " + std::to_string(vec_dim) + + " vs. table dimension:" + std::to_string(table_info.dimension_); return Status(error_code, error_msg); } - memcpy(&vec_f[i * table_info.dimension_], - insert_param_->row_record_array(i).vector_data().data(), + memcpy(&vec_f[i * table_info.dimension_], insert_param_->row_record_array(i).vector_data().data(), table_info.dimension_ * sizeof(float)); } rc.ElapseFromBegin("prepare vectors data"); - //step 5: insert vectors - auto vec_count = (uint64_t) insert_param_->row_record_array_size(); + // step 5: insert vectors + auto vec_count = (uint64_t)insert_param_->row_record_array_size(); std::vector vec_ids(insert_param_->row_id_array_size(), 0); if (!insert_param_->row_id_array().empty()) { - const int64_t *src_data = insert_param_->row_id_array().data(); - int64_t *target_data = vec_ids.data(); - memcpy(target_data, src_data, (size_t) (sizeof(int64_t) * insert_param_->row_id_array_size())); + const int64_t* src_data = insert_param_->row_id_array().data(); + int64_t* target_data = vec_ids.data(); + memcpy(target_data, src_data, (size_t)(sizeof(int64_t) * insert_param_->row_id_array_size())); } status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), vec_count, vec_f.data(), vec_ids); @@ -525,14 +506,14 @@ InsertTask::OnExecute() { auto ids_size = record_ids_->vector_id_array_size(); if (ids_size != vec_count) { - std::string msg = "Add " + std::to_string(vec_count) + " vectors but only return " - + std::to_string(ids_size) + " id"; + std::string msg = + "Add " + std::to_string(vec_count) + " vectors but only return " + std::to_string(ids_size) + " id"; return Status(SERVER_ILLEGAL_VECTOR_ID, msg); } - //step 6: update table flag - user_provide_ids ? table_info.flag_ |= engine::meta::FLAG_MASK_HAS_USERID - : table_info.flag_ |= engine::meta::FLAG_MASK_NO_USERID; + // step 6: update table flag + user_provide_ids ? table_info.flag_ |= engine::meta::FLAG_MASK_HAS_USERID : table_info.flag_ |= + engine::meta::FLAG_MASK_NO_USERID; status = DBWrapper::DB()->UpdateTableFlag(insert_param_->table_name(), table_info.flag_); #ifdef MILVUS_ENABLE_PROFILING @@ -541,7 +522,7 @@ InsertTask::OnExecute() { rc.RecordSection("add vectors to engine"); rc.ElapseFromBegin("total cost"); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -549,9 +530,8 @@ InsertTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -SearchTask::SearchTask(const ::milvus::grpc::SearchParam *search_vector_infos, - const std::vector &file_id_array, - ::milvus::grpc::TopKQueryResultList *response) +SearchTask::SearchTask(const ::milvus::grpc::SearchParam* search_vector_infos, + const std::vector& file_id_array, ::milvus::grpc::TopKQueryResultList* response) : GrpcBaseTask(DQL_TASK_GROUP), search_param_(search_vector_infos), file_id_array_(file_id_array), @@ -559,9 +539,8 @@ SearchTask::SearchTask(const ::milvus::grpc::SearchParam *search_vector_infos, } BaseTaskPtr -SearchTask::Create(const ::milvus::grpc::SearchParam *search_vector_infos, - const std::vector &file_id_array, - ::milvus::grpc::TopKQueryResultList *response) { +SearchTask::Create(const ::milvus::grpc::SearchParam* search_vector_infos, + const std::vector& file_id_array, ::milvus::grpc::TopKQueryResultList* response) { if (search_vector_infos == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; @@ -578,14 +557,14 @@ SearchTask::OnExecute() { std::string hdr = "SearchTask(k=" + std::to_string(top_k) + ", nprob=" + std::to_string(nprobe) + ")"; TimeRecorder rc(hdr); - //step 1: check table name + // step 1: check table name std::string table_name_ = search_param_->table_name(); auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { return status; } - //step 2: check table existence + // step 2: check table existence engine::meta::TableSchema table_info; table_info.table_id_ = table_name_; status = DBWrapper::DB()->DescribeTable(table_info); @@ -597,7 +576,7 @@ SearchTask::OnExecute() { } } - //step 3: check search parameter + // step 3: check search parameter status = ValidationUtil::ValidateSearchTopk(top_k, table_info); if (!status.ok()) { return status; @@ -612,7 +591,7 @@ SearchTask::OnExecute() { return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty"); } - //step 4: check date range, and convert to db dates + // step 4: check date range, and convert to db dates std::vector dates; std::vector<::milvus::grpc::Range> range_array; for (size_t i = 0; i < search_param_->query_range_array_size(); i++) { @@ -626,8 +605,7 @@ SearchTask::OnExecute() { rc.RecordSection("check validation"); - - //step 5: prepare float data + // step 5: prepare float data auto record_array_size = search_param_->query_record_array_size(); std::vector vec_f(record_array_size * table_info.dimension_, 0); for (size_t i = 0; i < record_array_size; i++) { @@ -637,33 +615,32 @@ SearchTask::OnExecute() { uint64_t query_vec_dim = search_param_->query_record_array(i).vector_data().size(); if (query_vec_dim != table_info.dimension_) { ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; - std::string error_msg = "Invalid row record dimension: " + std::to_string(query_vec_dim) - + " vs. table dimension:" + std::to_string(table_info.dimension_); + std::string error_msg = "Invalid row record dimension: " + std::to_string(query_vec_dim) + + " vs. table dimension:" + std::to_string(table_info.dimension_); return Status(error_code, error_msg); } - memcpy(&vec_f[i * table_info.dimension_], - search_param_->query_record_array(i).vector_data().data(), + memcpy(&vec_f[i * table_info.dimension_], search_param_->query_record_array(i).vector_data().data(), table_info.dimension_ * sizeof(float)); } rc.RecordSection("prepare vector data"); - //step 6: search vectors + // step 6: search vectors engine::QueryResults results; - auto record_count = (uint64_t) search_param_->query_record_array().size(); + auto record_count = (uint64_t)search_param_->query_record_array().size(); #ifdef MILVUS_ENABLE_PROFILING - std::string fname = "/tmp/search_nq_" + std::to_string(this->search_param_->query_record_array_size()) - + ".profiling"; + std::string fname = + "/tmp/search_nq_" + std::to_string(this->search_param_->query_record_array_size()) + ".profiling"; ProfilerStart(fname.c_str()); #endif if (file_id_array_.empty()) { - status = DBWrapper::DB()->Query(table_name_, (size_t) top_k, record_count, nprobe, - vec_f.data(), dates, results); + status = + DBWrapper::DB()->Query(table_name_, (size_t)top_k, record_count, nprobe, vec_f.data(), dates, results); } else { - status = DBWrapper::DB()->Query(table_name_, file_id_array_, (size_t) top_k, - record_count, nprobe, vec_f.data(), dates, results); + status = DBWrapper::DB()->Query(table_name_, file_id_array_, (size_t)top_k, record_count, nprobe, + vec_f.data(), dates, results); } #ifdef MILVUS_ENABLE_PROFILING @@ -676,29 +653,29 @@ SearchTask::OnExecute() { } if (results.empty()) { - return Status::OK(); //empty table + return Status::OK(); // empty table } if (results.size() != record_count) { - std::string msg = "Search " + std::to_string(record_count) + " vectors but only return " - + std::to_string(results.size()) + " results"; + std::string msg = "Search " + std::to_string(record_count) + " vectors but only return " + + std::to_string(results.size()) + " results"; return Status(SERVER_ILLEGAL_SEARCH_RESULT, msg); } - //step 7: construct result array - for (auto &result : results) { - ::milvus::grpc::TopKQueryResult *topk_query_result = topk_result_list->add_topk_query_result(); - for (auto &pair : result) { - ::milvus::grpc::QueryResult *grpc_result = topk_query_result->add_query_result_arrays(); + // step 7: construct result array + for (auto& result : results) { + ::milvus::grpc::TopKQueryResult* topk_query_result = topk_result_list->add_topk_query_result(); + for (auto& pair : result) { + ::milvus::grpc::QueryResult* grpc_result = topk_query_result->add_query_result_arrays(); grpc_result->set_id(pair.first); grpc_result->set_distance(pair.second); } } - //step 8: print time cost percent + // step 8: print time cost percent rc.RecordSection("construct result and send"); rc.ElapseFromBegin("totally cost"); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -706,14 +683,12 @@ SearchTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CountTableTask::CountTableTask(const std::string &table_name, int64_t &row_count) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name), - row_count_(row_count) { +CountTableTask::CountTableTask(const std::string& table_name, int64_t& row_count) + : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name), row_count_(row_count) { } BaseTaskPtr -CountTableTask::Create(const std::string &table_name, int64_t &row_count) { +CountTableTask::Create(const std::string& table_name, int64_t& row_count) { return std::shared_ptr(new CountTableTask(table_name, row_count)); } @@ -722,23 +697,23 @@ CountTableTask::OnExecute() { try { TimeRecorder rc("GetTableRowCountTask"); - //step 1: check arguments + // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { return status; } - //step 2: get row count + // step 2: get row count uint64_t row_count = 0; status = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); if (!status.ok()) { return status; } - row_count_ = (int64_t) row_count; + row_count_ = (int64_t)row_count; rc.ElapseFromBegin("total cost"); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -746,14 +721,12 @@ CountTableTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CmdTask::CmdTask(const std::string &cmd, std::string &result) - : GrpcBaseTask(PING_TASK_GROUP), - cmd_(cmd), - result_(result) { +CmdTask::CmdTask(const std::string& cmd, std::string& result) + : GrpcBaseTask(PING_TASK_GROUP), cmd_(cmd), result_(result) { } BaseTaskPtr -CmdTask::Create(const std::string &cmd, std::string &result) { +CmdTask::Create(const std::string& cmd, std::string& result) { return std::shared_ptr(new CmdTask(cmd, result)); } @@ -771,13 +744,12 @@ CmdTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DeleteByRangeTask::DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - delete_by_range_param_(delete_by_range_param) { +DeleteByRangeTask::DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param) + : GrpcBaseTask(DDL_DML_TASK_GROUP), delete_by_range_param_(delete_by_range_param) { } BaseTaskPtr -DeleteByRangeTask::Create(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param) { +DeleteByRangeTask::Create(const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param) { if (delete_by_range_param == nullptr) { SERVER_LOG_ERROR << "grpc input is null!"; return nullptr; @@ -791,14 +763,14 @@ DeleteByRangeTask::OnExecute() { try { TimeRecorder rc("DeleteByRangeTask"); - //step 1: check arguments + // step 1: check arguments std::string table_name = delete_by_range_param_->table_name(); auto status = ValidationUtil::ValidateTableName(table_name); if (!status.ok()) { return status; } - //step 2: check table existence + // step 2: check table existence engine::meta::TableSchema table_info; table_info.table_id_ = table_name; status = DBWrapper::DB()->DescribeTable(table_info); @@ -812,7 +784,7 @@ DeleteByRangeTask::OnExecute() { rc.ElapseFromBegin("check validation"); - //step 3: check date range, and convert to db dates + // step 3: check date range, and convert to db dates std::vector dates; ErrorCode error_code = SERVER_SUCCESS; std::string error_msg; @@ -832,7 +804,7 @@ DeleteByRangeTask::OnExecute() { if (!status.ok()) { return status; } - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -840,13 +812,12 @@ DeleteByRangeTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -PreloadTableTask::PreloadTableTask(const std::string &table_name) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name) { +PreloadTableTask::PreloadTableTask(const std::string& table_name) + : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name) { } BaseTaskPtr -PreloadTableTask::Create(const std::string &table_name) { +PreloadTableTask::Create(const std::string& table_name) { return std::shared_ptr(new PreloadTableTask(table_name)); } @@ -855,20 +826,20 @@ PreloadTableTask::OnExecute() { try { TimeRecorder rc("PreloadTableTask"); - //step 1: check arguments + // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { return status; } - //step 2: check table existence + // step 2: check table existence status = DBWrapper::DB()->PreloadTable(table_name_); if (!status.ok()) { return status; } rc.ElapseFromBegin("totally cost"); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -876,16 +847,12 @@ PreloadTableTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DescribeIndexTask::DescribeIndexTask(const std::string &table_name, - ::milvus::grpc::IndexParam *index_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name), - index_param_(index_param) { +DescribeIndexTask::DescribeIndexTask(const std::string& table_name, ::milvus::grpc::IndexParam* index_param) + : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name), index_param_(index_param) { } BaseTaskPtr -DescribeIndexTask::Create(const std::string &table_name, - ::milvus::grpc::IndexParam *index_param) { +DescribeIndexTask::Create(const std::string& table_name, ::milvus::grpc::IndexParam* index_param) { return std::shared_ptr(new DescribeIndexTask(table_name, index_param)); } @@ -894,13 +861,13 @@ DescribeIndexTask::OnExecute() { try { TimeRecorder rc("DescribeIndexTask"); - //step 1: check arguments + // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { return status; } - //step 2: check table existence + // step 2: check table existence engine::TableIndex index; status = DBWrapper::DB()->DescribeIndex(table_name_, index); if (!status.ok()) { @@ -912,7 +879,7 @@ DescribeIndexTask::OnExecute() { index_param_->mutable_index()->set_nlist(index.nlist_); rc.ElapseFromBegin("totally cost"); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } @@ -920,13 +887,12 @@ DescribeIndexTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DropIndexTask::DropIndexTask(const std::string &table_name) - : GrpcBaseTask(DDL_DML_TASK_GROUP), - table_name_(table_name) { +DropIndexTask::DropIndexTask(const std::string& table_name) + : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name) { } BaseTaskPtr -DropIndexTask::Create(const std::string &table_name) { +DropIndexTask::Create(const std::string& table_name) { return std::shared_ptr(new DropIndexTask(table_name)); } @@ -935,7 +901,7 @@ DropIndexTask::OnExecute() { try { TimeRecorder rc("DropIndexTask"); - //step 1: check arguments + // step 1: check arguments auto status = ValidationUtil::ValidateTableName(table_name_); if (!status.ok()) { return status; @@ -951,21 +917,21 @@ DropIndexTask::OnExecute() { return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); } - //step 2: check table existence + // step 2: check table existence status = DBWrapper::DB()->DropIndex(table_name_); if (!status.ok()) { return status; } rc.ElapseFromBegin("totally cost"); - } catch (std::exception &ex) { + } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } return Status::OK(); } -} // namespace grpc -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.h b/cpp/src/server/grpc_impl/GrpcRequestTask.h index 4c8c038d44..f0e77ea791 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.h +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.h @@ -17,17 +17,17 @@ #pragma once +#include "db/Types.h" #include "server/grpc_impl/GrpcRequestScheduler.h" #include "utils/Status.h" -#include "db/Types.h" #include "grpc/gen-milvus/milvus.grpc.pb.h" #include "grpc/gen-status/status.pb.h" #include #include -#include #include +#include namespace zilliz { namespace milvus { @@ -38,60 +38,60 @@ namespace grpc { class CreateTableTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const ::milvus::grpc::TableSchema *schema); + Create(const ::milvus::grpc::TableSchema* schema); protected: - explicit CreateTableTask(const ::milvus::grpc::TableSchema *request); + explicit CreateTableTask(const ::milvus::grpc::TableSchema* request); Status OnExecute() override; private: - const ::milvus::grpc::TableSchema *schema_; + const ::milvus::grpc::TableSchema* schema_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class HasTableTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const std::string &table_name, bool &has_table); + Create(const std::string& table_name, bool& has_table); protected: - HasTableTask(const std::string &request, bool &has_table); + HasTableTask(const std::string& request, bool& has_table); Status OnExecute() override; private: std::string table_name_; - bool &has_table_; + bool& has_table_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DescribeTableTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const std::string &table_name, ::milvus::grpc::TableSchema *schema); + Create(const std::string& table_name, ::milvus::grpc::TableSchema* schema); protected: - DescribeTableTask(const std::string &table_name, ::milvus::grpc::TableSchema *schema); + DescribeTableTask(const std::string& table_name, ::milvus::grpc::TableSchema* schema); Status OnExecute() override; private: std::string table_name_; - ::milvus::grpc::TableSchema *schema_; + ::milvus::grpc::TableSchema* schema_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DropTableTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const std::string &table_name); + Create(const std::string& table_name); protected: - explicit DropTableTask(const std::string &table_name); + explicit DropTableTask(const std::string& table_name); Status OnExecute() override; @@ -104,133 +104,129 @@ class DropTableTask : public GrpcBaseTask { class CreateIndexTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const ::milvus::grpc::IndexParam *index_Param); + Create(const ::milvus::grpc::IndexParam* index_Param); protected: - explicit CreateIndexTask(const ::milvus::grpc::IndexParam *index_Param); + explicit CreateIndexTask(const ::milvus::grpc::IndexParam* index_Param); Status OnExecute() override; private: - const ::milvus::grpc::IndexParam *index_param_; + const ::milvus::grpc::IndexParam* index_param_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class ShowTablesTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(::milvus::grpc::TableNameList *table_name_list); + Create(::milvus::grpc::TableNameList* table_name_list); protected: - explicit ShowTablesTask(::milvus::grpc::TableNameList *table_name_list); + explicit ShowTablesTask(::milvus::grpc::TableNameList* table_name_list); Status OnExecute() override; private: - ::milvus::grpc::TableNameList *table_name_list_; + ::milvus::grpc::TableNameList* table_name_list_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class InsertTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const ::milvus::grpc::InsertParam *insert_Param, - ::milvus::grpc::VectorIds *record_ids_); + Create(const ::milvus::grpc::InsertParam* insert_Param, ::milvus::grpc::VectorIds* record_ids_); protected: - InsertTask(const ::milvus::grpc::InsertParam *insert_Param, - ::milvus::grpc::VectorIds *record_ids_); + InsertTask(const ::milvus::grpc::InsertParam* insert_Param, ::milvus::grpc::VectorIds* record_ids_); Status OnExecute() override; private: - const ::milvus::grpc::InsertParam *insert_param_; - ::milvus::grpc::VectorIds *record_ids_; + const ::milvus::grpc::InsertParam* insert_param_; + ::milvus::grpc::VectorIds* record_ids_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class SearchTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const ::milvus::grpc::SearchParam *search_param, - const std::vector &file_id_array, - ::milvus::grpc::TopKQueryResultList *response); + Create(const ::milvus::grpc::SearchParam* search_param, const std::vector& file_id_array, + ::milvus::grpc::TopKQueryResultList* response); protected: - SearchTask(const ::milvus::grpc::SearchParam *search_param, - const std::vector &file_id_array, - ::milvus::grpc::TopKQueryResultList *response); + SearchTask(const ::milvus::grpc::SearchParam* search_param, const std::vector& file_id_array, + ::milvus::grpc::TopKQueryResultList* response); Status OnExecute() override; private: - const ::milvus::grpc::SearchParam *search_param_; + const ::milvus::grpc::SearchParam* search_param_; std::vector file_id_array_; - ::milvus::grpc::TopKQueryResultList *topk_result_list; + ::milvus::grpc::TopKQueryResultList* topk_result_list; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CountTableTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const std::string &table_name, int64_t &row_count); + Create(const std::string& table_name, int64_t& row_count); protected: - CountTableTask(const std::string &table_name, int64_t &row_count); + CountTableTask(const std::string& table_name, int64_t& row_count); Status OnExecute() override; private: std::string table_name_; - int64_t &row_count_; + int64_t& row_count_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class CmdTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const std::string &cmd, std::string &result); + Create(const std::string& cmd, std::string& result); protected: - CmdTask(const std::string &cmd, std::string &result); + CmdTask(const std::string& cmd, std::string& result); Status OnExecute() override; private: std::string cmd_; - std::string &result_; + std::string& result_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DeleteByRangeTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param); + Create(const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param); protected: - explicit DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param); + explicit DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param); Status OnExecute() override; private: - const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param_; + const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class PreloadTableTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const std::string &table_name); + Create(const std::string& table_name); protected: - explicit PreloadTableTask(const std::string &table_name); + explicit PreloadTableTask(const std::string& table_name); Status OnExecute() override; @@ -243,29 +239,27 @@ class PreloadTableTask : public GrpcBaseTask { class DescribeIndexTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const std::string &table_name, - ::milvus::grpc::IndexParam *index_param); + Create(const std::string& table_name, ::milvus::grpc::IndexParam* index_param); protected: - DescribeIndexTask(const std::string &table_name, - ::milvus::grpc::IndexParam *index_param); + DescribeIndexTask(const std::string& table_name, ::milvus::grpc::IndexParam* index_param); Status OnExecute() override; private: std::string table_name_; - ::milvus::grpc::IndexParam *index_param_; + ::milvus::grpc::IndexParam* index_param_; }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DropIndexTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const std::string &table_name); + Create(const std::string& table_name); protected: - explicit DropIndexTask(const std::string &table_name); + explicit DropIndexTask(const std::string& table_name); Status OnExecute() override; @@ -274,7 +268,7 @@ class DropIndexTask : public GrpcBaseTask { std::string table_name_; }; -} // namespace grpc -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcServer.cpp b/cpp/src/server/grpc_impl/GrpcServer.cpp index 065271dd55..dd7895c679 100644 --- a/cpp/src/server/grpc_impl/GrpcServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcServer.cpp @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. -#include "grpc/gen-milvus/milvus.grpc.pb.h" #include "server/grpc_impl/GrpcServer.h" +#include "GrpcRequestHandler.h" +#include "grpc/gen-milvus/milvus.grpc.pb.h" #include "server/Config.h" #include "server/DBWrapper.h" #include "utils/Log.h" -#include "GrpcRequestHandler.h" #include #include @@ -43,14 +43,16 @@ namespace grpc { constexpr int64_t MESSAGE_SIZE = -1; -//this class is to check port occupation during server start +// this class is to check port occupation during server start class NoReusePortOption : public ::grpc::ServerBuilderOption { public: - void UpdateArguments(::grpc::ChannelArguments *args) override { + void + UpdateArguments(::grpc::ChannelArguments* args) override { args->SetInt(GRPC_ARG_ALLOW_REUSEPORT, 0); } - void UpdatePlugins(std::vector> *plugins) override { + void + UpdatePlugins(std::vector>* plugins) override { } }; @@ -70,7 +72,7 @@ GrpcServer::Stop() { Status GrpcServer::StartService() { - Config &config = Config::GetInstance(); + Config& config = Config::GetInstance(); std::string address, port; Status s; @@ -87,7 +89,7 @@ GrpcServer::StartService() { ::grpc::ServerBuilder builder; builder.SetOption(std::unique_ptr<::grpc::ServerBuilderOption>(new NoReusePortOption)); - builder.SetMaxReceiveMessageSize(MESSAGE_SIZE); //default 4 * 1024 * 1024 + builder.SetMaxReceiveMessageSize(MESSAGE_SIZE); // default 4 * 1024 * 1024 builder.SetMaxSendMessageSize(MESSAGE_SIZE); builder.SetCompressionAlgorithmSupportStatus(GRPC_COMPRESS_STREAM_GZIP, true); @@ -114,7 +116,7 @@ GrpcServer::StopService() { return Status::OK(); } -} // namespace grpc -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/server/grpc_impl/GrpcServer.h b/cpp/src/server/grpc_impl/GrpcServer.h index 9101f144b3..9f3f827936 100644 --- a/cpp/src/server/grpc_impl/GrpcServer.h +++ b/cpp/src/server/grpc_impl/GrpcServer.h @@ -19,11 +19,11 @@ #include "utils/Status.h" -#include +#include #include +#include #include #include -#include namespace zilliz { namespace milvus { @@ -32,27 +32,32 @@ namespace grpc { class GrpcServer { public: - static GrpcServer &GetInstance() { + static GrpcServer& + GetInstance() { static GrpcServer grpc_server; return grpc_server; } - void Start(); - void Stop(); + void + Start(); + void + Stop(); private: GrpcServer() = default; ~GrpcServer() = default; - Status StartService(); - Status StopService(); + Status + StartService(); + Status + StopService(); private: std::unique_ptr<::grpc::Server> server_ptr_; std::shared_ptr thread_ptr_; }; -} // namespace grpc -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace grpc +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/BlockingQueue.h b/cpp/src/utils/BlockingQueue.h index b98fe28ef8..422f170cd6 100644 --- a/cpp/src/utils/BlockingQueue.h +++ b/cpp/src/utils/BlockingQueue.h @@ -27,29 +27,37 @@ namespace zilliz { namespace milvus { namespace server { -template +template class BlockingQueue { public: BlockingQueue() : mtx(), full_(), empty_() { } - BlockingQueue(const BlockingQueue &rhs) = delete; + BlockingQueue(const BlockingQueue& rhs) = delete; - BlockingQueue &operator=(const BlockingQueue &rhs) = delete; + BlockingQueue& + operator=(const BlockingQueue& rhs) = delete; - void Put(const T &task); + void + Put(const T& task); - T Take(); + T + Take(); - T Front(); + T + Front(); - T Back(); + T + Back(); - size_t Size(); + size_t + Size(); - bool Empty(); + bool + Empty(); - void SetCapacity(const size_t capacity); + void + SetCapacity(const size_t capacity); private: mutable std::mutex mtx; @@ -59,8 +67,8 @@ class BlockingQueue { size_t capacity_ = 32; }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz #include "./BlockingQueue.inl" diff --git a/cpp/src/utils/CommonUtil.cpp b/cpp/src/utils/CommonUtil.cpp index 0116e321e0..760c54c4a7 100644 --- a/cpp/src/utils/CommonUtil.cpp +++ b/cpp/src/utils/CommonUtil.cpp @@ -18,15 +18,15 @@ #include "utils/CommonUtil.h" #include "utils/Log.h" -#include -#include -#include -#include -#include #include +#include #include -#include +#include +#include #include +#include +#include +#include #include "boost/filesystem.hpp" @@ -45,29 +45,28 @@ namespace server { namespace fs = boost::filesystem; bool -CommonUtil::GetSystemMemInfo(uint64_t &total_mem, uint64_t &free_mem) { +CommonUtil::GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem) { struct sysinfo info; int ret = sysinfo(&info); total_mem = info.totalram; free_mem = info.freeram; - return ret == 0;//succeed 0, failed -1 + return ret == 0; // succeed 0, failed -1 } bool -CommonUtil::GetSystemAvailableThreads(uint32_t &thread_count) { - //threadCnt = std::thread::hardware_concurrency(); +CommonUtil::GetSystemAvailableThreads(uint32_t& thread_count) { + // threadCnt = std::thread::hardware_concurrency(); thread_count = sysconf(_SC_NPROCESSORS_CONF); thread_count *= THREAD_MULTIPLY_CPU; - if (thread_count == 0) - thread_count = 8; + if (thread_count == 0) thread_count = 8; return true; } bool -CommonUtil::IsDirectoryExist(const std::string &path) { - DIR *dp = nullptr; +CommonUtil::IsDirectoryExist(const std::string& path) { + DIR* dp = nullptr; if ((dp = opendir(path.c_str())) == nullptr) { return false; } @@ -77,7 +76,7 @@ CommonUtil::IsDirectoryExist(const std::string &path) { } Status -CommonUtil::CreateDirectory(const std::string &path) { +CommonUtil::CreateDirectory(const std::string& path) { if (path.empty()) { return Status::OK(); } @@ -85,7 +84,7 @@ CommonUtil::CreateDirectory(const std::string &path) { struct stat directory_stat; int status = stat(path.c_str(), &directory_stat); if (status == 0) { - return Status::OK();//already exist + return Status::OK(); // already exist } fs::path fs_path(path); @@ -97,7 +96,7 @@ CommonUtil::CreateDirectory(const std::string &path) { status = stat(path.c_str(), &directory_stat); if (status == 0) { - return Status::OK();//already exist + return Status::OK(); // already exist } int makeOK = mkdir(path.c_str(), S_IRWXU | S_IRGRP | S_IROTH); @@ -110,17 +109,16 @@ CommonUtil::CreateDirectory(const std::string &path) { namespace { void -RemoveDirectory(const std::string &path) { - DIR *dir = nullptr; - struct dirent *dmsg; +RemoveDirectory(const std::string& path) { + DIR* dir = nullptr; + struct dirent* dmsg; const int32_t buf_size = 256; char file_name[buf_size]; std::string folder_name = path + "/%s"; if ((dir = opendir(path.c_str())) != nullptr) { while ((dmsg = readdir(dir)) != nullptr) { - if (strcmp(dmsg->d_name, ".") != 0 - && strcmp(dmsg->d_name, "..") != 0) { + if (strcmp(dmsg->d_name, ".") != 0 && strcmp(dmsg->d_name, "..") != 0) { snprintf(file_name, buf_size, folder_name.c_str(), dmsg->d_name); std::string tmp = file_name; if (tmp.find(".") == std::string::npos) { @@ -136,10 +134,10 @@ RemoveDirectory(const std::string &path) { } remove(path.c_str()); } -} // namespace +} // namespace Status -CommonUtil::DeleteDirectory(const std::string &path) { +CommonUtil::DeleteDirectory(const std::string& path) { if (path.empty()) { return Status::OK(); } @@ -155,17 +153,17 @@ CommonUtil::DeleteDirectory(const std::string &path) { } bool -CommonUtil::IsFileExist(const std::string &path) { +CommonUtil::IsFileExist(const std::string& path) { return (access(path.c_str(), F_OK) == 0); } uint64_t -CommonUtil::GetFileSize(const std::string &path) { +CommonUtil::GetFileSize(const std::string& path) { struct stat file_info; if (stat(path.c_str(), &file_info) < 0) { return 0; } else { - return (uint64_t) file_info.st_size; + return (uint64_t)file_info.st_size; } } @@ -195,21 +193,13 @@ CommonUtil::GetExePath() { } bool -CommonUtil::TimeStrToTime(const std::string &time_str, - time_t &time_integer, - tm &time_struct, - const std::string &format) { +CommonUtil::TimeStrToTime(const std::string& time_str, time_t& time_integer, tm& time_struct, + const std::string& format) { time_integer = 0; memset(&time_struct, 0, sizeof(tm)); - int ret = sscanf(time_str.c_str(), - format.c_str(), - &(time_struct.tm_year), - &(time_struct.tm_mon), - &(time_struct.tm_mday), - &(time_struct.tm_hour), - &(time_struct.tm_min), - &(time_struct.tm_sec)); + int ret = sscanf(time_str.c_str(), format.c_str(), &(time_struct.tm_year), &(time_struct.tm_mon), + &(time_struct.tm_mday), &(time_struct.tm_hour), &(time_struct.tm_min), &(time_struct.tm_sec)); if (ret <= 0) { return false; } @@ -222,15 +212,15 @@ CommonUtil::TimeStrToTime(const std::string &time_str, } void -CommonUtil::ConvertTime(time_t time_integer, tm &time_struct) { +CommonUtil::ConvertTime(time_t time_integer, tm& time_struct) { localtime_r(&time_integer, &time_struct); } void -CommonUtil::ConvertTime(tm time_struct, time_t &time_integer) { +CommonUtil::ConvertTime(tm time_struct, time_t& time_integer) { time_integer = mktime(&time_struct); } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/CommonUtil.h b/cpp/src/utils/CommonUtil.h old mode 100755 new mode 100644 index b059067d50..58ac41ecb1 --- a/cpp/src/utils/CommonUtil.h +++ b/cpp/src/utils/CommonUtil.h @@ -19,8 +19,8 @@ #include "utils/Status.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -28,28 +28,37 @@ namespace server { class CommonUtil { public: - static bool GetSystemMemInfo(uint64_t &total_mem, uint64_t &free_mem); - static bool GetSystemAvailableThreads(uint32_t &thread_count); + static bool + GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem); + static bool + GetSystemAvailableThreads(uint32_t& thread_count); - static bool IsFileExist(const std::string &path); - static uint64_t GetFileSize(const std::string &path); - static bool IsDirectoryExist(const std::string &path); - static Status CreateDirectory(const std::string &path); - static Status DeleteDirectory(const std::string &path); + static bool + IsFileExist(const std::string& path); + static uint64_t + GetFileSize(const std::string& path); + static bool + IsDirectoryExist(const std::string& path); + static Status + CreateDirectory(const std::string& path); + static Status + DeleteDirectory(const std::string& path); - static std::string GetFileName(std::string filename); - static std::string GetExePath(); + static std::string + GetFileName(std::string filename); + static std::string + GetExePath(); - static bool TimeStrToTime(const std::string &time_str, - time_t &time_integer, - tm &time_struct, - const std::string &format = "%d-%d-%d %d:%d:%d"); + static bool + TimeStrToTime(const std::string& time_str, time_t& time_integer, tm& time_struct, + const std::string& format = "%d-%d-%d %d:%d:%d"); - static void ConvertTime(time_t time_integer, tm &time_struct); - static void ConvertTime(tm time_struct, time_t &time_integer); + static void + ConvertTime(time_t time_integer, tm& time_struct); + static void + ConvertTime(tm time_struct, time_t& time_integer); }; -} // namespace server -} // namespace milvus -} // namespace zilliz - +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index d0bcc5cf8d..59e9015afb 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -50,7 +50,7 @@ ToKnowhereErrorCode(const ErrorCode error_code) { return KNOWHERE_ERROR_CODE_BASE + error_code; } -//server error code +// server error code constexpr ErrorCode SERVER_UNEXPECTED_ERROR = ToServerErrorCode(1); constexpr ErrorCode SERVER_UNSUPPORTED_ERROR = ToServerErrorCode(2); constexpr ErrorCode SERVER_NULL_POINTER = ToServerErrorCode(3); @@ -83,7 +83,7 @@ constexpr ErrorCode SERVER_INVALID_INDEX_METRIC_TYPE = ToServerErrorCode(115); constexpr ErrorCode SERVER_INVALID_INDEX_FILE_SIZE = ToServerErrorCode(116); constexpr ErrorCode SERVER_OUT_OF_MEMORY = ToServerErrorCode(117); -//db error code +// db error code constexpr ErrorCode DB_META_TRANSACTION_FAILED = ToDbErrorCode(1); constexpr ErrorCode DB_ERROR = ToDbErrorCode(2); constexpr ErrorCode DB_NOT_FOUND = ToDbErrorCode(3); @@ -92,7 +92,7 @@ constexpr ErrorCode DB_INVALID_PATH = ToDbErrorCode(5); constexpr ErrorCode DB_INCOMPATIB_META = ToDbErrorCode(6); constexpr ErrorCode DB_INVALID_META_URI = ToDbErrorCode(7); -//knowhere error code +// knowhere error code constexpr ErrorCode KNOWHERE_ERROR = ToKnowhereErrorCode(1); constexpr ErrorCode KNOWHERE_INVALID_ARGUMENT = ToKnowhereErrorCode(2); constexpr ErrorCode KNOWHERE_UNEXPECTED_ERROR = ToKnowhereErrorCode(3); @@ -101,17 +101,18 @@ constexpr ErrorCode KNOWHERE_NO_SPACE = ToKnowhereErrorCode(4); namespace server { class ServerException : public std::exception { public: - ServerException(ErrorCode error_code, - const std::string &message = std::string()) + explicit ServerException(ErrorCode error_code, const std::string& message = std::string()) : error_code_(error_code), message_(message) { } public: - ErrorCode error_code() const { + ErrorCode + error_code() const { return error_code_; } - virtual const char *what() const noexcept { + virtual const char* + what() const noexcept { return message_.c_str(); } @@ -119,7 +120,7 @@ class ServerException : public std::exception { ErrorCode error_code_; std::string message_; }; -} // namespace server +} // namespace server } // namespace milvus } // namespace zilliz diff --git a/cpp/src/utils/Exception.h b/cpp/src/utils/Exception.h index 7e30c372bc..2ca61c4884 100644 --- a/cpp/src/utils/Exception.h +++ b/cpp/src/utils/Exception.h @@ -27,16 +27,16 @@ namespace milvus { class Exception : public std::exception { public: - Exception(ErrorCode code, const std::string &message) - : code_(code), - message_(message) { + Exception(ErrorCode code, const std::string& message) : code_(code), message_(message) { } - ErrorCode code() const throw() { + ErrorCode + code() const throw() { return code_; } - virtual const char *what() const throw() { + virtual const char* + what() const throw() { if (message_.empty()) { return "Default Exception."; } else { @@ -54,14 +54,12 @@ class Exception : public std::exception { class InvalidArgumentException : public Exception { public: - InvalidArgumentException() - : Exception(SERVER_INVALID_ARGUMENT, "Invalid Argument") { + InvalidArgumentException() : Exception(SERVER_INVALID_ARGUMENT, "Invalid Argument") { } - explicit InvalidArgumentException(const std::string &message) - : Exception(SERVER_INVALID_ARGUMENT, message) { + explicit InvalidArgumentException(const std::string& message) : Exception(SERVER_INVALID_ARGUMENT, message) { } }; -} // namespace milvus -} // namespace zilliz +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/Log.h b/cpp/src/utils/Log.h index b1402d9e3e..64a8622b73 100644 --- a/cpp/src/utils/Log.h +++ b/cpp/src/utils/Log.h @@ -52,5 +52,5 @@ namespace milvus { #define WRAPPER_LOG_ERROR LOG(ERROR) << WRAPPER_DOMAIN_NAME #define WRAPPER_LOG_FATAL LOG(FATAL) << WRAPPER_DOMAIN_NAME -} // namespace milvus -} // namespace zilliz +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp index 0e710a6e3a..32e942bca3 100644 --- a/cpp/src/utils/LogUtil.cpp +++ b/cpp/src/utils/LogUtil.cpp @@ -18,8 +18,8 @@ #include "utils/LogUtil.h" #include -#include #include +#include namespace zilliz { namespace milvus { @@ -36,16 +36,16 @@ static int fatal_idx = 0; // TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename void -RolloutHandler(const char *filename, std::size_t size, el::Level level) { - char *dirc = strdup(filename); - char *basec = strdup(filename); - char *dir = dirname(dirc); - char *base = basename(basec); +RolloutHandler(const char* filename, std::size_t size, el::Level level) { + char* dirc = strdup(filename); + char* basec = strdup(filename); + char* dir = dirname(dirc); + char* base = basename(basec); std::string s(base); std::stringstream ss; - std::string - list[] = {"\\", " ", "\'", "\"", "*", "\?", "{", "}", ";", "<", ">", "|", "^", "&", "$", "#", "!", "`", "~"}; + std::string list[] = {"\\", " ", "\'", "\"", "*", "\?", "{", "}", ";", "<", + ">", "|", "^", "&", "$", "#", "!", "`", "~"}; std::string::size_type position; for (auto substr : list) { position = 0; @@ -82,7 +82,7 @@ RolloutHandler(const char *filename, std::size_t size, el::Level level) { } Status -InitLog(const std::string &log_config_file) { +InitLog(const std::string& log_config_file) { el::Configurations conf(log_config_file); el::Loggers::reconfigureAllLoggers(conf); @@ -93,6 +93,6 @@ InitLog(const std::string &log_config_file) { return Status::OK(); } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/LogUtil.h b/cpp/src/utils/LogUtil.h index e3f7bed51f..eb396f266e 100644 --- a/cpp/src/utils/LogUtil.h +++ b/cpp/src/utils/LogUtil.h @@ -20,18 +20,18 @@ #include "utils/Status.h" #include "utils/easylogging++.h" -#include #include +#include namespace zilliz { namespace milvus { namespace server { Status -InitLog(const std::string &log_config_file); +InitLog(const std::string& log_config_file); void -RolloutHandler(const char *filename, std::size_t size, el::Level level); +RolloutHandler(const char* filename, std::size_t size, el::Level level); #define SHOW_LOCATION #ifdef SHOW_LOCATION @@ -40,6 +40,6 @@ RolloutHandler(const char *filename, std::size_t size, el::Level level); #define LOCATION_INFO "" #endif -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/SignalUtil.cpp b/cpp/src/utils/SignalUtil.cpp index 5f74852995..a01e7573c7 100644 --- a/cpp/src/utils/SignalUtil.cpp +++ b/cpp/src/utils/SignalUtil.cpp @@ -19,9 +19,9 @@ #include "src/server/Server.h" #include "utils/Log.h" -#include -#include #include +#include +#include namespace zilliz { namespace milvus { @@ -34,7 +34,7 @@ SignalUtil::HandleSignal(int signum) { case SIGUSR2: { SERVER_LOG_INFO << "Server received signal: " << signum; - server::Server &server = server::Server::GetInstance(); + server::Server& server = server::Server::GetInstance(); server.Stop(); exit(0); @@ -43,7 +43,7 @@ SignalUtil::HandleSignal(int signum) { SERVER_LOG_INFO << "Server received critical signal: " << signum; SignalUtil::PrintStacktrace(); - server::Server &server = server::Server::GetInstance(); + server::Server& server = server::Server::GetInstance(); server.Stop(); exit(1); @@ -56,9 +56,9 @@ SignalUtil::PrintStacktrace() { SERVER_LOG_INFO << "Call stack:"; const int size = 32; - void *array[size]; + void* array[size]; int stack_num = backtrace(array, size); - char **stacktrace = backtrace_symbols(array, stack_num); + char** stacktrace = backtrace_symbols(array, stack_num); for (int i = 0; i < stack_num; ++i) { std::string info = stacktrace[i]; SERVER_LOG_INFO << info; @@ -66,6 +66,6 @@ SignalUtil::PrintStacktrace() { free(stacktrace); } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/SignalUtil.h b/cpp/src/utils/SignalUtil.h index c9cd41839b..414b320321 100644 --- a/cpp/src/utils/SignalUtil.h +++ b/cpp/src/utils/SignalUtil.h @@ -23,10 +23,12 @@ namespace server { class SignalUtil { public: - static void HandleSignal(int signum); - static void PrintStacktrace(); + static void + HandleSignal(int signum); + static void + PrintStacktrace(); }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/Status.cpp b/cpp/src/utils/Status.cpp index 5b512b3369..3f807a8a51 100644 --- a/cpp/src/utils/Status.cpp +++ b/cpp/src/utils/Status.cpp @@ -24,12 +24,12 @@ namespace milvus { constexpr int CODE_WIDTH = sizeof(StatusCode); -Status::Status(StatusCode code, const std::string &msg) { - //4 bytes store code - //4 bytes store message length - //the left bytes store message string - const uint32_t length = (uint32_t) msg.size(); - char *result = new char[length + sizeof(length) + CODE_WIDTH]; +Status::Status(StatusCode code, const std::string& msg) { + // 4 bytes store code + // 4 bytes store message length + // the left bytes store message string + const uint32_t length = (uint32_t)msg.size(); + char* result = new char[length + sizeof(length) + CODE_WIDTH]; std::memcpy(result, &code, CODE_WIDTH); std::memcpy(result + CODE_WIDTH, &length, sizeof(length)); memcpy(result + sizeof(length) + CODE_WIDTH, msg.data(), length); @@ -37,38 +37,35 @@ Status::Status(StatusCode code, const std::string &msg) { state_ = result; } -Status::Status() - : state_(nullptr) { +Status::Status() : state_(nullptr) { } Status::~Status() { delete state_; } -Status::Status(const Status &s) - : state_(nullptr) { +Status::Status(const Status& s) : state_(nullptr) { CopyFrom(s); } -Status & -Status::operator=(const Status &s) { +Status& +Status::operator=(const Status& s) { CopyFrom(s); return *this; } -Status::Status(Status &&s) - : state_(nullptr) { +Status::Status(Status&& s) : state_(nullptr) { MoveFrom(s); } -Status & -Status::operator=(Status &&s) { +Status& +Status::operator=(Status&& s) { MoveFrom(s); return *this; } void -Status::CopyFrom(const Status &s) { +Status::CopyFrom(const Status& s) { delete state_; state_ = nullptr; if (s.state_ == nullptr) { @@ -79,11 +76,11 @@ Status::CopyFrom(const Status &s) { memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length)); int buff_len = length + sizeof(length) + CODE_WIDTH; state_ = new char[buff_len]; - memcpy((void *) state_, (void *) s.state_, buff_len); + memcpy((void*)state_, (void*)s.state_, buff_len); } void -Status::MoveFrom(Status &s) { +Status::MoveFrom(Status& s) { delete state_; state_ = s.state_; s.state_ = nullptr; @@ -113,19 +110,26 @@ Status::ToString() const { std::string result; switch (code()) { - case DB_SUCCESS:result = "OK "; + case DB_SUCCESS: + result = "OK "; break; - case DB_ERROR:result = "Error: "; + case DB_ERROR: + result = "Error: "; break; - case DB_META_TRANSACTION_FAILED:result = "Database error: "; + case DB_META_TRANSACTION_FAILED: + result = "Database error: "; break; - case DB_NOT_FOUND:result = "Not found: "; + case DB_NOT_FOUND: + result = "Not found: "; break; - case DB_ALREADY_EXIST:result = "Already exist: "; + case DB_ALREADY_EXIST: + result = "Already exist: "; break; - case DB_INVALID_PATH:result = "Invalid path: "; + case DB_INVALID_PATH: + result = "Invalid path: "; break; - default:result = "Error code(" + std::to_string(code()) + "): "; + default: + result = "Error code(" + std::to_string(code()) + "): "; break; } @@ -133,5 +137,5 @@ Status::ToString() const { return result; } -} // namespace milvus -} // namespace zilliz +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/Status.h b/cpp/src/utils/Status.h index 8f8f238979..4450a4c0e7 100644 --- a/cpp/src/utils/Status.h +++ b/cpp/src/utils/Status.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "utils/Error.h" @@ -29,19 +28,19 @@ using StatusCode = ErrorCode; class Status { public: - Status(StatusCode code, const std::string &msg); + Status(StatusCode code, const std::string& msg); Status(); ~Status(); - Status(const Status &s); + Status(const Status& s); - Status & - operator=(const Status &s); + Status& + operator=(const Status& s); - Status(Status &&s); + Status(Status&& s); - Status & - operator=(Status &&s); + Status& + operator=(Status&& s); static Status OK() { @@ -55,7 +54,7 @@ class Status { StatusCode code() const { - return (state_ == nullptr) ? 0 : *(StatusCode *) (state_); + return (state_ == nullptr) ? 0 : *(StatusCode*)(state_); } std::string @@ -66,14 +65,14 @@ class Status { private: inline void - CopyFrom(const Status &s); + CopyFrom(const Status& s); inline void - MoveFrom(Status &s); + MoveFrom(Status& s); private: - const char *state_ = nullptr; -}; // Status + const char* state_ = nullptr; +}; // Status -} // namespace milvus -} // namespace zilliz +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/StringHelpFunctions.cpp b/cpp/src/utils/StringHelpFunctions.cpp index 8c9e888d3a..444467845f 100644 --- a/cpp/src/utils/StringHelpFunctions.cpp +++ b/cpp/src/utils/StringHelpFunctions.cpp @@ -24,7 +24,7 @@ namespace milvus { namespace server { void -StringHelpFunctions::TrimStringBlank(std::string &string) { +StringHelpFunctions::TrimStringBlank(std::string& string) { if (!string.empty()) { static std::string s_format(" \n\r\t"); string.erase(0, string.find_first_not_of(s_format)); @@ -33,7 +33,7 @@ StringHelpFunctions::TrimStringBlank(std::string &string) { } void -StringHelpFunctions::TrimStringQuote(std::string &string, const std::string &qoute) { +StringHelpFunctions::TrimStringQuote(std::string& string, const std::string& qoute) { if (!string.empty()) { string.erase(0, string.find_first_not_of(qoute)); string.erase(string.find_last_not_of(qoute) + 1); @@ -41,9 +41,8 @@ StringHelpFunctions::TrimStringQuote(std::string &string, const std::string &qou } Status -StringHelpFunctions::SplitStringByDelimeter(const std::string &str, - const std::string &delimeter, - std::vector &result) { +StringHelpFunctions::SplitStringByDelimeter(const std::string& str, const std::string& delimeter, + std::vector& result) { if (str.empty()) { return Status::OK(); } @@ -64,10 +63,8 @@ StringHelpFunctions::SplitStringByDelimeter(const std::string &str, } Status -StringHelpFunctions::SplitStringByQuote(const std::string &str, - const std::string &delimeter, - const std::string "e, - std::vector &result) { +StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::string& delimeter, const std::string& quote, + std::vector& result) { if (quote.empty()) { return SplitStringByDelimeter(str, delimeter, result); } @@ -126,6 +123,6 @@ StringHelpFunctions::SplitStringByQuote(const std::string &str, return Status::OK(); } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/StringHelpFunctions.h b/cpp/src/utils/StringHelpFunctions.h index ebffa075ad..06615fc740 100644 --- a/cpp/src/utils/StringHelpFunctions.h +++ b/cpp/src/utils/StringHelpFunctions.h @@ -19,8 +19,8 @@ #include "utils/Status.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -31,34 +31,34 @@ class StringHelpFunctions { StringHelpFunctions() = default; public: - static void TrimStringBlank(std::string &string); + static void + TrimStringBlank(std::string& string); - static void TrimStringQuote(std::string &string, const std::string &qoute); + static void + TrimStringQuote(std::string& string, const std::string& qoute); - //split string by delimeter ',' + // split string by delimeter ',' // a,b,c a | b | c // a,b, a | b | // ,b,c | b | c // ,b, | b | // ,, | | // a a - static Status SplitStringByDelimeter(const std::string &str, - const std::string &delimeter, - std::vector &result); + static Status + SplitStringByDelimeter(const std::string& str, const std::string& delimeter, std::vector& result); - //assume the table has two columns, quote='\"', delimeter=',' + // assume the table has two columns, quote='\"', delimeter=',' // a,b a | b // "aa,gg,yy",b aa,gg,yy | b // aa"dd,rr"kk,pp aadd,rrkk | pp // "aa,bb" aa,bb // 55,1122\"aa,bb\",yyy,\"kkk\" 55 | 1122aa,bb | yyy | kkk // "55,1122"aa,bb",yyy,"kkk" illegal - static Status SplitStringByQuote(const std::string &str, - const std::string &delimeter, - const std::string "e, - std::vector &result); + static Status + SplitStringByQuote(const std::string& str, const std::string& delimeter, const std::string& quote, + std::vector& result); }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/ThreadPool.h b/cpp/src/utils/ThreadPool.h index 0524f6206d..24c4d20ad5 100644 --- a/cpp/src/utils/ThreadPool.h +++ b/cpp/src/utils/ThreadPool.h @@ -17,18 +17,18 @@ #pragma once -#include -#include -#include -#include -#include #include -#include #include +#include +#include +#include +#include #include +#include #include +#include -#define MAX_THREADS_NUM 32 +#define MAX_THREADS_NUM 32 namespace zilliz { namespace milvus { @@ -37,9 +37,9 @@ class ThreadPool { public: explicit ThreadPool(size_t threads, size_t queue_size = 1000); - template - auto enqueue(F &&f, Args &&... args) - -> std::future::type>; + template + auto + enqueue(F&& f, Args&&... args) -> std::future::type>; ~ThreadPool(); @@ -61,37 +61,30 @@ class ThreadPool { }; // the constructor just launches some amount of workers -inline ThreadPool::ThreadPool(size_t threads, size_t queue_size) - : max_queue_size_(queue_size), stop(false) { +inline ThreadPool::ThreadPool(size_t threads, size_t queue_size) : max_queue_size_(queue_size), stop(false) { for (size_t i = 0; i < threads; ++i) - workers_.emplace_back( - [this] { - for (;;) { - std::function task; + workers_.emplace_back([this] { + for (;;) { + std::function task; - { - std::unique_lock lock(this->queue_mutex_); - this->condition_.wait(lock, - [this] { - return this->stop || !this->tasks_.empty(); - }); - if (this->stop && this->tasks_.empty()) - return; - task = std::move(this->tasks_.front()); - this->tasks_.pop(); - } - this->condition_.notify_all(); - - task(); + { + std::unique_lock lock(this->queue_mutex_); + this->condition_.wait(lock, [this] { return this->stop || !this->tasks_.empty(); }); + if (this->stop && this->tasks_.empty()) return; + task = std::move(this->tasks_.front()); + this->tasks_.pop(); } - }); + this->condition_.notify_all(); + + task(); + } + }); } // add new work item to the pool -template +template auto -ThreadPool::enqueue(F &&f, Args &&... args) --> std::future::type> { +ThreadPool::enqueue(F&& f, Args&&... args) -> std::future::type> { using return_type = typename std::result_of::type; auto task = std::make_shared >( @@ -100,17 +93,11 @@ ThreadPool::enqueue(F &&f, Args &&... args) std::future res = task->get_future(); { std::unique_lock lock(queue_mutex_); - this->condition_.wait(lock, - [this] { - return this->tasks_.size() < max_queue_size_; - }); + this->condition_.wait(lock, [this] { return this->tasks_.size() < max_queue_size_; }); // don't allow enqueueing after stopping the pool - if (stop) - throw std::runtime_error("enqueue on stopped ThreadPool"); + if (stop) throw std::runtime_error("enqueue on stopped ThreadPool"); - tasks_.emplace([task]() { - (*task)(); - }); + tasks_.emplace([task]() { (*task)(); }); } condition_.notify_all(); return res; @@ -123,11 +110,10 @@ inline ThreadPool::~ThreadPool() { stop = true; } condition_.notify_all(); - for (std::thread &worker : workers_) { + for (std::thread& worker : workers_) { worker.join(); } } -} // namespace milvus -} // namespace zilliz - +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/TimeRecorder.cpp b/cpp/src/utils/TimeRecorder.cpp index 5246b35f13..64a0c70fc9 100644 --- a/cpp/src/utils/TimeRecorder.cpp +++ b/cpp/src/utils/TimeRecorder.cpp @@ -21,10 +21,7 @@ namespace zilliz { namespace milvus { -TimeRecorder::TimeRecorder(const std::string &header, - int64_t log_level) : - header_(header), - log_level_(log_level) { +TimeRecorder::TimeRecorder(const std::string& header, int64_t log_level) : header_(header), log_level_(log_level) { start_ = last_ = stdclock::now(); } @@ -40,7 +37,7 @@ TimeRecorder::GetTimeSpanStr(double span) { } void -TimeRecorder::PrintTimeRecord(const std::string &msg, double span) { +TimeRecorder::PrintTimeRecord(const std::string& msg, double span) { std::string str_log; if (!header_.empty()) str_log += header_ + ": "; str_log += msg; @@ -81,7 +78,7 @@ TimeRecorder::PrintTimeRecord(const std::string &msg, double span) { } double -TimeRecorder::RecordSection(const std::string &msg) { +TimeRecorder::RecordSection(const std::string& msg) { stdclock::time_point curr = stdclock::now(); double span = (std::chrono::duration(curr - last_)).count(); last_ = curr; @@ -91,7 +88,7 @@ TimeRecorder::RecordSection(const std::string &msg) { } double -TimeRecorder::ElapseFromBegin(const std::string &msg) { +TimeRecorder::ElapseFromBegin(const std::string& msg) { stdclock::time_point curr = stdclock::now(); double span = (std::chrono::duration(curr - start_)).count(); @@ -99,5 +96,5 @@ TimeRecorder::ElapseFromBegin(const std::string &msg) { return span; } -} // namespace milvus -} // namespace zilliz +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/TimeRecorder.h b/cpp/src/utils/TimeRecorder.h index 2bb937e71f..8f795f01c3 100644 --- a/cpp/src/utils/TimeRecorder.h +++ b/cpp/src/utils/TimeRecorder.h @@ -17,8 +17,8 @@ #pragma once -#include #include +#include namespace zilliz { namespace milvus { @@ -27,19 +27,22 @@ class TimeRecorder { using stdclock = std::chrono::high_resolution_clock; public: - TimeRecorder(const std::string &header, - int64_t log_level = 1); + explicit TimeRecorder(const std::string& header, int64_t log_level = 1); - ~TimeRecorder();//trace = 0, debug = 1, info = 2, warn = 3, error = 4, critical = 5 + ~TimeRecorder(); // trace = 0, debug = 1, info = 2, warn = 3, error = 4, critical = 5 - double RecordSection(const std::string &msg); + double + RecordSection(const std::string& msg); - double ElapseFromBegin(const std::string &msg); + double + ElapseFromBegin(const std::string& msg); - static std::string GetTimeSpanStr(double span); + static std::string + GetTimeSpanStr(double span); private: - void PrintTimeRecord(const std::string &msg, double span); + void + PrintTimeRecord(const std::string& msg, double span); private: std::string header_; @@ -48,5 +51,5 @@ class TimeRecorder { int64_t log_level_; }; -} // namespace milvus -} // namespace zilliz +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index a6d83af1dc..7047e65d30 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -15,16 +15,15 @@ // specific language governing permissions and limitations // under the License. - #include "utils/ValidationUtil.h" -#include "db/engine/ExecutionEngine.h" #include "Log.h" +#include "db/engine/ExecutionEngine.h" -#include -#include #include -#include +#include #include +#include +#include namespace zilliz { namespace milvus { @@ -32,10 +31,10 @@ namespace server { constexpr size_t TABLE_NAME_SIZE_LIMIT = 255; constexpr int64_t TABLE_DIMENSION_LIMIT = 16384; -constexpr int32_t INDEX_FILE_SIZE_LIMIT = 4096; //index trigger size max = 4096 MB +constexpr int32_t INDEX_FILE_SIZE_LIMIT = 4096; // index trigger size max = 4096 MB Status -ValidationUtil::ValidateTableName(const std::string &table_name) { +ValidationUtil::ValidateTableName(const std::string& table_name) { // Table name shouldn't be empty. if (table_name.empty()) { std::string msg = "Empty table name"; @@ -84,8 +83,8 @@ ValidationUtil::ValidateTableDimension(int64_t dimension) { Status ValidationUtil::ValidateTableIndexType(int32_t index_type) { - int engine_type = (int) engine::EngineType(index_type); - if (engine_type <= 0 || engine_type > (int) engine::EngineType::MAX_VALUE) { + int engine_type = (int)engine::EngineType(index_type); + if (engine_type <= 0 || engine_type > (int)engine::EngineType::MAX_VALUE) { std::string msg = "Invalid index type: " + std::to_string(index_type); SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_INDEX_TYPE, msg); @@ -118,7 +117,7 @@ ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) { Status ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { - if (metric_type != (int32_t) engine::MetricType::L2 && metric_type != (int32_t) engine::MetricType::IP) { + if (metric_type != (int32_t)engine::MetricType::L2 && metric_type != (int32_t)engine::MetricType::IP) { std::string msg = "Invalid metric type: " + std::to_string(metric_type); SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_INDEX_METRIC_TYPE, msg); @@ -127,7 +126,7 @@ ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { } Status -ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema &table_schema) { +ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema) { if (top_k <= 0 || top_k > 2048) { std::string msg = "Invalid top k value: " + std::to_string(top_k); SERVER_LOG_ERROR << msg; @@ -138,7 +137,7 @@ ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchem } Status -ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema &table_schema) { +ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema) { if (nprobe <= 0 || nprobe > table_schema.nlist_) { std::string msg = "Invalid nprobe value: " + std::to_string(nprobe); SERVER_LOG_ERROR << msg; @@ -168,7 +167,7 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { } Status -ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t &memory) { +ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) { cudaDeviceProp deviceProp; auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index); if (cuda_err) { @@ -182,13 +181,14 @@ ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t &memory) { } Status -ValidationUtil::ValidateIpAddress(const std::string &ip_address) { +ValidationUtil::ValidateIpAddress(const std::string& ip_address) { struct in_addr address; int result = inet_pton(AF_INET, ip_address.c_str(), &address); switch (result) { - case 1:return Status::OK(); + case 1: + return Status::OK(); case 0: { std::string msg = "Invalid IP address: " + ip_address; SERVER_LOG_ERROR << msg; @@ -203,7 +203,7 @@ ValidationUtil::ValidateIpAddress(const std::string &ip_address) { } Status -ValidationUtil::ValidateStringIsNumber(const std::string &str) { +ValidationUtil::ValidateStringIsNumber(const std::string& str) { if (str.empty() || !std::all_of(str.begin(), str.end(), ::isdigit)) { return Status(SERVER_INVALID_ARGUMENT, "Invalid number"); } @@ -216,11 +216,10 @@ ValidationUtil::ValidateStringIsNumber(const std::string &str) { } Status -ValidationUtil::ValidateStringIsBool(const std::string &str) { +ValidationUtil::ValidateStringIsBool(const std::string& str) { std::string s = str; std::transform(s.begin(), s.end(), s.begin(), ::tolower); - if (s == "true" || s == "on" || s == "yes" || s == "1" || - s == "false" || s == "off" || s == "no" || s == "0" || + if (s == "true" || s == "on" || s == "yes" || s == "1" || s == "false" || s == "off" || s == "no" || s == "0" || s.empty()) { return Status::OK(); } else { @@ -229,7 +228,7 @@ ValidationUtil::ValidateStringIsBool(const std::string &str) { } Status -ValidationUtil::ValidateStringIsFloat(const std::string &str) { +ValidationUtil::ValidateStringIsFloat(const std::string& str) { try { float val = std::stof(str); } catch (...) { @@ -239,19 +238,15 @@ ValidationUtil::ValidateStringIsFloat(const std::string &str) { } Status -ValidationUtil::ValidateDbURI(const std::string &uri) { +ValidationUtil::ValidateDbURI(const std::string& uri) { std::string dialectRegex = "(.*)"; std::string usernameRegex = "(.*)"; std::string passwordRegex = "(.*)"; std::string hostRegex = "(.*)"; std::string portRegex = "(.*)"; std::string dbNameRegex = "(.*)"; - std::string uriRegexStr = dialectRegex + "\\:\\/\\/" + - usernameRegex + "\\:" + - passwordRegex + "\\@" + - hostRegex + "\\:" + - portRegex + "\\/" + - dbNameRegex; + std::string uriRegexStr = dialectRegex + "\\:\\/\\/" + usernameRegex + "\\:" + passwordRegex + "\\@" + hostRegex + + "\\:" + portRegex + "\\/" + dbNameRegex; std::regex uriRegex(uriRegexStr); std::smatch pieces_match; @@ -265,17 +260,17 @@ ValidationUtil::ValidateDbURI(const std::string &uri) { okay = false; } -/* - * Could be DNS, skip checking - * - std::string host = pieces_match[4].str(); - if (!host.empty() && host != "localhost") { - if (ValidateIpAddress(host) != SERVER_SUCCESS) { - SERVER_LOG_ERROR << "Invalid host ip address in uri = " << host; - okay = false; - } - } -*/ + /* + * Could be DNS, skip checking + * + std::string host = pieces_match[4].str(); + if (!host.empty() && host != "localhost") { + if (ValidateIpAddress(host) != SERVER_SUCCESS) { + SERVER_LOG_ERROR << "Invalid host ip address in uri = " << host; + okay = false; + } + } + */ std::string port = pieces_match[5].str(); if (!port.empty()) { @@ -293,6 +288,6 @@ ValidationUtil::ValidateDbURI(const std::string &uri) { return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid db backend uri")); } -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/utils/ValidationUtil.h b/cpp/src/utils/ValidationUtil.h index 44d6065a64..415347cec5 100644 --- a/cpp/src/utils/ValidationUtil.h +++ b/cpp/src/utils/ValidationUtil.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "db/meta/MetaTypes.h" @@ -33,7 +32,7 @@ class ValidationUtil { public: static Status - ValidateTableName(const std::string &table_name); + ValidateTableName(const std::string& table_name); static Status ValidateTableDimension(int64_t dimension); @@ -51,33 +50,33 @@ class ValidationUtil { ValidateTableIndexMetricType(int32_t metric_type); static Status - ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema &table_schema); + ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema); static Status - ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema &table_schema); + ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema); static Status ValidateGpuIndex(uint32_t gpu_index); static Status - GetGpuMemory(uint32_t gpu_index, size_t &memory); + GetGpuMemory(uint32_t gpu_index, size_t& memory); static Status - ValidateIpAddress(const std::string &ip_address); + ValidateIpAddress(const std::string& ip_address); static Status - ValidateStringIsNumber(const std::string &str); + ValidateStringIsNumber(const std::string& str); static Status - ValidateStringIsBool(const std::string &str); + ValidateStringIsBool(const std::string& str); static Status - ValidateStringIsFloat(const std::string &str); + ValidateStringIsFloat(const std::string& str); static Status - ValidateDbURI(const std::string &uri); + ValidateDbURI(const std::string& uri); }; -} // namespace server -} // namespace milvus -} // namespace zilliz +} // namespace server +} // namespace milvus +} // namespace zilliz From aabd91a1563f071fb091a2e84108f0428c0251a4 Mon Sep 17 00:00:00 2001 From: starlord Date: Sat, 28 Sep 2019 12:36:14 +0800 Subject: [PATCH 142/217] format code by clang-tidy Former-commit-id: f9e606566555ad381b8e95e1a4e2572fc61678f0 --- .clang-format | 2 + cpp/src/config/YamlConfigMgr.h | 2 +- .../core/knowhere/knowhere/common/Config.h | 6 +- .../core/knowhere/knowhere/common/Dataset.h | 4 +- .../vector_index/helpers/IndexParameter.cpp | 11 +- .../vector_index/helpers/IndexParameter.h | 31 +- .../knowhere/index/vector_index/nsg/NSG.cpp | 32 +- .../index/vector_index/nsg/NSGHelper.cpp | 2 +- cpp/src/core/test/utils.cpp | 8 +- cpp/src/db/DBImpl.cpp | 3 +- cpp/src/db/Options.h | 4 +- cpp/src/db/Utils.cpp | 16 +- cpp/src/db/engine/ExecutionEngineImpl.cpp | 3 +- cpp/src/main.cpp | 12 +- cpp/src/metrics/PrometheusMetrics.cpp | 73 +++- cpp/src/metrics/SystemInfo.cpp | 21 +- cpp/src/scheduler/ResourceMgr.h | 2 +- .../scheduler/action/PushTaskToNeighbour.cpp | 6 +- cpp/src/scheduler/resource/Resource.cpp | 9 +- cpp/src/scheduler/task/DeleteTask.h | 2 +- cpp/src/sdk/grpc/ClientProxy.cpp | 13 +- cpp/src/sdk/include/Status.h | 2 +- cpp/src/sdk/interface/Status.cpp | 4 +- cpp/src/server/Config.cpp | 378 +++++++++++++----- cpp/src/server/DBWrapper.cpp | 45 ++- cpp/src/server/DBWrapper.h | 2 +- cpp/src/server/Server.cpp | 2 +- cpp/src/server/grpc_impl/GrpcRequestHandler.h | 2 +- .../server/grpc_impl/GrpcRequestScheduler.cpp | 3 +- .../server/grpc_impl/GrpcRequestScheduler.h | 2 +- cpp/src/server/grpc_impl/GrpcRequestTask.cpp | 12 +- cpp/src/server/grpc_impl/GrpcRequestTask.h | 12 +- cpp/src/utils/CommonUtil.cpp | 8 +- cpp/src/utils/Status.cpp | 4 +- cpp/src/utils/Status.h | 2 +- cpp/src/utils/ThreadPool.h | 6 +- cpp/src/utils/TimeRecorder.cpp | 3 +- cpp/src/utils/ValidationUtil.cpp | 12 +- 38 files changed, 532 insertions(+), 229 deletions(-) diff --git a/.clang-format b/.clang-format index 1e124f7be8..73dc2aeb8f 100644 --- a/.clang-format +++ b/.clang-format @@ -23,3 +23,5 @@ AccessModifierOffset: -3 AlwaysBreakAfterReturnType: All AllowShortBlocksOnASingleLine: false AllowShortFunctionsOnASingleLine: false +AllowShortIfStatementsOnASingleLine: false +AlignTrailingComments: true diff --git a/cpp/src/config/YamlConfigMgr.h b/cpp/src/config/YamlConfigMgr.h index 9697ce3cae..743276b504 100644 --- a/cpp/src/config/YamlConfigMgr.h +++ b/cpp/src/config/YamlConfigMgr.h @@ -47,7 +47,7 @@ class YamlConfigMgr : public ConfigMgr { SetConfigValue(const YAML::Node& node, const std::string& key, ConfigNode& config); bool - SetChildConfig(const YAML::Node& node, const std::string& name, ConfigNode& config); + SetChildConfig(const YAML::Node& node, const std::string& child_name, ConfigNode& config); bool SetSequence(const YAML::Node& node, const std::string& child_name, ConfigNode& config); diff --git a/cpp/src/core/knowhere/knowhere/common/Config.h b/cpp/src/core/knowhere/knowhere/common/Config.h index 208641d666..cfee35c35b 100644 --- a/cpp/src/core/knowhere/knowhere/common/Config.h +++ b/cpp/src/core/knowhere/knowhere/common/Config.h @@ -46,12 +46,14 @@ struct Cfg { const int64_t &k, const int64_t &gpu_id, METRICTYPE type) - : d(dim), k(k), gpu_id(gpu_id), metric_type(type) {} + : metric_type(type), k(k), gpu_id(gpu_id), d(dim) {} Cfg() = default; virtual bool - CheckValid(){}; + CheckValid(){ + return true; + }; }; using Config = std::shared_ptr; diff --git a/cpp/src/core/knowhere/knowhere/common/Dataset.h b/cpp/src/core/knowhere/knowhere/common/Dataset.h index c2d74a59e5..91fb165811 100644 --- a/cpp/src/core/knowhere/knowhere/common/Dataset.h +++ b/cpp/src/core/knowhere/knowhere/common/Dataset.h @@ -127,10 +127,10 @@ class Dataset { //} private: - SchemaPtr array_schema_; - SchemaPtr tensor_schema_; std::vector array_; + SchemaPtr array_schema_; std::vector tensor_; + SchemaPtr tensor_schema_; //Config meta_; }; diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp index 503f0c2f67..9999444368 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp @@ -25,9 +25,14 @@ namespace zilliz { namespace knowhere { faiss::MetricType GetMetricType(METRICTYPE &type) { - if (type == METRICTYPE::L2) return faiss::METRIC_L2; - if (type == METRICTYPE::IP) return faiss::METRIC_INNER_PRODUCT; - if (type == METRICTYPE::INVALID) KNOWHERE_THROW_MSG("Metric type is invalid"); + if (type == METRICTYPE::L2) { + return faiss::METRIC_L2; + } + if (type == METRICTYPE::IP) { + return faiss::METRIC_INNER_PRODUCT; + } + + KNOWHERE_THROW_MSG("Metric type is invalid"); } diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h index 521754e2e3..cd873ed251 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h @@ -52,12 +52,15 @@ struct IVFCfg : public Cfg { const int64_t &nlist, const int64_t &nprobe, METRICTYPE type) - : nlist(nlist), nprobe(nprobe), Cfg(dim, k, gpu_id, type) {} + : Cfg(dim, k, gpu_id, type), nlist(nlist), nprobe(nprobe) { + } IVFCfg() = default; bool - CheckValid() override {}; + CheckValid() override { + return true; + }; }; using IVFConfig = std::shared_ptr; @@ -72,12 +75,15 @@ struct IVFSQCfg : public IVFCfg { const int64_t &nprobe, const int64_t &nbits, METRICTYPE type) - : nbits(nbits), IVFCfg(dim, k, gpu_id, nlist, nprobe, type) {} + : IVFCfg(dim, k, gpu_id, nlist, nprobe, type), nbits(nbits) { + } IVFSQCfg() = default; bool - CheckValid() override {}; + CheckValid() override { + return true; + }; }; using IVFSQConfig = std::shared_ptr; @@ -98,12 +104,15 @@ struct IVFPQCfg : public IVFCfg { const int64_t &nbits, const int64_t &m, METRICTYPE type) - : nbits(nbits), m(m), IVFCfg(dim, k, gpu_id, nlist, nprobe, type) {} + : IVFCfg(dim, k, gpu_id, nlist, nprobe, type), m(m), nbits(nbits) { + } IVFPQCfg() = default; bool - CheckValid() override {}; + CheckValid() override { + return true; + }; }; using IVFPQConfig = std::shared_ptr; @@ -123,13 +132,17 @@ struct NSGCfg : public IVFCfg { const int64_t &out_degree, const int64_t &candidate_size, METRICTYPE type) - : knng(knng), search_length(search_length), out_degree(out_degree), candidate_pool_size(candidate_size), - IVFCfg(dim, k, gpu_id, nlist, nprobe, type) {} + : IVFCfg(dim, k, gpu_id, nlist, nprobe, type), + knng(knng), search_length(search_length), + out_degree(out_degree), candidate_pool_size(candidate_size) { + } NSGCfg() = default; bool - CheckValid() override {}; + CheckValid() override { + return true; + }; }; using NSGConfig = std::shared_ptr; diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp index 0f3bb639c2..fcf77625db 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp @@ -81,7 +81,7 @@ void NsgIndex::Build_with_ids(size_t nb, const float *data, const long *ids, con //>> Debug code /// int total_degree = 0; - for (int i = 0; i < ntotal; ++i) { + for (size_t i = 0; i < ntotal; ++i) { total_degree += nsg[i].size(); } @@ -172,7 +172,7 @@ void NsgIndex::GetNeighbors(const float *query, for (size_t i = 0; i < init_ids.size(); ++i) { node_t id = init_ids[i]; - if (id >= ntotal) { + if (id >= static_cast(ntotal)) { KNOWHERE_THROW_MSG("Build Index Error, id > ntotal"); continue; } @@ -262,7 +262,7 @@ void NsgIndex::GetNeighbors(const float *query, std::vector &resset, s for (size_t i = 0; i < init_ids.size(); ++i) { node_t id = init_ids[i]; - if (id >= ntotal) { + if (id >= static_cast(ntotal)) { KNOWHERE_THROW_MSG("Build Index Error, id > ntotal"); continue; } @@ -350,7 +350,7 @@ void NsgIndex::GetNeighbors(const float *query, node_t id = init_ids[i]; //assert(id < ntotal); - if (id >= ntotal) { + if (id >= static_cast(ntotal)) { KNOWHERE_THROW_MSG("Build Index Error, id > ntotal"); continue; } @@ -461,7 +461,7 @@ void NsgIndex::Link() { //} ///// - for (int i = 0; i < ntotal; ++i) { + for (size_t i = 0; i < ntotal; ++i) { nsg[i].shrink_to_fit(); } } @@ -483,7 +483,9 @@ void NsgIndex::SyncPrune(size_t n, unsigned cursor = 0; std::sort(pool.begin(), pool.end()); std::vector result; - if (pool[cursor].id == n) cursor++; + if (pool[cursor].id == static_cast(n)) { + cursor++; + } result.push_back(pool[cursor]); // init result with nearest neighbor SelectEdge(cursor, pool, result, true); @@ -518,7 +520,7 @@ void NsgIndex::InterInsert(unsigned n, std::vector &mutex_vec, float int duplicate = false; { LockGuard lk(mutex_vec[current_neighbor]); - for (int j = 0; j < out_degree; ++j) { + for (size_t j = 0; j < out_degree; ++j) { if (nsn_dist_pool[j] == -1) break; // 保证至少有一条边能连回来 @@ -551,14 +553,14 @@ void NsgIndex::InterInsert(unsigned n, std::vector &mutex_vec, float { LockGuard lk(mutex_vec[current_neighbor]); - for (int j = 0; j < result.size(); ++j) { + for (size_t j = 0; j < result.size(); ++j) { nsn_id_pool[j] = result[j].id; nsn_dist_pool[j] = result[j].distance; } } } else { LockGuard lk(mutex_vec[current_neighbor]); - for (int j = 0; j < out_degree; ++j) { + for (size_t j = 0; j < out_degree; ++j) { if (nsn_dist_pool[j] == -1) { nsn_id_pool.push_back(current_as_neighbor.id); nsn_dist_pool[j] = current_as_neighbor.distance; @@ -605,9 +607,11 @@ void NsgIndex::CheckConnectivity() { boost::dynamic_bitset<> has_linked{ntotal, 0}; int64_t linked_count = 0; - while (linked_count < ntotal) { + while (linked_count < static_cast(ntotal)) { DFS(root, has_linked, linked_count); - if (linked_count >= ntotal) break; + if (linked_count >= static_cast(ntotal)) { + break; + } FindUnconnectedNode(has_linked, root); } } @@ -697,7 +701,7 @@ void NsgIndex::Search(const float *query, } else{ //#pragma omp parallel for schedule(dynamic, 50) #pragma omp parallel for - for (int i = 0; i < nq; ++i) { + for (unsigned int i = 0; i < nq; ++i) { // TODO(linxj): when to use openmp auto single_query = query + i * dim; GetNeighbors(single_query, resset[i], nsg, ¶ms); @@ -705,8 +709,8 @@ void NsgIndex::Search(const float *query, } rc.ElapseFromBegin("cost"); - for (int i = 0; i < nq; ++i) { - for (int j = 0; j < k; ++j) { + for (unsigned int i = 0; i < nq; ++i) { + for (unsigned int j = 0; j < k; ++j) { //ids[i * k + j] = resset[i][j].id; // Fix(linxj): bug, reset[i][j] out of range diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.cpp index eb87f97f3e..77cf0e9464 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.cpp @@ -29,7 +29,7 @@ namespace algo { // TODO: impl search && insert && return insert pos. why not just find and swap? int InsertIntoPool(Neighbor *addr, unsigned K, Neighbor nn) { //>> Fix: Add assert - for (int i = 0; i < K; ++i) { + for (unsigned int i = 0; i < K; ++i) { assert(addr[i].id != nn.id); } diff --git a/cpp/src/core/test/utils.cpp b/cpp/src/core/test/utils.cpp index 4e240f5f74..df9410bd6c 100644 --- a/cpp/src/core/test/utils.cpp +++ b/cpp/src/core/test/utils.cpp @@ -38,8 +38,8 @@ void DataGen::Generate(const int &dim, const int &nb, const int &nq) { this->dim = dim; GenAll(dim, nb, xb, ids, nq, xq); - assert(xb.size() == dim * nb); - assert(xq.size() == dim * nq); + assert(xb.size() == (size_t)dim * nb); + assert(xq.size() == (size_t)dim * nq); base_dataset = generate_dataset(nb, dim, xb.data(), ids.data()); query_dataset = generate_query_dataset(nq, dim, xq.data()); @@ -47,7 +47,7 @@ void DataGen::Generate(const int &dim, const int &nb, const int &nq) { } zilliz::knowhere::DatasetPtr DataGen::GenQuery(const int &nq) { xq.resize(nq * dim); - for (size_t i = 0; i < nq * dim; ++i) { + for (int i = 0; i < nq * dim; ++i) { xq[i] = xb[i]; } return generate_query_dataset(nq, dim, xq.data()); @@ -72,7 +72,7 @@ void GenAll(const int64_t &dim, const int64_t &nq, float *xq) { GenBase(dim, nb, xb, ids); - for (size_t i = 0; i < nq * dim; ++i) { + for (int64_t i = 0; i < nq * dim; ++i) { xq[i] = xb[i]; } } diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 2780d0f763..2e6955a81d 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -638,7 +638,8 @@ DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, const m ENGINE_LOG_DEBUG << "Merging file " << file_schema.file_id_; index_size = index->Size(); - if (index_size >= file_schema.index_file_size_) break; + if (index_size >= file_schema.index_file_size_) + break; } // step 3: serialize to disk diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index df3b1fc0c5..d51a30f2b4 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -53,9 +53,9 @@ struct ArchiveConf { private: void - ParseCritirias(const std::string& type); + ParseCritirias(const std::string& criterias); void - ParseType(const std::string& criterias); + ParseType(const std::string& type); std::string type_; CriteriaT criterias_; diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index 84e099b579..2055ff391c 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -143,14 +143,14 @@ GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file if (boost::filesystem::exists(file_path)) { table_file.location_ = file_path; return Status::OK(); - } else { - for (auto& path : options.slave_paths_) { - parent_path = ConstructParentFolder(path, table_file); - file_path = parent_path + "/" + table_file.file_id_; - if (boost::filesystem::exists(file_path)) { - table_file.location_ = file_path; - return Status::OK(); - } + } + + for (auto& path : options.slave_paths_) { + parent_path = ConstructParentFolder(path, table_file); + file_path = parent_path + "/" + table_file.file_id_; + if (boost::filesystem::exists(file_path)) { + table_file.location_ = file_path; + return Status::OK(); } } diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index e8f9ba395d..52b4992dab 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -342,7 +342,8 @@ Status ExecutionEngineImpl::Init() { server::Config& config = server::Config::GetInstance(); Status s = config.GetDBConfigBuildIndexGPU(gpu_num_); - if (!s.ok()) return s; + if (!s.ok()) + return s; return Status::OK(); } diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index bced85af6d..5d808a7e4c 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -39,12 +39,12 @@ main(int argc, char* argv[]) { std::cout << std::endl << "Welcome to use Milvus by Zilliz!" << std::endl; std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << " built at " << BUILD_TIME << std::endl; - static struct option long_options[] = {{"conf_file", required_argument, 0, 'c'}, - {"log_conf_file", required_argument, 0, 'l'}, - {"help", no_argument, 0, 'h'}, - {"daemon", no_argument, 0, 'd'}, - {"pid_file", required_argument, 0, 'p'}, - {NULL, 0, 0, 0}}; + static struct option long_options[] = {{"conf_file", required_argument, nullptr, 'c'}, + {"log_conf_file", required_argument, nullptr, 'l'}, + {"help", no_argument, nullptr, 'h'}, + {"daemon", no_argument, nullptr, 'd'}, + {"pid_file", required_argument, nullptr, 'p'}, + {nullptr, 0, nullptr, 0}}; int option_index = 0; int64_t start_daemonized = 0; diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index dd6410b46a..72b26eb63d 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -33,13 +33,20 @@ PrometheusMetrics::Init() { try { Config& config = Config::GetInstance(); Status s = config.GetMetricConfigEnableMonitor(startup_); - if (!s.ok()) return s.code(); - if (!startup_) return SERVER_SUCCESS; + if (!s.ok()) { + return s.code(); + } + if (!startup_) { + return SERVER_SUCCESS; + } // Following should be read from config file. std::string bind_address; s = config.GetMetricConfigPrometheusPort(bind_address); - if (!s.ok()) return s.code(); + if (!s.ok()) { + return s.code(); + } + const std::string uri = std::string("/tmp/metrics"); const std::size_t num_threads = 2; @@ -58,21 +65,30 @@ PrometheusMetrics::Init() { void PrometheusMetrics::CPUUsagePercentSet() { - if (!startup_) return; + if (!startup_) { + return; + } + double usage_percent = server::SystemInfo::GetInstance().CPUPercent(); CPU_usage_percent_.Set(usage_percent); } void PrometheusMetrics::RAMUsagePercentSet() { - if (!startup_) return; + if (!startup_) { + return; + } + double usage_percent = server::SystemInfo::GetInstance().MemoryPercent(); RAM_usage_percent_.Set(usage_percent); } void PrometheusMetrics::GPUPercentGaugeSet() { - if (!startup_) return; + if (!startup_) { + return; + } + int numDevice = server::SystemInfo::GetInstance().num_device(); std::vector used_total = server::SystemInfo::GetInstance().GPUMemoryTotal(); std::vector used_memory = server::SystemInfo::GetInstance().GPUMemoryUsed(); @@ -86,7 +102,10 @@ PrometheusMetrics::GPUPercentGaugeSet() { void PrometheusMetrics::GPUMemoryUsageGaugeSet() { - if (!startup_) return; + if (!startup_) { + return; + } + std::vector values = server::SystemInfo::GetInstance().GPUMemoryUsed(); constexpr uint64_t MtoB = 1024 * 1024; int numDevice = server::SystemInfo::GetInstance().num_device(); @@ -100,7 +119,9 @@ PrometheusMetrics::GPUMemoryUsageGaugeSet() { void PrometheusMetrics::AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) { // MB/s - if (!startup_) return; + if (!startup_) { + return; + } int64_t MtoB = 1024 * 1024; int64_t size = num_vector * dim * 4; @@ -109,7 +130,10 @@ PrometheusMetrics::AddVectorsPerSecondGaugeSet(int num_vector, int dim, double t void PrometheusMetrics::QueryIndexTypePerSecondSet(std::string type, double value) { - if (!startup_) return; + if (!startup_) { + return; + } + if (type == "IVF") { query_index_IVF_type_per_second_gauge_.Set(value); } else if (type == "IDMap") { @@ -119,19 +143,27 @@ PrometheusMetrics::QueryIndexTypePerSecondSet(std::string type, double value) { void PrometheusMetrics::ConnectionGaugeIncrement() { - if (!startup_) return; + if (!startup_) { + return; + } + connection_gauge_.Increment(); } void PrometheusMetrics::ConnectionGaugeDecrement() { - if (!startup_) return; + if (!startup_) { + return; + } + connection_gauge_.Decrement(); } void PrometheusMetrics::OctetsSet() { - if (!startup_) return; + if (!startup_) { + return; + } // get old stats and reset them uint64_t old_inoctets = SystemInfo::GetInstance().get_inoctets(); @@ -147,14 +179,19 @@ PrometheusMetrics::OctetsSet() { auto now_time = std::chrono::system_clock::now(); auto total_microsecond = METRICS_MICROSECONDS(old_time, now_time); auto total_second = total_microsecond * micro_to_second; - if (total_second == 0) return; + if (total_second == 0) { + return; + } + inoctets_gauge_.Set((in_and_out_octets.first - old_inoctets) / total_second); outoctets_gauge_.Set((in_and_out_octets.second - old_outoctets) / total_second); } void PrometheusMetrics::CPUCoreUsagePercentSet() { - if (!startup_) return; + if (!startup_) { + return; + } std::vector cpu_core_percent = server::SystemInfo::GetInstance().CPUCorePercent(); @@ -166,7 +203,9 @@ PrometheusMetrics::CPUCoreUsagePercentSet() { void PrometheusMetrics::GPUTemperature() { - if (!startup_) return; + if (!startup_) { + return; + } std::vector GPU_temperatures = server::SystemInfo::GetInstance().GPUTemperature(); @@ -178,7 +217,9 @@ PrometheusMetrics::GPUTemperature() { void PrometheusMetrics::CPUTemperature() { - if (!startup_) return; + if (!startup_) { + return; + } std::vector CPU_temperatures = server::SystemInfo::GetInstance().CPUTemperature(); diff --git a/cpp/src/metrics/SystemInfo.cpp b/cpp/src/metrics/SystemInfo.cpp index 68c1182726..4647b1435b 100644 --- a/cpp/src/metrics/SystemInfo.cpp +++ b/cpp/src/metrics/SystemInfo.cpp @@ -31,7 +31,8 @@ namespace server { void SystemInfo::Init() { - if (initialized_) return; + if (initialized_) + return; initialized_ = true; @@ -45,7 +46,8 @@ SystemInfo::Init() { file = fopen("/proc/cpuinfo", "r"); num_processors_ = 0; while (fgets(line, 128, file) != NULL) { - if (strncmp(line, "processor", 9) == 0) num_processors_++; + if (strncmp(line, "processor", 9) == 0) + num_processors_++; if (strncmp(line, "physical", 8) == 0) { num_physical_processors_ = ParseLine(line); } @@ -116,7 +118,8 @@ SystemInfo::GetProcessUsedMemory() { double SystemInfo::MemoryPercent() { - if (!initialized_) Init(); + if (!initialized_) + Init(); return (double)(GetProcessUsedMemory() * 100) / (double)total_ram_; } @@ -171,7 +174,8 @@ SystemInfo::getTotalCpuTime(std::vector& work_time_array) { double SystemInfo::CPUPercent() { - if (!initialized_) Init(); + if (!initialized_) + Init(); struct tms time_sample; clock_t now; double percent; @@ -195,7 +199,8 @@ SystemInfo::CPUPercent() { std::vector SystemInfo::GPUMemoryTotal() { // get GPU usage percent - if (!initialized_) Init(); + if (!initialized_) + Init(); std::vector result; nvmlMemory_t nvmlMemory; for (int i = 0; i < num_device_; ++i) { @@ -209,7 +214,8 @@ SystemInfo::GPUMemoryTotal() { std::vector SystemInfo::GPUTemperature() { - if (!initialized_) Init(); + if (!initialized_) + Init(); std::vector result; for (int i = 0; i < num_device_; i++) { nvmlDevice_t device; @@ -241,7 +247,8 @@ SystemInfo::CPUTemperature() { std::vector SystemInfo::GPUMemoryUsed() { // get GPU memory used - if (!initialized_) Init(); + if (!initialized_) + Init(); std::vector result; nvmlMemory_t nvmlMemory; diff --git a/cpp/src/scheduler/ResourceMgr.h b/cpp/src/scheduler/ResourceMgr.h index b6720a2054..05393041bf 100644 --- a/cpp/src/scheduler/ResourceMgr.h +++ b/cpp/src/scheduler/ResourceMgr.h @@ -48,7 +48,7 @@ class ResourceMgr { Add(ResourcePtr&& resource); bool - Connect(const std::string& res1, const std::string& res2, Connection& connection); + Connect(const std::string& name1, const std::string& name2, Connection& connection); void Clear(); diff --git a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp index c7514b1e2d..4cafffa365 100644 --- a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp @@ -30,7 +30,8 @@ get_neighbours(const ResourcePtr& self) { std::vector neighbours; for (auto& neighbour_node : self->GetNeighbours()) { auto node = neighbour_node.neighbour_node.lock(); - if (not node) continue; + if (not node) + continue; auto resource = std::static_pointer_cast(node); // if (not resource->HasExecutor()) continue; @@ -45,7 +46,8 @@ get_neighbours_with_connetion(const ResourcePtr& self) { std::vector> neighbours; for (auto& neighbour_node : self->GetNeighbours()) { auto node = neighbour_node.neighbour_node.lock(); - if (not node) continue; + if (not node) + continue; auto resource = std::static_pointer_cast(node); // if (not resource->HasExecutor()) continue; diff --git a/cpp/src/scheduler/resource/Resource.cpp b/cpp/src/scheduler/resource/Resource.cpp index 8273d69d33..48208a07a5 100644 --- a/cpp/src/scheduler/resource/Resource.cpp +++ b/cpp/src/scheduler/resource/Resource.cpp @@ -92,7 +92,8 @@ uint64_t Resource::NumOfTaskToExec() { uint64_t count = 0; for (auto& task : task_table_) { - if (task->state == TaskTableItemState::LOADED) ++count; + if (task->state == TaskTableItemState::LOADED) + ++count; } return count; } @@ -102,7 +103,8 @@ Resource::pick_task_load() { auto indexes = task_table_.PickToLoad(10); for (auto index : indexes) { // try to set one task loading, then return - if (task_table_.Load(index)) return task_table_.Get(index); + if (task_table_.Load(index)) + return task_table_.Get(index); // else try next } return nullptr; @@ -113,7 +115,8 @@ Resource::pick_task_execute() { auto indexes = task_table_.PickToExecute(3); for (auto index : indexes) { // try to set one task executing, then return - if (task_table_.Execute(index)) return task_table_.Get(index); + if (task_table_.Execute(index)) + return task_table_.Get(index); // else try next } return nullptr; diff --git a/cpp/src/scheduler/task/DeleteTask.h b/cpp/src/scheduler/task/DeleteTask.h index 65e8724940..473fee2681 100644 --- a/cpp/src/scheduler/task/DeleteTask.h +++ b/cpp/src/scheduler/task/DeleteTask.h @@ -26,7 +26,7 @@ namespace scheduler { class XDeleteTask : public Task { public: - explicit XDeleteTask(const scheduler::DeleteJobPtr& job); + explicit XDeleteTask(const scheduler::DeleteJobPtr& delete_job); void Load(LoadType type, uint8_t device_id) override; diff --git a/cpp/src/sdk/grpc/ClientProxy.cpp b/cpp/src/sdk/grpc/ClientProxy.cpp index 8ebd577e3a..db4f717eaa 100644 --- a/cpp/src/sdk/grpc/ClientProxy.cpp +++ b/cpp/src/sdk/grpc/ClientProxy.cpp @@ -31,9 +31,8 @@ UriCheck(const std::string& uri) { size_t index = uri.find_first_of(':', 0); if (index == std::string::npos) { return false; - } else { - return true; } + return true; } Status @@ -45,11 +44,11 @@ ClientProxy::Connect(const ConnectParam& param) { connected_ = true; client_ptr_ = std::make_shared(channel_); return Status::OK(); - } else { - std::string reason = "connect failed!"; - connected_ = false; - return Status(StatusCode::NotConnected, reason); } + + std::string reason = "connect failed!"; + connected_ = false; + return Status(StatusCode::NotConnected, reason); } Status @@ -134,7 +133,7 @@ ClientProxy::CreateIndex(const IndexParam& index_param) { // TODO: add index params ::milvus::grpc::IndexParam grpc_index_param; grpc_index_param.set_table_name(index_param.table_name); - grpc_index_param.mutable_index()->set_index_type((int32_t)index_param.index_type); + grpc_index_param.mutable_index()->set_index_type(static_cast(index_param.index_type)); grpc_index_param.mutable_index()->set_nlist(index_param.nlist); return client_ptr_->CreateIndex(grpc_index_param); } catch (std::exception& ex) { diff --git a/cpp/src/sdk/include/Status.h b/cpp/src/sdk/include/Status.h index e9b267c5f7..a81116b31d 100644 --- a/cpp/src/sdk/include/Status.h +++ b/cpp/src/sdk/include/Status.h @@ -85,7 +85,7 @@ class Status { MoveFrom(Status& s); private: - const char* state_ = nullptr; + char* state_ = nullptr; }; // Status } // namespace milvus diff --git a/cpp/src/sdk/interface/Status.cpp b/cpp/src/sdk/interface/Status.cpp index cd46c8c245..a5e89556f2 100644 --- a/cpp/src/sdk/interface/Status.cpp +++ b/cpp/src/sdk/interface/Status.cpp @@ -28,7 +28,7 @@ Status::Status(StatusCode code, const std::string& msg) { // 4 bytes store message length // the left bytes store message string const uint32_t length = (uint32_t)msg.size(); - char* result = new char[length + sizeof(length) + CODE_WIDTH]; + auto result = new char[length + sizeof(length) + CODE_WIDTH]; memcpy(result, &code, CODE_WIDTH); memcpy(result + CODE_WIDTH, &length, sizeof(length)); memcpy(result + sizeof(length) + CODE_WIDTH, msg.data(), length); @@ -75,7 +75,7 @@ Status::CopyFrom(const Status& s) { memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length)); int buff_len = length + sizeof(length) + CODE_WIDTH; state_ = new char[buff_len]; - memcpy((void*)state_, (void*)s.state_, buff_len); + memcpy(state_, s.state_, buff_len); } void diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 788f792feb..76ac246c24 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -77,100 +77,146 @@ Config::ValidateConfig() { /* server config */ std::string server_addr; s = GetServerConfigAddress(server_addr); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } std::string server_port; s = GetServerConfigPort(server_port); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } std::string server_mode; s = GetServerConfigDeployMode(server_mode); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } std::string server_time_zone; s = GetServerConfigTimeZone(server_time_zone); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } /* db config */ std::string db_primary_path; s = GetDBConfigPrimaryPath(db_primary_path); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } std::string db_secondary_path; s = GetDBConfigSecondaryPath(db_secondary_path); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } std::string db_backend_url; s = GetDBConfigBackendUrl(db_backend_url); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } int32_t db_archive_disk_threshold; s = GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } int32_t db_archive_days_threshold; s = GetDBConfigArchiveDaysThreshold(db_archive_days_threshold); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } int32_t db_insert_buffer_size; s = GetDBConfigInsertBufferSize(db_insert_buffer_size); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } int32_t db_build_index_gpu; s = GetDBConfigBuildIndexGPU(db_build_index_gpu); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } /* metric config */ bool metric_enable_monitor; s = GetMetricConfigEnableMonitor(metric_enable_monitor); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } std::string metric_collector; s = GetMetricConfigCollector(metric_collector); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } std::string metric_prometheus_port; s = GetMetricConfigPrometheusPort(metric_prometheus_port); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } /* cache config */ int32_t cache_cpu_cache_capacity; s = GetCacheConfigCpuCacheCapacity(cache_cpu_cache_capacity); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } float cache_cpu_cache_threshold; s = GetCacheConfigCpuCacheThreshold(cache_cpu_cache_threshold); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } int32_t cache_gpu_cache_capacity; s = GetCacheConfigGpuCacheCapacity(cache_gpu_cache_capacity); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } float cache_gpu_cache_threshold; s = GetCacheConfigGpuCacheThreshold(cache_gpu_cache_threshold); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } bool cache_insert_data; s = GetCacheConfigCacheInsertData(cache_insert_data); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } /* engine config */ int32_t engine_use_blas_threshold; s = GetEngineConfigUseBlasThreshold(engine_use_blas_threshold); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } int32_t engine_omp_thread_num; s = GetEngineConfigOmpThreadNum(engine_omp_thread_num); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } /* resource config */ std::string resource_mode; s = GetResourceConfigMode(resource_mode); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } std::vector resource_pool; s = GetResourceConfigPool(resource_pool); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } return Status::OK(); } @@ -181,75 +227,119 @@ Config::ResetDefaultConfig() { /* server config */ s = SetServerConfigAddress(CONFIG_SERVER_ADDRESS_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } s = SetServerConfigPort(CONFIG_SERVER_PORT_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } s = SetServerConfigDeployMode(CONFIG_SERVER_DEPLOY_MODE_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } s = SetServerConfigTimeZone(CONFIG_SERVER_TIME_ZONE_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } /* db config */ s = SetDBConfigPrimaryPath(CONFIG_DB_PRIMARY_PATH_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } s = SetDBConfigSecondaryPath(CONFIG_DB_SECONDARY_PATH_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } s = SetDBConfigBackendUrl(CONFIG_DB_BACKEND_URL_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } s = SetDBConfigArchiveDiskThreshold(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } s = SetDBConfigArchiveDaysThreshold(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } s = SetDBConfigInsertBufferSize(CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } s = SetDBConfigBuildIndexGPU(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } /* metric config */ s = SetMetricConfigEnableMonitor(CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } s = SetMetricConfigCollector(CONFIG_METRIC_COLLECTOR_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } s = SetMetricConfigPrometheusPort(CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } /* cache config */ s = SetCacheConfigCpuCacheCapacity(CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } s = SetCacheConfigCpuCacheThreshold(CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } s = SetCacheConfigGpuCacheCapacity(CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } s = SetCacheConfigGpuCacheThreshold(CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } s = SetCacheConfigCacheInsertData(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } /* engine config */ s = SetEngineConfigUseBlasThreshold(CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } s = SetEngineConfigOmpThreadNum(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } /* resource config */ s = SetResourceConfigMode(CONFIG_RESOURCE_MODE_DEFAULT); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } return Status::OK(); } @@ -423,13 +513,16 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (cpu_cache_capacity >= total_mem) { return Status(SERVER_INVALID_ARGUMENT, "Cache config cpu_cache_capacity exceed system memory: " + value); - } else if (cpu_cache_capacity > (double)total_mem * 0.9) { + } else if (cpu_cache_capacity > static_cast(total_mem * 0.9)) { std::cerr << "Warning: cpu_cache_capacity value is too big" << std::endl; } int32_t buffer_value; Status s = GetDBConfigInsertBufferSize(buffer_value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + int64_t insert_buffer_size = buffer_value * GB; if (insert_buffer_size + cpu_cache_capacity >= total_mem) { return Status(SERVER_INVALID_ARGUMENT, "Sum of cpu_cache_capacity and buffer_size exceed system memory"); @@ -459,7 +552,10 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { uint64_t gpu_cache_capacity = std::stoi(value) * GB; int gpu_index; Status s = GetDBConfigBuildIndexGPU(gpu_index); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + size_t gpu_memory; if (!ValidationUtil::GetGpuMemory(gpu_index, gpu_memory).ok()) { return Status(SERVER_UNEXPECTED_ERROR, @@ -507,12 +603,13 @@ Status Config::CheckEngineConfigOmpThreadNum(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config omp_thread_num: " + value); - } else { - int32_t omp_thread = std::stoi(value); - uint32_t sys_thread_cnt = 8; - if (omp_thread > CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config omp_thread_num: " + value); - } + } + + int32_t omp_thread = std::stoi(value); + uint32_t sys_thread_cnt = 8; + CommonUtil::GetSystemAvailableThreads(sys_thread_cnt); + if (omp_thread > static_cast(sys_thread_cnt)) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config omp_thread_num: " + value); } return Status::OK(); } @@ -548,9 +645,8 @@ Config::GetConfigValueInMem(const std::string& parent_key, const std::string& ch config_map_[parent_key].find(child_key) != config_map_[parent_key].end()) { value = config_map_[parent_key][child_key]; return Status::OK(); - } else { - return Status(SERVER_UNEXPECTED_ERROR, "key not exist"); } + return Status(SERVER_UNEXPECTED_ERROR, "key not exist"); } void @@ -849,7 +945,10 @@ Status Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { std::string str = GetDBConfigStrArchiveDiskThreshold(); Status s = CheckDBConfigArchiveDiskThreshold(str); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + value = std::stoi(str); return Status::OK(); } @@ -858,7 +957,10 @@ Status Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { std::string str = GetDBConfigStrArchiveDaysThreshold(); Status s = CheckDBConfigArchiveDaysThreshold(str); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + value = std::stoi(str); return Status::OK(); } @@ -867,7 +969,10 @@ Status Config::GetDBConfigInsertBufferSize(int32_t& value) { std::string str = GetDBConfigStrInsertBufferSize(); Status s = CheckDBConfigInsertBufferSize(str); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + value = std::stoi(str); return Status::OK(); } @@ -876,7 +981,10 @@ Status Config::GetDBConfigBuildIndexGPU(int32_t& value) { std::string str = GetDBConfigStrBuildIndexGPU(); Status s = CheckDBConfigBuildIndexGPU(str); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + value = std::stoi(str); return Status::OK(); } @@ -885,7 +993,10 @@ Status Config::GetMetricConfigEnableMonitor(bool& value) { std::string str = GetMetricConfigStrEnableMonitor(); Status s = CheckMetricConfigEnableMonitor(str); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); return Status::OK(); @@ -907,7 +1018,10 @@ Status Config::GetCacheConfigCpuCacheCapacity(int32_t& value) { std::string str = GetCacheConfigStrCpuCacheCapacity(); Status s = CheckCacheConfigCpuCacheCapacity(str); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + value = std::stoi(str); return Status::OK(); } @@ -916,7 +1030,10 @@ Status Config::GetCacheConfigCpuCacheThreshold(float& value) { std::string str = GetCacheConfigStrCpuCacheThreshold(); Status s = CheckCacheConfigCpuCacheThreshold(str); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + value = std::stof(str); return Status::OK(); } @@ -925,7 +1042,10 @@ Status Config::GetCacheConfigGpuCacheCapacity(int32_t& value) { std::string str = GetCacheConfigStrGpuCacheCapacity(); Status s = CheckCacheConfigGpuCacheCapacity(str); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + value = std::stoi(str); return Status::OK(); } @@ -934,7 +1054,10 @@ Status Config::GetCacheConfigGpuCacheThreshold(float& value) { std::string str = GetCacheConfigStrGpuCacheThreshold(); Status s = CheckCacheConfigGpuCacheThreshold(str); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + value = std::stof(str); return Status::OK(); } @@ -943,7 +1066,10 @@ Status Config::GetCacheConfigCacheInsertData(bool& value) { std::string str = GetCacheConfigStrCacheInsertData(); Status s = CheckCacheConfigCacheInsertData(str); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); return Status::OK(); @@ -953,7 +1079,10 @@ Status Config::GetEngineConfigUseBlasThreshold(int32_t& value) { std::string str = GetEngineConfigStrUseBlasThreshold(); Status s = CheckEngineConfigUseBlasThreshold(str); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + value = std::stoi(str); return Status::OK(); } @@ -962,7 +1091,10 @@ Status Config::GetEngineConfigOmpThreadNum(int32_t& value) { std::string str = GetEngineConfigStrOmpThreadNum(); Status s = CheckEngineConfigOmpThreadNum(str); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + value = std::stoi(str); return Status::OK(); } @@ -985,7 +1117,10 @@ Config::GetResourceConfigPool(std::vector& value) { Status Config::SetServerConfigAddress(const std::string& value) { Status s = CheckServerConfigAddress(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); return Status::OK(); } @@ -993,7 +1128,10 @@ Config::SetServerConfigAddress(const std::string& value) { Status Config::SetServerConfigPort(const std::string& value) { Status s = CheckServerConfigPort(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); return Status::OK(); } @@ -1001,7 +1139,10 @@ Config::SetServerConfigPort(const std::string& value) { Status Config::SetServerConfigDeployMode(const std::string& value) { Status s = CheckServerConfigDeployMode(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); return Status::OK(); } @@ -1009,7 +1150,10 @@ Config::SetServerConfigDeployMode(const std::string& value) { Status Config::SetServerConfigTimeZone(const std::string& value) { Status s = CheckServerConfigTimeZone(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); return Status::OK(); } @@ -1018,7 +1162,10 @@ Config::SetServerConfigTimeZone(const std::string& value) { Status Config::SetDBConfigPrimaryPath(const std::string& value) { Status s = CheckDBConfigPrimaryPath(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); return Status::OK(); } @@ -1026,7 +1173,10 @@ Config::SetDBConfigPrimaryPath(const std::string& value) { Status Config::SetDBConfigSecondaryPath(const std::string& value) { Status s = CheckDBConfigSecondaryPath(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); return Status::OK(); } @@ -1034,7 +1184,10 @@ Config::SetDBConfigSecondaryPath(const std::string& value) { Status Config::SetDBConfigBackendUrl(const std::string& value) { Status s = CheckDBConfigBackendUrl(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); return Status::OK(); } @@ -1042,7 +1195,10 @@ Config::SetDBConfigBackendUrl(const std::string& value) { Status Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { Status s = CheckDBConfigArchiveDiskThreshold(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); return Status::OK(); } @@ -1050,7 +1206,10 @@ Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { Status Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { Status s = CheckDBConfigArchiveDaysThreshold(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); return Status::OK(); } @@ -1058,7 +1217,10 @@ Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { Status Config::SetDBConfigInsertBufferSize(const std::string& value) { Status s = CheckDBConfigInsertBufferSize(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); return Status::OK(); } @@ -1066,7 +1228,10 @@ Config::SetDBConfigInsertBufferSize(const std::string& value) { Status Config::SetDBConfigBuildIndexGPU(const std::string& value) { Status s = CheckDBConfigBuildIndexGPU(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); return Status::OK(); } @@ -1075,7 +1240,10 @@ Config::SetDBConfigBuildIndexGPU(const std::string& value) { Status Config::SetMetricConfigEnableMonitor(const std::string& value) { Status s = CheckMetricConfigEnableMonitor(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_ENABLE_MONITOR, value); return Status::OK(); } @@ -1083,7 +1251,10 @@ Config::SetMetricConfigEnableMonitor(const std::string& value) { Status Config::SetMetricConfigCollector(const std::string& value) { Status s = CheckMetricConfigCollector(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_COLLECTOR, value); return Status::OK(); } @@ -1091,7 +1262,10 @@ Config::SetMetricConfigCollector(const std::string& value) { Status Config::SetMetricConfigPrometheusPort(const std::string& value) { Status s = CheckMetricConfigPrometheusPort(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_PROMETHEUS_PORT, value); return Status::OK(); } @@ -1100,7 +1274,10 @@ Config::SetMetricConfigPrometheusPort(const std::string& value) { Status Config::SetCacheConfigCpuCacheCapacity(const std::string& value) { Status s = CheckCacheConfigCpuCacheCapacity(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_CAPACITY, value); return Status::OK(); } @@ -1108,7 +1285,10 @@ Config::SetCacheConfigCpuCacheCapacity(const std::string& value) { Status Config::SetCacheConfigCpuCacheThreshold(const std::string& value) { Status s = CheckCacheConfigCpuCacheThreshold(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); return Status::OK(); } @@ -1116,7 +1296,10 @@ Config::SetCacheConfigCpuCacheThreshold(const std::string& value) { Status Config::SetCacheConfigGpuCacheCapacity(const std::string& value) { Status s = CheckCacheConfigGpuCacheCapacity(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_CAPACITY, value); return Status::OK(); } @@ -1124,7 +1307,10 @@ Config::SetCacheConfigGpuCacheCapacity(const std::string& value) { Status Config::SetCacheConfigGpuCacheThreshold(const std::string& value) { Status s = CheckCacheConfigGpuCacheThreshold(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value); return Status::OK(); } @@ -1132,7 +1318,10 @@ Config::SetCacheConfigGpuCacheThreshold(const std::string& value) { Status Config::SetCacheConfigCacheInsertData(const std::string& value) { Status s = CheckCacheConfigCacheInsertData(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CACHE_INSERT_DATA, value); return Status::OK(); } @@ -1141,7 +1330,10 @@ Config::SetCacheConfigCacheInsertData(const std::string& value) { Status Config::SetEngineConfigUseBlasThreshold(const std::string& value) { Status s = CheckEngineConfigUseBlasThreshold(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value); return Status::OK(); } @@ -1149,7 +1341,10 @@ Config::SetEngineConfigUseBlasThreshold(const std::string& value) { Status Config::SetEngineConfigOmpThreadNum(const std::string& value) { Status s = CheckEngineConfigOmpThreadNum(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_OMP_THREAD_NUM, value); return Status::OK(); } @@ -1158,7 +1353,10 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) { Status Config::SetResourceConfigMode(const std::string& value) { Status s = CheckResourceConfigMode(value); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_MODE, value); return Status::OK(); } diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index c0e02f24e4..713714cadc 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -30,9 +30,6 @@ namespace zilliz { namespace milvus { namespace server { -DBWrapper::DBWrapper() { -} - Status DBWrapper::StartService() { Config& config = Config::GetInstance(); @@ -41,27 +38,37 @@ DBWrapper::StartService() { engine::DBOptions opt; s = config.GetDBConfigBackendUrl(opt.meta_.backend_uri_); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } std::string path; s = config.GetDBConfigPrimaryPath(path); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } opt.meta_.path_ = path + "/db"; std::string db_slave_path; s = config.GetDBConfigSecondaryPath(db_slave_path); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta_.slave_paths_); // cache config s = config.GetCacheConfigCacheInsertData(opt.insert_cache_immediately_); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } std::string mode; s = config.GetServerConfigDeployMode(mode); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } if (mode == "single") { opt.mode_ = engine::DBOptions::MODE::SINGLE; @@ -78,14 +85,17 @@ DBWrapper::StartService() { // engine config int32_t omp_thread; s = config.GetEngineConfigOmpThreadNum(omp_thread); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + if (omp_thread > 0) { omp_set_num_threads(omp_thread); SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; } else { uint32_t sys_thread_cnt = 8; if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { - omp_thread = (int32_t)ceil(sys_thread_cnt * 0.5); + omp_thread = static_cast(ceil(sys_thread_cnt * 0.5)); omp_set_num_threads(omp_thread); } } @@ -93,20 +103,29 @@ DBWrapper::StartService() { // init faiss global variable int32_t use_blas_threshold; s = config.GetEngineConfigUseBlasThreshold(use_blas_threshold); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + faiss::distance_compute_blas_threshold = use_blas_threshold; // set archive config engine::ArchiveConf::CriteriaT criterial; int32_t disk, days; s = config.GetDBConfigArchiveDiskThreshold(disk); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + if (disk > 0) { criterial[engine::ARCHIVE_CONF_DISK] = disk; } s = config.GetDBConfigArchiveDaysThreshold(days); - if (!s.ok()) return s; + if (!s.ok()) { + return s; + } + if (days > 0) { criterial[engine::ARCHIVE_CONF_DAYS] = days; } diff --git a/cpp/src/server/DBWrapper.h b/cpp/src/server/DBWrapper.h index 5a035e44bc..6adc68dcb3 100644 --- a/cpp/src/server/DBWrapper.h +++ b/cpp/src/server/DBWrapper.h @@ -28,7 +28,7 @@ namespace server { class DBWrapper { private: - DBWrapper(); + DBWrapper() = default; ~DBWrapper() = default; public: diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index b86f386acc..2e06cccf8a 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -146,7 +146,7 @@ Server::Daemonize() { void Server::Start() { - if (daemonized_) { + if (daemonized_ != 0) { Daemonize(); } diff --git a/cpp/src/server/grpc_impl/GrpcRequestHandler.h b/cpp/src/server/grpc_impl/GrpcRequestHandler.h index 73f05572de..66ec126371 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestHandler.h +++ b/cpp/src/server/grpc_impl/GrpcRequestHandler.h @@ -227,7 +227,7 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { */ ::grpc::Status ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, - ::milvus::grpc::TableNameList* table_name_list) override; + ::milvus::grpc::TableNameList* response) override; /** * @brief Give the server status diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp index fc302414c2..9135cd6f6c 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp @@ -163,7 +163,8 @@ GrpcRequestScheduler::Stop() { } for (auto iter : execute_threads_) { - if (iter == nullptr) continue; + if (iter == nullptr) + continue; iter->join(); } diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.h b/cpp/src/server/grpc_impl/GrpcRequestScheduler.h index b5a87c0bfa..d38f4c67ac 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.h +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.h @@ -69,7 +69,7 @@ class GrpcBaseTask { OnExecute() = 0; Status - SetStatus(ErrorCode error_code, const std::string& msg); + SetStatus(ErrorCode error_code, const std::string& error_msg); protected: mutable std::mutex finish_mtx_; diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index 7ced2e69c1..a76552b65b 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -159,7 +159,7 @@ CreateTableTask::OnExecute() { // step 2: construct table schema engine::meta::TableSchema table_info; table_info.table_id_ = schema_->table_name(); - table_info.dimension_ = (uint16_t)schema_->dimension(); + table_info.dimension_ = static_cast(schema_->dimension()); table_info.index_file_size_ = schema_->index_file_size(); table_info.metric_type_ = schema_->metric_type(); @@ -446,13 +446,13 @@ InsertTask::OnExecute() { // all user provide id, or all internal id bool user_provide_ids = !insert_param_->row_id_array().empty(); // user already provided id before, all insert action require user id - if ((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) && !user_provide_ids) { + if ((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) != 0 && !user_provide_ids) { return Status(SERVER_ILLEGAL_VECTOR_ID, "Table vector ids are user defined, please provide id for this batch"); } // user didn't provided id before, no need to provide user id - if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) && user_provide_ids) { + if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) != 0 && user_provide_ids) { return Status(SERVER_ILLEGAL_VECTOR_ID, "Table vector ids are auto generated, no need to provide id for this batch"); } @@ -487,12 +487,12 @@ InsertTask::OnExecute() { rc.ElapseFromBegin("prepare vectors data"); // step 5: insert vectors - auto vec_count = (uint64_t)insert_param_->row_record_array_size(); + auto vec_count = static_cast(insert_param_->row_record_array_size()); std::vector vec_ids(insert_param_->row_id_array_size(), 0); if (!insert_param_->row_id_array().empty()) { const int64_t* src_data = insert_param_->row_id_array().data(); int64_t* target_data = vec_ids.data(); - memcpy(target_data, src_data, (size_t)(sizeof(int64_t) * insert_param_->row_id_array_size())); + memcpy(target_data, src_data, static_cast(sizeof(int64_t) * insert_param_->row_id_array_size())); } status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), vec_count, vec_f.data(), vec_ids); @@ -710,7 +710,7 @@ CountTableTask::OnExecute() { return status; } - row_count_ = (int64_t)row_count; + row_count_ = static_cast(row_count); rc.ElapseFromBegin("total cost"); } catch (std::exception& ex) { diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.h b/cpp/src/server/grpc_impl/GrpcRequestTask.h index f0e77ea791..af5ed2bc80 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.h +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.h @@ -41,7 +41,7 @@ class CreateTableTask : public GrpcBaseTask { Create(const ::milvus::grpc::TableSchema* schema); protected: - explicit CreateTableTask(const ::milvus::grpc::TableSchema* request); + explicit CreateTableTask(const ::milvus::grpc::TableSchema* schema); Status OnExecute() override; @@ -57,7 +57,7 @@ class HasTableTask : public GrpcBaseTask { Create(const std::string& table_name, bool& has_table); protected: - HasTableTask(const std::string& request, bool& has_table); + HasTableTask(const std::string& table_name, bool& has_table); Status OnExecute() override; @@ -104,10 +104,10 @@ class DropTableTask : public GrpcBaseTask { class CreateIndexTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const ::milvus::grpc::IndexParam* index_Param); + Create(const ::milvus::grpc::IndexParam* index_param); protected: - explicit CreateIndexTask(const ::milvus::grpc::IndexParam* index_Param); + explicit CreateIndexTask(const ::milvus::grpc::IndexParam* index_param); Status OnExecute() override; @@ -136,10 +136,10 @@ class ShowTablesTask : public GrpcBaseTask { class InsertTask : public GrpcBaseTask { public: static BaseTaskPtr - Create(const ::milvus::grpc::InsertParam* insert_Param, ::milvus::grpc::VectorIds* record_ids_); + Create(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids); protected: - InsertTask(const ::milvus::grpc::InsertParam* insert_Param, ::milvus::grpc::VectorIds* record_ids_); + InsertTask(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids); Status OnExecute() override; diff --git a/cpp/src/utils/CommonUtil.cpp b/cpp/src/utils/CommonUtil.cpp index 760c54c4a7..14ed34481f 100644 --- a/cpp/src/utils/CommonUtil.cpp +++ b/cpp/src/utils/CommonUtil.cpp @@ -59,7 +59,9 @@ CommonUtil::GetSystemAvailableThreads(uint32_t& thread_count) { // threadCnt = std::thread::hardware_concurrency(); thread_count = sysconf(_SC_NPROCESSORS_CONF); thread_count *= THREAD_MULTIPLY_CPU; - if (thread_count == 0) thread_count = 8; + if (thread_count == 0) { + thread_count = 8; + } return true; } @@ -162,9 +164,9 @@ CommonUtil::GetFileSize(const std::string& path) { struct stat file_info; if (stat(path.c_str(), &file_info) < 0) { return 0; - } else { - return (uint64_t)file_info.st_size; } + + return static_cast(file_info.st_size); } std::string diff --git a/cpp/src/utils/Status.cpp b/cpp/src/utils/Status.cpp index 3f807a8a51..95132d3a2a 100644 --- a/cpp/src/utils/Status.cpp +++ b/cpp/src/utils/Status.cpp @@ -29,7 +29,7 @@ Status::Status(StatusCode code, const std::string& msg) { // 4 bytes store message length // the left bytes store message string const uint32_t length = (uint32_t)msg.size(); - char* result = new char[length + sizeof(length) + CODE_WIDTH]; + auto result = new char[length + sizeof(length) + CODE_WIDTH]; std::memcpy(result, &code, CODE_WIDTH); std::memcpy(result + CODE_WIDTH, &length, sizeof(length)); memcpy(result + sizeof(length) + CODE_WIDTH, msg.data(), length); @@ -76,7 +76,7 @@ Status::CopyFrom(const Status& s) { memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length)); int buff_len = length + sizeof(length) + CODE_WIDTH; state_ = new char[buff_len]; - memcpy((void*)state_, (void*)s.state_, buff_len); + memcpy(state_, s.state_, buff_len); } void diff --git a/cpp/src/utils/Status.h b/cpp/src/utils/Status.h index 4450a4c0e7..cb6c134781 100644 --- a/cpp/src/utils/Status.h +++ b/cpp/src/utils/Status.h @@ -71,7 +71,7 @@ class Status { MoveFrom(Status& s); private: - const char* state_ = nullptr; + char* state_ = nullptr; }; // Status } // namespace milvus diff --git a/cpp/src/utils/ThreadPool.h b/cpp/src/utils/ThreadPool.h index 24c4d20ad5..db4c9d11ad 100644 --- a/cpp/src/utils/ThreadPool.h +++ b/cpp/src/utils/ThreadPool.h @@ -70,7 +70,8 @@ inline ThreadPool::ThreadPool(size_t threads, size_t queue_size) : max_queue_siz { std::unique_lock lock(this->queue_mutex_); this->condition_.wait(lock, [this] { return this->stop || !this->tasks_.empty(); }); - if (this->stop && this->tasks_.empty()) return; + if (this->stop && this->tasks_.empty()) + return; task = std::move(this->tasks_.front()); this->tasks_.pop(); } @@ -95,7 +96,8 @@ ThreadPool::enqueue(F&& f, Args&&... args) -> std::future lock(queue_mutex_); this->condition_.wait(lock, [this] { return this->tasks_.size() < max_queue_size_; }); // don't allow enqueueing after stopping the pool - if (stop) throw std::runtime_error("enqueue on stopped ThreadPool"); + if (stop) + throw std::runtime_error("enqueue on stopped ThreadPool"); tasks_.emplace([task]() { (*task)(); }); } diff --git a/cpp/src/utils/TimeRecorder.cpp b/cpp/src/utils/TimeRecorder.cpp index 64a0c70fc9..cffcbc98ef 100644 --- a/cpp/src/utils/TimeRecorder.cpp +++ b/cpp/src/utils/TimeRecorder.cpp @@ -39,7 +39,8 @@ TimeRecorder::GetTimeSpanStr(double span) { void TimeRecorder::PrintTimeRecord(const std::string& msg, double span) { std::string str_log; - if (!header_.empty()) str_log += header_ + ": "; + if (!header_.empty()) + str_log += header_ + ": "; str_log += msg; str_log += " ("; str_log += TimeRecorder::GetTimeSpanStr(span); diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index 7047e65d30..a39427ef15 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -83,8 +83,8 @@ ValidationUtil::ValidateTableDimension(int64_t dimension) { Status ValidationUtil::ValidateTableIndexType(int32_t index_type) { - int engine_type = (int)engine::EngineType(index_type); - if (engine_type <= 0 || engine_type > (int)engine::EngineType::MAX_VALUE) { + int engine_type = static_cast(engine::EngineType(index_type)); + if (engine_type <= 0 || engine_type > static_cast(engine::EngineType::MAX_VALUE)) { std::string msg = "Invalid index type: " + std::to_string(index_type); SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_INDEX_TYPE, msg); @@ -117,7 +117,8 @@ ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) { Status ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { - if (metric_type != (int32_t)engine::MetricType::L2 && metric_type != (int32_t)engine::MetricType::IP) { + if (metric_type != static_cast(engine::MetricType::L2) && + metric_type != static_cast(engine::MetricType::IP)) { std::string msg = "Invalid metric type: " + std::to_string(metric_type); SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_INDEX_METRIC_TYPE, msg); @@ -151,7 +152,7 @@ Status ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { int num_devices = 0; auto cuda_err = cudaGetDeviceCount(&num_devices); - if (cuda_err) { + if (cuda_err != cudaSuccess) { std::string msg = "Failed to get gpu card number, cuda error:" + std::to_string(cuda_err); SERVER_LOG_ERROR << msg; return Status(SERVER_UNEXPECTED_ERROR, msg); @@ -222,9 +223,8 @@ ValidationUtil::ValidateStringIsBool(const std::string& str) { if (s == "true" || s == "on" || s == "yes" || s == "1" || s == "false" || s == "off" || s == "no" || s == "0" || s.empty()) { return Status::OK(); - } else { - return Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str); } + return Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str); } Status From d1251fb0c1e2d07d3d39c0b73800fa1b472fa1f7 Mon Sep 17 00:00:00 2001 From: starlord Date: Sat, 28 Sep 2019 15:00:26 +0800 Subject: [PATCH 143/217] format code by clang-tidy Former-commit-id: 437b1fff79da965b8aafce026c8be4f9d582c909 --- .clang-tidy | 4 ++- cpp/build.sh | 16 +++++------ cpp/src/cache/CpuCacheMgr.h | 2 +- cpp/src/core/test/kdtree.cpp | 2 +- cpp/src/core/test/test_ivf.cpp | 2 +- cpp/src/core/test/test_kdt.cpp | 2 +- .../SPTAG/AnnService/src/Core/MetadataSet.cpp | 4 +-- cpp/src/db/DBImpl.cpp | 16 ++++++----- cpp/src/db/Utils.cpp | 2 +- cpp/src/db/engine/ExecutionEngineImpl.cpp | 3 +- cpp/src/db/meta/MySQLConnectionPool.cpp | 2 +- cpp/src/db/meta/MySQLMetaImpl.cpp | 15 +++++----- cpp/src/db/meta/SqliteMetaImpl.cpp | 4 +-- cpp/src/metrics/SystemInfo.cpp | 28 ++++++++++++------- cpp/src/scheduler/ResourceMgr.cpp | 2 +- cpp/src/scheduler/ResourceMgr.h | 4 +-- cpp/src/scheduler/SchedInst.cpp | 2 +- cpp/src/scheduler/Scheduler.cpp | 2 +- cpp/src/scheduler/Scheduler.h | 2 +- cpp/src/scheduler/TaskCreator.cpp | 2 +- cpp/src/scheduler/TaskTable.h | 4 +-- .../scheduler/action/PushTaskToNeighbour.cpp | 2 +- cpp/src/scheduler/resource/Node.h | 2 +- cpp/src/scheduler/resource/Resource.h | 6 ++-- cpp/src/scheduler/task/SearchTask.h | 2 +- cpp/src/sdk/examples/grpcsimple/main.cpp | 8 +++--- cpp/src/sdk/grpc/ClientProxy.cpp | 12 +++----- cpp/src/server/grpc_impl/GrpcRequestTask.cpp | 2 +- cpp/unittest/db/db_tests.cpp | 2 +- cpp/unittest/server/cache_test.cpp | 2 +- 30 files changed, 83 insertions(+), 75 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 1f7850efe7..1695fa6685 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -18,8 +18,10 @@ Checks: 'clang-diagnostic-*,clang-analyzer-*,-clang-analyzer-alpha*,google-*,modernize-*,readability-*' # produce HeaderFilterRegex from cpp/build-support/lint_exclusions.txt with: # echo -n '^('; sed -e 's/*/\.*/g' cpp/build-support/lint_exclusions.txt | tr '\n' '|'; echo ')$' -HeaderFilterRegex: '^(.*cmake-build-debug.*|.*cmake-build-release.*|.*cmake_build.*|.*src/thirdparty.*|.*src/core/thirdparty.*|.*src/grpc.*|)$' +HeaderFilterRegex: '^(.*cmake-build-debug.*|.*cmake-build-release.*|.*cmake_build.*|.*src/core/thirdparty.*|.*thirdparty.*|.*easylogging++.*|.*SqliteMetaImpl.cpp|.*src/grpc.*|.*src/core.*|.*src/wrapper.*)$' AnalyzeTemporaryDtors: true +ChainedConditionalReturn: 1 +ChainedConditionalAssignment: 1 CheckOptions: - key: google-readability-braces-around-statements.ShortStatementLines value: '1' diff --git a/cpp/build.sh b/cpp/build.sh index 69fd4f0f29..deda35ab27 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -111,14 +111,14 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then exit 1 fi echo "clang-format check passed!" -# -# # clang-tidy check -# make check-clang-tidy -# if [ $? -ne 0 ]; then -# echo "ERROR! clang-tidy check failed" -# exit 1 -# fi -# echo "clang-tidy check passed!" + + # clang-tidy check + make check-clang-tidy + if [ $? -ne 0 ]; then + echo "ERROR! clang-tidy check failed" + exit 1 + fi + echo "clang-tidy check passed!" else # compile and build make -j 4 || exit 1 diff --git a/cpp/src/cache/CpuCacheMgr.h b/cpp/src/cache/CpuCacheMgr.h index fbc4f6dfb3..ad6087de7c 100644 --- a/cpp/src/cache/CpuCacheMgr.h +++ b/cpp/src/cache/CpuCacheMgr.h @@ -32,7 +32,7 @@ class CpuCacheMgr : public CacheMgr { CpuCacheMgr(); public: - // TODO: use smart pointer instead + // TODO(myh): use smart pointer instead static CpuCacheMgr* GetInstance(); diff --git a/cpp/src/core/test/kdtree.cpp b/cpp/src/core/test/kdtree.cpp index 0b63e657e1..b3de8f345c 100644 --- a/cpp/src/core/test/kdtree.cpp +++ b/cpp/src/core/test/kdtree.cpp @@ -44,7 +44,7 @@ DatasetPtr generate_queries(int64_t n, int64_t d, int64_t k, int64_t base) { size_t size = sizeof(float) * n * d; auto v = (float *) malloc(size); - // TODO: check malloc + // TODO(lxj): check malloc for (auto i = 0; i < n; ++i) { for (auto j = 0; j < d; ++j) { v[i * d + j] = float(base + i); diff --git a/cpp/src/core/test/test_ivf.cpp b/cpp/src/core/test/test_ivf.cpp index 1eeb1f9872..7196fe15e4 100644 --- a/cpp/src/core/test/test_ivf.cpp +++ b/cpp/src/core/test/test_ivf.cpp @@ -694,4 +694,4 @@ TEST_F(GPURESTEST, TrainAndSearch) { -// TODO(linxj): Add exception test +// TODO(lxj): Add exception test diff --git a/cpp/src/core/test/test_kdt.cpp b/cpp/src/core/test/test_kdt.cpp index fce233aca7..3780f655cc 100644 --- a/cpp/src/core/test/test_kdt.cpp +++ b/cpp/src/core/test/test_kdt.cpp @@ -85,7 +85,7 @@ void PrintResult(const DatasetPtr &result, std::cout << "dist\n" << ss_dist.str() << std::endl; } -// TODO(linxj): add test about count() and dimension() +// TODO(lxj): add test about count() and dimension() TEST_F(KDTTest, kdt_basic) { assert(!xb.empty()); diff --git a/cpp/src/core/thirdparty/SPTAG/AnnService/src/Core/MetadataSet.cpp b/cpp/src/core/thirdparty/SPTAG/AnnService/src/Core/MetadataSet.cpp index 2eefcea68f..a5d410ce5e 100644 --- a/cpp/src/core/thirdparty/SPTAG/AnnService/src/Core/MetadataSet.cpp +++ b/cpp/src/core/thirdparty/SPTAG/AnnService/src/Core/MetadataSet.cpp @@ -179,13 +179,13 @@ FileMetadataSet::SaveMetadata(const std::string& p_metaFile, const std::string& ErrorCode FileMetadataSet::SaveMetadataToMemory(void **pGraphMemFile, int64_t &len) { - // TODO: serialize file to mem? + // TODO(lxj): serialize file to mem? return ErrorCode::Fail; } ErrorCode FileMetadataSet::LoadMetadataFromMemory(void *pGraphMemFile) { - // TODO: not support yet + // TODO(lxj): not support yet return ErrorCode::Fail; } diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 2e6955a81d..10cdf60b5a 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -242,7 +242,7 @@ DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) { } Status -DBImpl::InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) { +DBImpl::InsertVectors(const std::string& table_id, uint64_t n, const float* vectors, IDNumbers& vector_ids) { // ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache"; if (shutting_down_.load(std::memory_order_acquire)) { return Status(DB_ERROR, "Milsvus server is shutdown!"); @@ -250,7 +250,7 @@ DBImpl::InsertVectors(const std::string& table_id_, uint64_t n, const float* vec Status status; zilliz::milvus::server::CollectInsertMetrics metrics(n, status); - status = mem_mgr_->InsertVectors(table_id_, n, vectors, vector_ids_); + status = mem_mgr_->InsertVectors(table_id, n, vectors, vector_ids); // std::chrono::microseconds time_span = // std::chrono::duration_cast(end_time - start_time); // double average_time = double(time_span.count()) / n; @@ -295,15 +295,17 @@ DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) { // for IDMAP type, only wait all NEW file converted to RAW file // for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files std::vector file_types; - if (index.engine_type_ == (int)EngineType::FAISS_IDMAP) { + if (index.engine_type_ == static_cast(EngineType::FAISS_IDMAP)) { file_types = { - (int)meta::TableFileSchema::NEW, (int)meta::TableFileSchema::NEW_MERGE, + static_cast(meta::TableFileSchema::NEW), static_cast(meta::TableFileSchema::NEW_MERGE), }; } else { file_types = { - (int)meta::TableFileSchema::RAW, (int)meta::TableFileSchema::NEW, - (int)meta::TableFileSchema::NEW_MERGE, (int)meta::TableFileSchema::NEW_INDEX, - (int)meta::TableFileSchema::TO_INDEX, + static_cast(meta::TableFileSchema::RAW), + static_cast(meta::TableFileSchema::NEW), + static_cast(meta::TableFileSchema::NEW_MERGE), + static_cast(meta::TableFileSchema::NEW_INDEX), + static_cast(meta::TableFileSchema::TO_INDEX), }; } diff --git a/cpp/src/db/Utils.cpp b/cpp/src/db/Utils.cpp index 2055ff391c..d9e500f84a 100644 --- a/cpp/src/db/Utils.cpp +++ b/cpp/src/db/Utils.cpp @@ -228,7 +228,7 @@ ParseMetaUri(const std::string& uri, MetaUriInfo& info) { info.port_ = pieces_match[5].str(); info.db_name_ = pieces_match[6].str(); - // TODO: verify host, port... + // TODO(myh): verify host, port... } else { return Status(DB_INVALID_META_URI, "Invalid meta uri: " + uri); } diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 52b4992dab..2a239ddbea 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -342,8 +342,9 @@ Status ExecutionEngineImpl::Init() { server::Config& config = server::Config::GetInstance(); Status s = config.GetDBConfigBuildIndexGPU(gpu_num_); - if (!s.ok()) + if (!s.ok()) { return s; + } return Status::OK(); } diff --git a/cpp/src/db/meta/MySQLConnectionPool.cpp b/cpp/src/db/meta/MySQLConnectionPool.cpp index 2d09874d3f..6c93120107 100644 --- a/cpp/src/db/meta/MySQLConnectionPool.cpp +++ b/cpp/src/db/meta/MySQLConnectionPool.cpp @@ -68,7 +68,7 @@ MySQLConnectionPool::create() { try { // Create connection using the parameters we were passed upon // creation. - mysqlpp::Connection* conn = new mysqlpp::Connection(); + auto conn = new mysqlpp::Connection(); conn->set_option(new mysqlpp::ReconnectOption(true)); conn->connect(db_.empty() ? 0 : db_.c_str(), server_.empty() ? 0 : server_.c_str(), user_.empty() ? 0 : user_.c_str(), password_.empty() ? 0 : password_.c_str(), port_); diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index e9654c7866..db93ce9ec6 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -50,11 +50,11 @@ HandleException(const std::string& desc, const char* what = nullptr) { if (what == nullptr) { ENGINE_LOG_ERROR << desc; return Status(DB_META_TRANSACTION_FAILED, desc); - } else { - std::string msg = desc + ":" + what; - ENGINE_LOG_ERROR << msg; - return Status(DB_META_TRANSACTION_FAILED, msg); } + + std::string msg = desc + ":" + what; + ENGINE_LOG_ERROR << msg; + return Status(DB_META_TRANSACTION_FAILED, msg); } class MetaField { @@ -324,7 +324,7 @@ MySQLMetaImpl::Initialize() { return Status::OK(); } -// PXU TODO: Temp solution. Will fix later +// TODO(myh): Delete single vecotor by id Status MySQLMetaImpl::DropPartitionsByDates(const std::string& table_id, const DatesT& dates) { if (dates.empty()) { @@ -1379,7 +1379,7 @@ MySQLMetaImpl::GetTableFiles(const std::string& table_id, const std::vector '9') p++; + while (*p < '0' || *p > '9') { + p++; + } line[i - 3] = '\0'; i = atoi(p); return static_cast(i); @@ -105,7 +109,7 @@ SystemInfo::GetProcessUsedMemory() { constexpr uint64_t KB_SIZE = 1024; char line[line_length]; - while (fgets(line, line_length, file) != NULL) { + while (fgets(line, line_length, file) != nullptr) { if (strncmp(line, "VmRSS:", 6) == 0) { result = ParseLine(line); break; @@ -118,9 +122,12 @@ SystemInfo::GetProcessUsedMemory() { double SystemInfo::MemoryPercent() { - if (!initialized_) + if (!initialized_) { Init(); - return (double)(GetProcessUsedMemory() * 100) / (double)total_ram_; + } + + double mem_used = static_cast(GetProcessUsedMemory() * 100); + return mem_used / static_cast(total_ram_); } std::vector @@ -174,8 +181,9 @@ SystemInfo::getTotalCpuTime(std::vector& work_time_array) { double SystemInfo::CPUPercent() { - if (!initialized_) + if (!initialized_) { Init(); + } struct tms time_sample; clock_t now; double percent; @@ -233,7 +241,7 @@ SystemInfo::CPUTemperature() { for (int i = 0; i <= num_physical_processors_; ++i) { std::string path = "/sys/class/thermal/thermal_zone" + std::to_string(i) + "/temp"; FILE* file = fopen(path.data(), "r"); - if (file == NULL) { + if (file == nullptr) { perror("Could not open thermal file"); return result; } @@ -269,7 +277,7 @@ SystemInfo::Octets() { std::ifstream file(filename); std::string lastline = ""; std::string line = ""; - while (file) { + while (true) { getline(file, line); if (file.fail()) { break; diff --git a/cpp/src/scheduler/ResourceMgr.cpp b/cpp/src/scheduler/ResourceMgr.cpp index eefa30560c..ce9fcc9fb9 100644 --- a/cpp/src/scheduler/ResourceMgr.cpp +++ b/cpp/src/scheduler/ResourceMgr.cpp @@ -75,7 +75,7 @@ ResourceMgr::Connect(const std::string& name1, const std::string& name2, Connect auto res2 = GetResource(name2); if (res1 && res2) { res1->AddNeighbour(std::static_pointer_cast(res2), connection); - // TODO: enable when task balance supported + // TODO(wxy): enable when task balance supported // res2->AddNeighbour(std::static_pointer_cast(res1), connection); return true; } diff --git a/cpp/src/scheduler/ResourceMgr.h b/cpp/src/scheduler/ResourceMgr.h index 05393041bf..531491066f 100644 --- a/cpp/src/scheduler/ResourceMgr.h +++ b/cpp/src/scheduler/ResourceMgr.h @@ -65,7 +65,7 @@ class ResourceMgr { return disk_resources_; } - // TODO: why return shared pointer + // TODO(wxy): why return shared pointer inline std::vector GetAllResources() { return resources_; @@ -90,7 +90,7 @@ class ResourceMgr { GetNumGpuResource() const; public: - // TODO: add stats interface(low) + // TODO(wxy): add stats interface(low) public: /******** Utility Functions ********/ diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index 5c04eeb931..cdeb834f57 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -147,7 +147,7 @@ load_advance_config() { // } // } catch (const char *msg) { // SERVER_LOG_ERROR << msg; - // // TODO: throw exception instead + // // TODO(wxy): throw exception instead // exit(-1); //// throw std::exception(); // } diff --git a/cpp/src/scheduler/Scheduler.cpp b/cpp/src/scheduler/Scheduler.cpp index 5cffa99968..8272bf6219 100644 --- a/cpp/src/scheduler/Scheduler.cpp +++ b/cpp/src/scheduler/Scheduler.cpp @@ -93,7 +93,7 @@ Scheduler::Process(const EventPtr& event) { process_event(event); } -// TODO: refactor the function +// TODO(wxy): refactor the function void Scheduler::OnLoadCompleted(const EventPtr& event) { auto load_completed_event = std::static_pointer_cast(event); diff --git a/cpp/src/scheduler/Scheduler.h b/cpp/src/scheduler/Scheduler.h index e72e40d1ba..58cb4cbce8 100644 --- a/cpp/src/scheduler/Scheduler.h +++ b/cpp/src/scheduler/Scheduler.h @@ -32,7 +32,7 @@ namespace zilliz { namespace milvus { namespace scheduler { -// TODO: refactor, not friendly to unittest, logical in framework code +// TODO(wxy): refactor, not friendly to unittest, logical in framework code class Scheduler { public: explicit Scheduler(ResourceMgrWPtr res_mgr); diff --git a/cpp/src/scheduler/TaskCreator.cpp b/cpp/src/scheduler/TaskCreator.cpp index 4f8979703f..cfcb0eb448 100644 --- a/cpp/src/scheduler/TaskCreator.cpp +++ b/cpp/src/scheduler/TaskCreator.cpp @@ -33,7 +33,7 @@ TaskCreator::Create(const JobPtr& job) { return Create(std::static_pointer_cast(job)); } default: { - // TODO: error + // TODO(wxy): error return std::vector(); } } diff --git a/cpp/src/scheduler/TaskTable.h b/cpp/src/scheduler/TaskTable.h index eb6a69324d..80c8438eb1 100644 --- a/cpp/src/scheduler/TaskTable.h +++ b/cpp/src/scheduler/TaskTable.h @@ -126,7 +126,7 @@ class TaskTable { Get(uint64_t index); /* - * TODO(wxyu): BIG GC + * TODO(wxy): BIG GC * Remove sequence task which is DONE or MOVED from front; * Called by ? */ @@ -174,7 +174,7 @@ class TaskTable { public: /******** Action ********/ - // TODO: bool to Status + // TODO(wxy): bool to Status /* * Load a task; * Set state loading; diff --git a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp index 4cafffa365..6fb861c8f3 100644 --- a/cpp/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/cpp/src/scheduler/action/PushTaskToNeighbour.cpp @@ -83,7 +83,7 @@ Action::PushTaskToNeighbourRandomly(const TaskPtr& task, const ResourcePtr& self } } else { - // TODO: process + // TODO(wxy): process } } diff --git a/cpp/src/scheduler/resource/Node.h b/cpp/src/scheduler/resource/Node.h index 0713dafa2d..588eef5f2e 100644 --- a/cpp/src/scheduler/resource/Node.h +++ b/cpp/src/scheduler/resource/Node.h @@ -41,7 +41,7 @@ struct Neighbour { Connection connection; }; -// TODO(linxj): return type void -> Status +// TODO(lxj): return type void -> Status class Node { public: Node(); diff --git a/cpp/src/scheduler/resource/Resource.h b/cpp/src/scheduler/resource/Resource.h index f6ba88c2c3..2bffaae5cf 100644 --- a/cpp/src/scheduler/resource/Resource.h +++ b/cpp/src/scheduler/resource/Resource.h @@ -39,7 +39,7 @@ namespace zilliz { namespace milvus { namespace scheduler { -// TODO(wxyu): Storage, Route, Executor +// TODO(wxy): Storage, Route, Executor enum class ResourceType { DISK = 0, CPU = 1, @@ -115,11 +115,11 @@ class Resource : public Node, public std::enable_shared_from_this { return enable_executor_; } - // TODO: const + // TODO(wxy): const uint64_t NumOfTaskToExec(); - // TODO: need double ? + // TODO(wxy): need double ? inline uint64_t TaskAvgCost() const { return total_cost_ / total_task_; diff --git a/cpp/src/scheduler/task/SearchTask.h b/cpp/src/scheduler/task/SearchTask.h index 872187a077..a89df4637b 100644 --- a/cpp/src/scheduler/task/SearchTask.h +++ b/cpp/src/scheduler/task/SearchTask.h @@ -27,7 +27,7 @@ namespace zilliz { namespace milvus { namespace scheduler { -// TODO: rewrite +// TODO(wxy): rewrite class XSearchTask : public Task { public: explicit XSearchTask(TableFileSchemaPtr file); diff --git a/cpp/src/sdk/examples/grpcsimple/main.cpp b/cpp/src/sdk/examples/grpcsimple/main.cpp index 76c60ac1f8..c31f491afb 100644 --- a/cpp/src/sdk/examples/grpcsimple/main.cpp +++ b/cpp/src/sdk/examples/grpcsimple/main.cpp @@ -30,10 +30,10 @@ main(int argc, char* argv[]) { printf("Client start...\n"); std::string app_name = basename(argv[0]); - static struct option long_options[] = {{"server", optional_argument, 0, 's'}, - {"port", optional_argument, 0, 'p'}, - {"help", no_argument, 0, 'h'}, - {NULL, 0, 0, 0}}; + static struct option long_options[] = {{"server", optional_argument, nullptr, 's'}, + {"port", optional_argument, nullptr, 'p'}, + {"help", no_argument, nullptr, 'h'}, + {nullptr, 0, nullptr, 0}}; int option_index = 0; std::string address = "127.0.0.1", port = "19530"; diff --git a/cpp/src/sdk/grpc/ClientProxy.cpp b/cpp/src/sdk/grpc/ClientProxy.cpp index db4f717eaa..7e1955b04b 100644 --- a/cpp/src/sdk/grpc/ClientProxy.cpp +++ b/cpp/src/sdk/grpc/ClientProxy.cpp @@ -29,10 +29,7 @@ namespace milvus { bool UriCheck(const std::string& uri) { size_t index = uri.find_first_of(':', 0); - if (index == std::string::npos) { - return false; - } - return true; + return (index != std::string::npos); } Status @@ -99,7 +96,7 @@ ClientProxy::CreateTable(const TableSchema& param) { schema.set_table_name(param.table_name); schema.set_dimension(param.dimension); schema.set_index_file_size(param.index_file_size); - schema.set_metric_type((int32_t)param.metric_type); + schema.set_metric_type(static_cast(param.metric_type)); return client_ptr_->CreateTable(schema); } catch (std::exception& ex) { @@ -130,7 +127,6 @@ ClientProxy::DropTable(const std::string& table_name) { Status ClientProxy::CreateIndex(const IndexParam& index_param) { try { - // TODO: add index params ::milvus::grpc::IndexParam grpc_index_param; grpc_index_param.set_table_name(index_param.table_name); grpc_index_param.mutable_index()->set_index_type(static_cast(index_param.index_type)); @@ -274,7 +270,7 @@ ClientProxy::DescribeTable(const std::string& table_name, TableSchema& table_sch table_schema.table_name = grpc_schema.table_name(); table_schema.dimension = grpc_schema.dimension(); table_schema.index_file_size = grpc_schema.index_file_size(); - table_schema.metric_type = (MetricType)grpc_schema.metric_type(); + table_schema.metric_type = static_cast(grpc_schema.metric_type()); return status; } catch (std::exception& ex) { @@ -384,7 +380,7 @@ ClientProxy::DescribeIndex(const std::string& table_name, IndexParam& index_para grpc_table_name.set_table_name(table_name); ::milvus::grpc::IndexParam grpc_index_param; Status status = client_ptr_->DescribeIndex(grpc_table_name, grpc_index_param); - index_param.index_type = (IndexType)(grpc_index_param.mutable_index()->index_type()); + index_param.index_type = static_cast(grpc_index_param.mutable_index()->index_type()); index_param.nlist = grpc_index_param.mutable_index()->nlist(); return status; diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index a76552b65b..241e24e8f6 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -468,7 +468,7 @@ InsertTask::OnExecute() { // step 4: prepare float data std::vector vec_f(insert_param_->row_record_array_size() * table_info.dimension_, 0); - // TODO: change to one dimension array in protobuf or use multiple-thread to copy the data + // TODO(yk): change to one dimension array or use multiple-thread to copy the data for (size_t i = 0; i < insert_param_->row_record_array_size(); i++) { if (insert_param_->row_record_array(i).vector_data().empty()) { return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array data is empty"); diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 365fc3ab2d..0bb74437b0 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -291,7 +291,7 @@ TEST_F(DBTest, SEARCH_TEST) { ASSERT_TRUE(stat.ok()); } - // TODO(linxj): add groundTruth assert + // TODO(lxj): add groundTruth assert } TEST_F(DBTest, PRELOADTABLE_TEST) { diff --git a/cpp/unittest/server/cache_test.cpp b/cpp/unittest/server/cache_test.cpp index 056d60185a..8fea11fa62 100644 --- a/cpp/unittest/server/cache_test.cpp +++ b/cpp/unittest/server/cache_test.cpp @@ -193,7 +193,7 @@ TEST(CacheTest, GPU_CACHE_TEST) { ASSERT_EQ(gpu_mgr->ItemCount(), 0); for (auto i = 0; i < 3; i++) { - // TODO: use gpu index to mock + // TODO(myh): use gpu index to mock //each vector is 1k byte, total size less than 2G ms::engine::VecIndexPtr mock_index = std::make_shared(256, 2000000); ms::cache::DataObjPtr data_obj = std::make_shared(mock_index); From 7633437e5d50b392eb83c30d70a28f1337d3f35c Mon Sep 17 00:00:00 2001 From: starlord Date: Sat, 28 Sep 2019 15:56:40 +0800 Subject: [PATCH 144/217] format wrapper code Former-commit-id: 75fc82a41b150fbd3f1ddcc9d09c7c4b4e43c93b --- cpp/build-support/lint_exclusions.txt | 3 +- cpp/src/wrapper/ConfAdapter.cpp | 44 ++++++------- cpp/src/wrapper/ConfAdapter.h | 36 ++++++----- cpp/src/wrapper/ConfAdapterMgr.cpp | 20 +++--- cpp/src/wrapper/ConfAdapterMgr.h | 27 ++++---- cpp/src/wrapper/DataTransfer.cpp | 23 ++++--- cpp/src/wrapper/DataTransfer.h | 11 ++-- cpp/src/wrapper/KnowhereResource.cpp | 33 +++++----- cpp/src/wrapper/KnowhereResource.h | 7 +-- cpp/src/wrapper/VecImpl.cpp | 89 ++++++++++++--------------- cpp/src/wrapper/VecImpl.h | 61 ++++++++---------- cpp/src/wrapper/VecIndex.cpp | 79 +++++++++++------------- cpp/src/wrapper/VecIndex.h | 53 ++++++---------- 13 files changed, 217 insertions(+), 269 deletions(-) diff --git a/cpp/build-support/lint_exclusions.txt b/cpp/build-support/lint_exclusions.txt index 61fad726e9..b9c1eaa760 100644 --- a/cpp/build-support/lint_exclusions.txt +++ b/cpp/build-support/lint_exclusions.txt @@ -6,5 +6,4 @@ *easylogging++* *SqliteMetaImpl.cpp *src/grpc* -*src/core* -*src/wrapper* \ No newline at end of file +*src/core* \ No newline at end of file diff --git a/cpp/src/wrapper/ConfAdapter.cpp b/cpp/src/wrapper/ConfAdapter.cpp index 1ccbaee977..df1ac6bb70 100644 --- a/cpp/src/wrapper/ConfAdapter.cpp +++ b/cpp/src/wrapper/ConfAdapter.cpp @@ -15,13 +15,14 @@ // specific language governing permissions and limitations // under the License. +#include "wrapper/ConfAdapter.h" +#include "knowhere/index/vector_index/helpers/IndexParameter.h" +#include "utils/Log.h" #include -#include "ConfAdapter.h" -#include "src/utils/Log.h" -#include "knowhere/index/vector_index/helpers/IndexParameter.h" +#include -// TODO: add conf checker +// TODO(lxj): add conf checker namespace zilliz { namespace milvus { @@ -42,7 +43,7 @@ ConfAdapter::MatchBase(knowhere::Config conf) { } knowhere::Config -ConfAdapter::Match(const TempMetaConf &metaconf) { +ConfAdapter::Match(const TempMetaConf& metaconf) { auto conf = std::make_shared(); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; @@ -52,14 +53,14 @@ ConfAdapter::Match(const TempMetaConf &metaconf) { } knowhere::Config -ConfAdapter::MatchSearch(const TempMetaConf &metaconf, const IndexType &type) { +ConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) { auto conf = std::make_shared(); conf->k = metaconf.k; return conf; } knowhere::Config -IVFConfAdapter::Match(const TempMetaConf &metaconf) { +IVFConfAdapter::Match(const TempMetaConf& metaconf) { auto conf = std::make_shared(); conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; @@ -72,7 +73,7 @@ IVFConfAdapter::Match(const TempMetaConf &metaconf) { static constexpr float TYPICAL_COUNT = 1000000.0; int64_t -IVFConfAdapter::MatchNlist(const int64_t &size, const int64_t &nlist) { +IVFConfAdapter::MatchNlist(const int64_t& size, const int64_t& nlist) { if (size <= TYPICAL_COUNT / 16384 + 1) { // handle less row count, avoid nlist set to 0 return 1; @@ -84,7 +85,7 @@ IVFConfAdapter::MatchNlist(const int64_t &size, const int64_t &nlist) { } knowhere::Config -IVFConfAdapter::MatchSearch(const TempMetaConf &metaconf, const IndexType &type) { +IVFConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) { auto conf = std::make_shared(); conf->k = metaconf.k; conf->nprobe = metaconf.nprobe; @@ -95,17 +96,16 @@ IVFConfAdapter::MatchSearch(const TempMetaConf &metaconf, const IndexType &type) case IndexType::FAISS_IVFPQ_GPU: if (conf->nprobe > GPU_MAX_NRPOBE) { WRAPPER_LOG_WARNING << "When search with GPU, nprobe shoud be no more than " << GPU_MAX_NRPOBE - << ", but you passed " << conf->nprobe - << ". Search with " << GPU_MAX_NRPOBE << " instead"; + << ", but you passed " << conf->nprobe << ". Search with " << GPU_MAX_NRPOBE + << " instead"; conf->nprobe = GPU_MAX_NRPOBE; - } } return conf; } knowhere::Config -IVFSQConfAdapter::Match(const TempMetaConf &metaconf) { +IVFSQConfAdapter::Match(const TempMetaConf& metaconf) { auto conf = std::make_shared(); conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; @@ -117,7 +117,7 @@ IVFSQConfAdapter::Match(const TempMetaConf &metaconf) { } knowhere::Config -IVFPQConfAdapter::Match(const TempMetaConf &metaconf) { +IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { auto conf = std::make_shared(); conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; @@ -130,7 +130,7 @@ IVFPQConfAdapter::Match(const TempMetaConf &metaconf) { } knowhere::Config -NSGConfAdapter::Match(const TempMetaConf &metaconf) { +NSGConfAdapter::Match(const TempMetaConf& metaconf) { auto conf = std::make_shared(); conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; @@ -146,20 +146,20 @@ NSGConfAdapter::Match(const TempMetaConf &metaconf) { conf->candidate_pool_size = 200 + 100 * scale_factor; MatchBase(conf); -// WRAPPER_LOG_DEBUG << "nlist: " << conf->nlist -// << ", gpu_id: " << conf->gpu_id << ", d: " << conf->d -// << ", nprobe: " << conf->nprobe << ", knng: " << conf->knng; + // WRAPPER_LOG_DEBUG << "nlist: " << conf->nlist + // << ", gpu_id: " << conf->gpu_id << ", d: " << conf->d + // << ", nprobe: " << conf->nprobe << ", knng: " << conf->knng; return conf; } knowhere::Config -NSGConfAdapter::MatchSearch(const TempMetaConf &metaconf, const IndexType &type) { +NSGConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) { auto conf = std::make_shared(); conf->k = metaconf.k; conf->search_length = metaconf.search_length; return conf; } -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/ConfAdapter.h b/cpp/src/wrapper/ConfAdapter.h index d8281d8887..e02d76e441 100644 --- a/cpp/src/wrapper/ConfAdapter.h +++ b/cpp/src/wrapper/ConfAdapter.h @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include "knowhere/common/Config.h" #include "VecIndex.h" +#include "knowhere/common/Config.h" +#include namespace zilliz { namespace milvus { @@ -42,16 +42,17 @@ struct TempMetaConf { class ConfAdapter { public: virtual knowhere::Config - Match(const TempMetaConf &metaconf); + Match(const TempMetaConf& metaconf); virtual knowhere::Config - MatchSearch(const TempMetaConf &metaconf, const IndexType &type); + MatchSearch(const TempMetaConf& metaconf, const IndexType& type); -// virtual void -// Dump(){} + // virtual void + // Dump(){} protected: - static void MatchBase(knowhere::Config conf); + static void + MatchBase(knowhere::Config conf); }; using ConfAdapterPtr = std::shared_ptr; @@ -59,36 +60,37 @@ using ConfAdapterPtr = std::shared_ptr; class IVFConfAdapter : public ConfAdapter { public: knowhere::Config - Match(const TempMetaConf &metaconf) override; + Match(const TempMetaConf& metaconf) override; knowhere::Config - MatchSearch(const TempMetaConf &metaconf, const IndexType &type) override; + MatchSearch(const TempMetaConf& metaconf, const IndexType& type) override; protected: - static int64_t MatchNlist(const int64_t &size, const int64_t &nlist); + static int64_t + MatchNlist(const int64_t& size, const int64_t& nlist); }; class IVFSQConfAdapter : public IVFConfAdapter { public: knowhere::Config - Match(const TempMetaConf &metaconf) override; + Match(const TempMetaConf& metaconf) override; }; class IVFPQConfAdapter : public IVFConfAdapter { public: knowhere::Config - Match(const TempMetaConf &metaconf) override; + Match(const TempMetaConf& metaconf) override; }; class NSGConfAdapter : public IVFConfAdapter { public: knowhere::Config - Match(const TempMetaConf &metaconf) override; + Match(const TempMetaConf& metaconf) override; knowhere::Config - MatchSearch(const TempMetaConf &metaconf, const IndexType &type) final; + MatchSearch(const TempMetaConf& metaconf, const IndexType& type) final; }; -} -} -} +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/ConfAdapterMgr.cpp b/cpp/src/wrapper/ConfAdapterMgr.cpp index cb7331a99d..4ed80b22b7 100644 --- a/cpp/src/wrapper/ConfAdapterMgr.cpp +++ b/cpp/src/wrapper/ConfAdapterMgr.cpp @@ -15,18 +15,17 @@ // specific language governing permissions and limitations // under the License. - -#include "src/utils/Exception.h" -#include "ConfAdapterMgr.h" - +#include "wrapper/ConfAdapterMgr.h" +#include "utils/Exception.h" namespace zilliz { namespace milvus { namespace engine { ConfAdapterPtr -AdapterMgr::GetAdapter(const IndexType &indexType) { - if (!init_) RegisterAdapter(); +AdapterMgr::GetAdapter(const IndexType& indexType) { + if (!init_) + RegisterAdapter(); auto it = table_.find(indexType); if (it != table_.end()) { @@ -36,8 +35,8 @@ AdapterMgr::GetAdapter(const IndexType &indexType) { } } +#define REGISTER_CONF_ADAPTER(T, KEY, NAME) static AdapterMgr::register_t reg_##NAME##_(KEY) -#define REGISTER_CONF_ADAPTER(T, KEY, NAME) static AdapterMgr::register_treg_##NAME##_(KEY) void AdapterMgr::RegisterAdapter() { init_ = true; @@ -58,7 +57,6 @@ AdapterMgr::RegisterAdapter() { REGISTER_CONF_ADAPTER(NSGConfAdapter, IndexType::NSG_MIX, nsg_mix); } -} // engine -} // milvus -} // zilliz - +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/ConfAdapterMgr.h b/cpp/src/wrapper/ConfAdapterMgr.h index fe8b4f07fa..1ac0647c16 100644 --- a/cpp/src/wrapper/ConfAdapterMgr.h +++ b/cpp/src/wrapper/ConfAdapterMgr.h @@ -15,12 +15,13 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include "VecIndex.h" #include "ConfAdapter.h" +#include "VecIndex.h" +#include +#include namespace zilliz { namespace milvus { @@ -28,23 +29,21 @@ namespace engine { class AdapterMgr { public: - template + template struct register_t { - explicit register_t(const IndexType &key) { - AdapterMgr::GetInstance().table_.emplace(key, [] { - return std::make_shared(); - }); + explicit register_t(const IndexType& key) { + AdapterMgr::GetInstance().table_.emplace(key, [] { return std::make_shared(); }); } }; - static AdapterMgr & + static AdapterMgr& GetInstance() { static AdapterMgr instance; return instance; } ConfAdapterPtr - GetAdapter(const IndexType &indexType); + GetAdapter(const IndexType& indexType); void RegisterAdapter(); @@ -54,10 +53,6 @@ class AdapterMgr { std::map > table_; }; - -} // engine -} // milvus -} // zilliz - - - +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/DataTransfer.cpp b/cpp/src/wrapper/DataTransfer.cpp index c6603b1991..ac771c1895 100644 --- a/cpp/src/wrapper/DataTransfer.cpp +++ b/cpp/src/wrapper/DataTransfer.cpp @@ -15,39 +15,38 @@ // specific language governing permissions and limitations // under the License. - #include "wrapper/DataTransfer.h" -#include #include #include +#include namespace zilliz { namespace milvus { namespace engine { knowhere::DatasetPtr -GenDatasetWithIds(const int64_t &nb, const int64_t &dim, const float *xb, const int64_t *ids) { +GenDatasetWithIds(const int64_t& nb, const int64_t& dim, const float* xb, const int64_t* ids) { std::vector shape{nb, dim}; - auto tensor = knowhere::ConstructFloatTensor((uint8_t *) xb, nb * dim * sizeof(float), shape); + auto tensor = knowhere::ConstructFloatTensor((uint8_t*)xb, nb * dim * sizeof(float), shape); std::vector tensors{tensor}; std::vector tensor_fields{knowhere::ConstructFloatField("data")}; auto tensor_schema = std::make_shared(tensor_fields); - auto id_array = knowhere::ConstructInt64Array((uint8_t *) ids, nb * sizeof(int64_t)); + auto id_array = knowhere::ConstructInt64Array((uint8_t*)ids, nb * sizeof(int64_t)); std::vector arrays{id_array}; std::vector array_fields{knowhere::ConstructInt64Field("id")}; auto array_schema = std::make_shared(tensor_fields); - auto dataset = std::make_shared(std::move(arrays), array_schema, - std::move(tensors), tensor_schema); + auto dataset = + std::make_shared(std::move(arrays), array_schema, std::move(tensors), tensor_schema); return dataset; } knowhere::DatasetPtr -GenDataset(const int64_t &nb, const int64_t &dim, const float *xb) { +GenDataset(const int64_t& nb, const int64_t& dim, const float* xb) { std::vector shape{nb, dim}; - auto tensor = knowhere::ConstructFloatTensor((uint8_t *) xb, nb * dim * sizeof(float), shape); + auto tensor = knowhere::ConstructFloatTensor((uint8_t*)xb, nb * dim * sizeof(float), shape); std::vector tensors{tensor}; std::vector tensor_fields{knowhere::ConstructFloatField("data")}; auto tensor_schema = std::make_shared(tensor_fields); @@ -56,6 +55,6 @@ GenDataset(const int64_t &nb, const int64_t &dim, const float *xb) { return dataset; } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/DataTransfer.h b/cpp/src/wrapper/DataTransfer.h index 070adca055..ad5361c4f5 100644 --- a/cpp/src/wrapper/DataTransfer.h +++ b/cpp/src/wrapper/DataTransfer.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "knowhere/adapter/Structure.h" @@ -25,11 +24,11 @@ namespace milvus { namespace engine { extern zilliz::knowhere::DatasetPtr -GenDatasetWithIds(const int64_t &nb, const int64_t &dim, const float *xb, const int64_t *ids); +GenDatasetWithIds(const int64_t& nb, const int64_t& dim, const float* xb, const int64_t* ids); extern zilliz::knowhere::DatasetPtr -GenDataset(const int64_t &nb, const int64_t &dim, const float *xb); +GenDataset(const int64_t& nb, const int64_t& dim, const float* xb); -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/KnowhereResource.cpp b/cpp/src/wrapper/KnowhereResource.cpp index b46acb1a4a..7194ba2374 100644 --- a/cpp/src/wrapper/KnowhereResource.cpp +++ b/cpp/src/wrapper/KnowhereResource.cpp @@ -15,16 +15,15 @@ // specific language governing permissions and limitations // under the License. - #include "wrapper/KnowhereResource.h" #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #include "server/Config.h" #include #include -#include #include #include +#include namespace zilliz { namespace milvus { @@ -43,22 +42,24 @@ KnowhereResource::Initialize() { GpuResourcesArray gpu_resources; Status s; - //get build index gpu resource - server::Config &config = server::Config::GetInstance(); + // get build index gpu resource + server::Config& config = server::Config::GetInstance(); int32_t build_index_gpu; s = config.GetDBConfigBuildIndexGPU(build_index_gpu); - if (!s.ok()) return s; + if (!s.ok()) + return s; gpu_resources.insert(std::make_pair(build_index_gpu, GpuResourceSetting())); - //get search gpu resource + // get search gpu resource std::vector pool; s = config.GetResourceConfigPool(pool); - if (!s.ok()) return s; + if (!s.ok()) + return s; std::set gpu_ids; - for (auto &resource : pool) { + for (auto& resource : pool) { if (resource.length() < 4 || resource.substr(0, 3) != "gpu") { // invalid continue; @@ -67,12 +68,10 @@ KnowhereResource::Initialize() { gpu_resources.insert(std::make_pair(gpu_id, GpuResourceSetting())); } - //init gpu resources + // init gpu resources for (auto iter = gpu_resources.begin(); iter != gpu_resources.end(); ++iter) { - knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(iter->first, - iter->second.pinned_memory, - iter->second.temp_memory, - iter->second.resource_num); + knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(iter->first, iter->second.pinned_memory, + iter->second.temp_memory, iter->second.resource_num); } return Status::OK(); @@ -80,10 +79,10 @@ KnowhereResource::Initialize() { Status KnowhereResource::Finalize() { - knowhere::FaissGpuResourceMgr::GetInstance().Free(); // free gpu resource. + knowhere::FaissGpuResourceMgr::GetInstance().Free(); // free gpu resource. return Status::OK(); } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/KnowhereResource.h b/cpp/src/wrapper/KnowhereResource.h index a8726f9542..144d632c07 100644 --- a/cpp/src/wrapper/KnowhereResource.h +++ b/cpp/src/wrapper/KnowhereResource.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "utils/Status.h" @@ -33,6 +32,6 @@ class KnowhereResource { Finalize(); }; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/VecImpl.cpp b/cpp/src/wrapper/VecImpl.cpp index 65d4c60796..0e90ab135e 100644 --- a/cpp/src/wrapper/VecImpl.cpp +++ b/cpp/src/wrapper/VecImpl.cpp @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. - #include "wrapper/VecImpl.h" -#include "utils/Log.h" -#include "knowhere/index/vector_index/IndexIDMAP.h" -#include "knowhere/index/vector_index/IndexGPUIVF.h" -#include "knowhere/common/Exception.h" -#include "knowhere/index/vector_index/helpers/Cloner.h" #include "DataTransfer.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexIDMAP.h" +#include "knowhere/index/vector_index/helpers/Cloner.h" +#include "utils/Log.h" /* * no parameter check in this layer. @@ -34,12 +33,8 @@ namespace milvus { namespace engine { Status -VecIndexImpl::BuildAll(const int64_t &nb, - const float *xb, - const int64_t *ids, - const Config &cfg, - const int64_t &nt, - const float *xt) { +VecIndexImpl::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt, + const float* xt) { try { dim = cfg->d; auto dataset = GenDatasetWithIds(nb, dim, xb, ids); @@ -49,10 +44,10 @@ VecIndexImpl::BuildAll(const int64_t &nb, auto model = index_->Train(dataset, cfg); index_->set_index_model(model); index_->Add(dataset, cfg); - } catch (knowhere::KnowhereException &e) { + } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (std::exception &e) { + } catch (std::exception& e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_ERROR, e.what()); } @@ -60,15 +55,15 @@ VecIndexImpl::BuildAll(const int64_t &nb, } Status -VecIndexImpl::Add(const int64_t &nb, const float *xb, const int64_t *ids, const Config &cfg) { +VecIndexImpl::Add(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg) { try { auto dataset = GenDatasetWithIds(nb, dim, xb, ids); index_->Add(dataset, cfg); - } catch (knowhere::KnowhereException &e) { + } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (std::exception &e) { + } catch (std::exception& e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_ERROR, e.what()); } @@ -76,7 +71,7 @@ VecIndexImpl::Add(const int64_t &nb, const float *xb, const int64_t *ids, const } Status -VecIndexImpl::Search(const int64_t &nq, const float *xq, float *dist, int64_t *ids, const Config &cfg) { +VecIndexImpl::Search(const int64_t& nq, const float* xq, float* dist, int64_t* ids, const Config& cfg) { try { auto k = cfg->k; auto dataset = GenDataset(nq, dim, xq); @@ -110,10 +105,10 @@ VecIndexImpl::Search(const int64_t &nq, const float *xq, float *dist, int64_t *i // TODO(linxj): avoid copy here. memcpy(ids, p_ids, sizeof(int64_t) * nq * k); memcpy(dist, p_dist, sizeof(float) * nq * k); - } catch (knowhere::KnowhereException &e) { + } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (std::exception &e) { + } catch (std::exception& e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_ERROR, e.what()); } @@ -127,7 +122,7 @@ VecIndexImpl::Serialize() { } Status -VecIndexImpl::Load(const zilliz::knowhere::BinarySet &index_binary) { +VecIndexImpl::Load(const zilliz::knowhere::BinarySet& index_binary) { index_->Load(index_binary); dim = Dimension(); return Status::OK(); @@ -149,7 +144,7 @@ VecIndexImpl::GetType() { } VecIndexPtr -VecIndexImpl::CopyToGpu(const int64_t &device_id, const Config &cfg) { +VecIndexImpl::CopyToGpu(const int64_t& device_id, const Config& cfg) { // TODO(linxj): exception handle auto gpu_index = zilliz::knowhere::cloner::CopyCpuToGpu(index_, device_id, cfg); auto new_index = std::make_shared(gpu_index, ConvertToGpuIndexType(type)); @@ -158,7 +153,7 @@ VecIndexImpl::CopyToGpu(const int64_t &device_id, const Config &cfg) { } VecIndexPtr -VecIndexImpl::CopyToCpu(const Config &cfg) { +VecIndexImpl::CopyToCpu(const Config& cfg) { // TODO(linxj): exception handle auto cpu_index = zilliz::knowhere::cloner::CopyGpuToCpu(index_, cfg); auto new_index = std::make_shared(cpu_index, ConvertToCpuIndexType(type)); @@ -180,30 +175,32 @@ VecIndexImpl::GetDeviceId() { return device_idx->GetGpuDevice(); } // else - return -1; // -1 == cpu + return -1; // -1 == cpu } -float * +float* BFIndex::GetRawVectors() { auto raw_index = std::dynamic_pointer_cast(index_); - if (raw_index) { return raw_index->GetRawVectors(); } + if (raw_index) { + return raw_index->GetRawVectors(); + } return nullptr; } -int64_t * +int64_t* BFIndex::GetRawIds() { return std::static_pointer_cast(index_)->GetRawIds(); } ErrorCode -BFIndex::Build(const Config &cfg) { +BFIndex::Build(const Config& cfg) { try { dim = cfg->d; std::static_pointer_cast(index_)->Train(cfg); - } catch (knowhere::KnowhereException &e) { + } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); return KNOWHERE_UNEXPECTED_ERROR; - } catch (std::exception &e) { + } catch (std::exception& e) { WRAPPER_LOG_ERROR << e.what(); return KNOWHERE_ERROR; } @@ -211,22 +208,18 @@ BFIndex::Build(const Config &cfg) { } Status -BFIndex::BuildAll(const int64_t &nb, - const float *xb, - const int64_t *ids, - const Config &cfg, - const int64_t &nt, - const float *xt) { +BFIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt, + const float* xt) { try { dim = cfg->d; auto dataset = GenDatasetWithIds(nb, dim, xb, ids); std::static_pointer_cast(index_)->Train(cfg); index_->Add(dataset, cfg); - } catch (knowhere::KnowhereException &e) { + } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (std::exception &e) { + } catch (std::exception& e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_ERROR, e.what()); } @@ -235,12 +228,8 @@ BFIndex::BuildAll(const int64_t &nb, // TODO(linxj): add lock here. Status -IVFMixIndex::BuildAll(const int64_t &nb, - const float *xb, - const int64_t *ids, - const Config &cfg, - const int64_t &nt, - const float *xt) { +IVFMixIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt, + const float* xt) { try { dim = cfg->d; auto dataset = GenDatasetWithIds(nb, dim, xb, ids); @@ -259,10 +248,10 @@ IVFMixIndex::BuildAll(const int64_t &nb, WRAPPER_LOG_ERROR << "Build IVFMIXIndex Failed"; return Status(KNOWHERE_ERROR, "Build IVFMIXIndex Failed"); } - } catch (knowhere::KnowhereException &e) { + } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (std::exception &e) { + } catch (std::exception& e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_ERROR, e.what()); } @@ -270,12 +259,12 @@ IVFMixIndex::BuildAll(const int64_t &nb, } Status -IVFMixIndex::Load(const zilliz::knowhere::BinarySet &index_binary) { +IVFMixIndex::Load(const zilliz::knowhere::BinarySet& index_binary) { index_->Load(index_binary); dim = Dimension(); return Status::OK(); } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/VecImpl.h b/cpp/src/wrapper/VecImpl.h index a91b24562b..fe015424f6 100644 --- a/cpp/src/wrapper/VecImpl.h +++ b/cpp/src/wrapper/VecImpl.h @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include "knowhere/index/vector_index/VectorIndex.h" #include "VecIndex.h" +#include "knowhere/index/vector_index/VectorIndex.h" -#include #include +#include namespace zilliz { namespace milvus { @@ -30,23 +29,19 @@ namespace engine { class VecIndexImpl : public VecIndex { public: - explicit VecIndexImpl(std::shared_ptr index, const IndexType &type) + explicit VecIndexImpl(std::shared_ptr index, const IndexType& type) : index_(std::move(index)), type(type) { } Status - BuildAll(const int64_t &nb, - const float *xb, - const int64_t *ids, - const Config &cfg, - const int64_t &nt, - const float *xt) override; + BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt, + const float* xt) override; VecIndexPtr - CopyToGpu(const int64_t &device_id, const Config &cfg) override; + CopyToGpu(const int64_t& device_id, const Config& cfg) override; VecIndexPtr - CopyToCpu(const Config &cfg) override; + CopyToCpu(const Config& cfg) override; IndexType GetType() override; @@ -58,13 +53,13 @@ class VecIndexImpl : public VecIndex { Count() override; Status - Add(const int64_t &nb, const float *xb, const int64_t *ids, const Config &cfg) override; + Add(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg) override; zilliz::knowhere::BinarySet Serialize() override; Status - Load(const zilliz::knowhere::BinarySet &index_binary) override; + Load(const zilliz::knowhere::BinarySet& index_binary) override; VecIndexPtr Clone() override; @@ -73,7 +68,7 @@ class VecIndexImpl : public VecIndex { GetDeviceId() override; Status - Search(const int64_t &nq, const float *xq, float *dist, int64_t *ids, const Config &cfg) override; + Search(const int64_t& nq, const float* xq, float* dist, int64_t* ids, const Config& cfg) override; protected: int64_t dim = 0; @@ -85,46 +80,38 @@ class VecIndexImpl : public VecIndex { class IVFMixIndex : public VecIndexImpl { public: - explicit IVFMixIndex(std::shared_ptr index, const IndexType &type) + explicit IVFMixIndex(std::shared_ptr index, const IndexType& type) : VecIndexImpl(std::move(index), type) { } Status - BuildAll(const int64_t &nb, - const float *xb, - const int64_t *ids, - const Config &cfg, - const int64_t &nt, - const float *xt) override; + BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt, + const float* xt) override; Status - Load(const zilliz::knowhere::BinarySet &index_binary) override; + Load(const zilliz::knowhere::BinarySet& index_binary) override; }; class BFIndex : public VecIndexImpl { public: - explicit BFIndex(std::shared_ptr index) : VecIndexImpl(std::move(index), - IndexType::FAISS_IDMAP) { + explicit BFIndex(std::shared_ptr index) + : VecIndexImpl(std::move(index), IndexType::FAISS_IDMAP) { } ErrorCode - Build(const Config &cfg); + Build(const Config& cfg); - float * + float* GetRawVectors(); Status - BuildAll(const int64_t &nb, - const float *xb, - const int64_t *ids, - const Config &cfg, - const int64_t &nt, - const float *xt) override; + BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt, + const float* xt) override; - int64_t * + int64_t* GetRawIds(); }; -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/VecIndex.cpp b/cpp/src/wrapper/VecIndex.cpp index ffc14ebd87..c5ebd684fa 100644 --- a/cpp/src/wrapper/VecIndex.cpp +++ b/cpp/src/wrapper/VecIndex.cpp @@ -16,17 +16,17 @@ // under the License. #include "wrapper/VecIndex.h" -#include "knowhere/index/vector_index/IndexIVF.h" +#include "VecImpl.h" +#include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" -#include "knowhere/index/vector_index/IndexIVFSQ.h" -#include "knowhere/index/vector_index/IndexGPUIVFSQ.h" -#include "knowhere/index/vector_index/IndexIVFPQ.h" #include "knowhere/index/vector_index/IndexGPUIVFPQ.h" +#include "knowhere/index/vector_index/IndexGPUIVFSQ.h" #include "knowhere/index/vector_index/IndexIDMAP.h" +#include "knowhere/index/vector_index/IndexIVF.h" +#include "knowhere/index/vector_index/IndexIVFPQ.h" +#include "knowhere/index/vector_index/IndexIVFSQ.h" #include "knowhere/index/vector_index/IndexKDT.h" #include "knowhere/index/vector_index/IndexNSG.h" -#include "knowhere/common/Exception.h" -#include "VecImpl.h" #include "utils/Log.h" #include @@ -39,18 +39,18 @@ struct FileIOReader { std::fstream fs; std::string name; - explicit FileIOReader(const std::string &fname); + explicit FileIOReader(const std::string& fname); ~FileIOReader(); size_t - operator()(void *ptr, size_t size); + operator()(void* ptr, size_t size); size_t - operator()(void *ptr, size_t size, size_t pos); + operator()(void* ptr, size_t size, size_t pos); }; -FileIOReader::FileIOReader(const std::string &fname) { +FileIOReader::FileIOReader(const std::string& fname) { name = fname; fs = std::fstream(name, std::ios::in | std::ios::binary); } @@ -60,12 +60,12 @@ FileIOReader::~FileIOReader() { } size_t -FileIOReader::operator()(void *ptr, size_t size) { - fs.read(reinterpret_cast(ptr), size); +FileIOReader::operator()(void* ptr, size_t size) { + fs.read(reinterpret_cast(ptr), size); } size_t -FileIOReader::operator()(void *ptr, size_t size, size_t pos) { +FileIOReader::operator()(void* ptr, size_t size, size_t pos) { return 0; } @@ -73,12 +73,13 @@ struct FileIOWriter { std::fstream fs; std::string name; - explicit FileIOWriter(const std::string &fname); + explicit FileIOWriter(const std::string& fname); ~FileIOWriter(); - size_t operator()(void *ptr, size_t size); + size_t + operator()(void* ptr, size_t size); }; -FileIOWriter::FileIOWriter(const std::string &fname) { +FileIOWriter::FileIOWriter(const std::string& fname) { name = fname; fs = std::fstream(name, std::ios::out | std::ios::binary); } @@ -88,14 +89,14 @@ FileIOWriter::~FileIOWriter() { } size_t -FileIOWriter::operator()(void *ptr, size_t size) { - fs.write(reinterpret_cast(ptr), size); +FileIOWriter::operator()(void* ptr, size_t size) { + fs.write(reinterpret_cast(ptr), size); } VecIndexPtr -GetVecIndexFactory(const IndexType &type, const Config &cfg) { +GetVecIndexFactory(const IndexType& type, const Config& cfg) { std::shared_ptr index; - auto gpu_device = -1; // TODO(linxj): remove hardcode here + auto gpu_device = -1; // TODO(linxj): remove hardcode here switch (type) { case IndexType::FAISS_IDMAP: { index = std::make_shared(); @@ -141,22 +142,20 @@ GetVecIndexFactory(const IndexType &type, const Config &cfg) { index = std::make_shared(gpu_device); break; } - default: { - return nullptr; - } + default: { return nullptr; } } return std::make_shared(index, type); } VecIndexPtr -LoadVecIndex(const IndexType &index_type, const zilliz::knowhere::BinarySet &index_binary) { +LoadVecIndex(const IndexType& index_type, const zilliz::knowhere::BinarySet& index_binary) { auto index = GetVecIndexFactory(index_type); index->Load(index_binary); return index; } VecIndexPtr -read_index(const std::string &location) { +read_index(const std::string& location) { knowhere::BinarySet load_data_list; FileIOReader reader(location); reader.fs.seekg(0, reader.fs.end); @@ -201,28 +200,28 @@ read_index(const std::string &location) { } Status -write_index(VecIndexPtr index, const std::string &location) { +write_index(VecIndexPtr index, const std::string& location) { try { auto binaryset = index->Serialize(); auto index_type = index->GetType(); FileIOWriter writer(location); writer(&index_type, sizeof(IndexType)); - for (auto &iter : binaryset.binary_map_) { + for (auto& iter : binaryset.binary_map_) { auto meta = iter.first.c_str(); size_t meta_length = iter.first.length(); writer(&meta_length, sizeof(meta_length)); - writer((void *) meta, meta_length); + writer((void*)meta, meta_length); auto binary = iter.second; int64_t binary_length = binary->size; writer(&binary_length, sizeof(binary_length)); - writer((void *) binary->data.get(), binary_length); + writer((void*)binary->data.get(), binary_length); } - } catch (knowhere::KnowhereException &e) { + } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (std::exception &e) { + } catch (std::exception& e) { WRAPPER_LOG_ERROR << e.what(); std::string estring(e.what()); if (estring.find("No space left on device") != estring.npos) { @@ -236,7 +235,7 @@ write_index(VecIndexPtr index, const std::string &location) { } IndexType -ConvertToCpuIndexType(const IndexType &type) { +ConvertToCpuIndexType(const IndexType& type) { // TODO(linxj): add IDMAP switch (type) { case IndexType::FAISS_IVFFLAT_GPU: @@ -247,14 +246,12 @@ ConvertToCpuIndexType(const IndexType &type) { case IndexType::FAISS_IVFSQ8_MIX: { return IndexType::FAISS_IVFSQ8_CPU; } - default: { - return type; - } + default: { return type; } } } IndexType -ConvertToGpuIndexType(const IndexType &type) { +ConvertToGpuIndexType(const IndexType& type) { switch (type) { case IndexType::FAISS_IVFFLAT_MIX: case IndexType::FAISS_IVFFLAT_CPU: { @@ -264,12 +261,10 @@ ConvertToGpuIndexType(const IndexType &type) { case IndexType::FAISS_IVFSQ8_CPU: { return IndexType::FAISS_IVFSQ8_GPU; } - default: { - return type; - } + default: { return type; } } } -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/wrapper/VecIndex.h b/cpp/src/wrapper/VecIndex.h index e4fe387260..4bc51021ce 100644 --- a/cpp/src/wrapper/VecIndex.h +++ b/cpp/src/wrapper/VecIndex.h @@ -15,15 +15,14 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include #include +#include -#include "utils/Status.h" -#include "knowhere/common/Config.h" #include "knowhere/common/BinarySet.h" +#include "knowhere/common/Config.h" +#include "utils/Status.h" namespace zilliz { namespace milvus { @@ -36,7 +35,7 @@ enum class IndexType { FAISS_IDMAP = 1, FAISS_IVFFLAT_CPU, FAISS_IVFFLAT_GPU, - FAISS_IVFFLAT_MIX, // build on gpu and search on cpu + FAISS_IVFFLAT_MIX, // build on gpu and search on cpu FAISS_IVFPQ_CPU, FAISS_IVFPQ_GPU, SPTAG_KDT_RNT_CPU, @@ -53,32 +52,20 @@ using VecIndexPtr = std::shared_ptr; class VecIndex { public: virtual Status - BuildAll(const int64_t &nb, - const float *xb, - const int64_t *ids, - const Config &cfg, - const int64_t &nt = 0, - const float *xt = nullptr) = 0; + BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt = 0, + const float* xt = nullptr) = 0; virtual Status - Add(const int64_t &nb, - const float *xb, - const int64_t *ids, - const Config &cfg = Config()) = 0; + Add(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg = Config()) = 0; virtual Status - Search(const int64_t &nq, - const float *xq, - float *dist, - int64_t *ids, - const Config &cfg = Config()) = 0; + Search(const int64_t& nq, const float* xq, float* dist, int64_t* ids, const Config& cfg = Config()) = 0; virtual VecIndexPtr - CopyToGpu(const int64_t &device_id, - const Config &cfg = Config()) = 0; + CopyToGpu(const int64_t& device_id, const Config& cfg = Config()) = 0; virtual VecIndexPtr - CopyToCpu(const Config &cfg = Config()) = 0; + CopyToCpu(const Config& cfg = Config()) = 0; virtual VecIndexPtr Clone() = 0; @@ -99,27 +86,27 @@ class VecIndex { Serialize() = 0; virtual Status - Load(const zilliz::knowhere::BinarySet &index_binary) = 0; + Load(const zilliz::knowhere::BinarySet& index_binary) = 0; }; extern Status -write_index(VecIndexPtr index, const std::string &location); +write_index(VecIndexPtr index, const std::string& location); extern VecIndexPtr -read_index(const std::string &location); +read_index(const std::string& location); extern VecIndexPtr -GetVecIndexFactory(const IndexType &type, const Config &cfg = Config()); +GetVecIndexFactory(const IndexType& type, const Config& cfg = Config()); extern VecIndexPtr -LoadVecIndex(const IndexType &index_type, const zilliz::knowhere::BinarySet &index_binary); +LoadVecIndex(const IndexType& index_type, const zilliz::knowhere::BinarySet& index_binary); extern IndexType -ConvertToCpuIndexType(const IndexType &type); +ConvertToCpuIndexType(const IndexType& type); extern IndexType -ConvertToGpuIndexType(const IndexType &type); +ConvertToGpuIndexType(const IndexType& type); -} // namespace engine -} // namespace milvus -} // namespace zilliz +} // namespace engine +} // namespace milvus +} // namespace zilliz From c54a787ae19abf9c6184bb89ab5f147b17ef8a4a Mon Sep 17 00:00:00 2001 From: starlord Date: Sat, 28 Sep 2019 19:01:11 +0800 Subject: [PATCH 145/217] format knowhere code Former-commit-id: eb06e117c192b568d9da3a33d94faa127e7cf0b7 --- cpp/build-support/lint_exclusions.txt | 3 +- .../knowhere/adapter/ArrowAdapter.cpp | 27 +- .../knowhere/knowhere/adapter/ArrowAdapter.h | 12 +- .../knowhere/adapter/SptagAdapter.cpp | 59 ++- .../knowhere/knowhere/adapter/SptagAdapter.h | 15 +- .../knowhere/knowhere/adapter/Structure.cpp | 27 +- .../knowhere/knowhere/adapter/Structure.h | 26 +- .../knowhere/knowhere/adapter/VectorAdapter.h | 16 +- cpp/src/core/knowhere/knowhere/common/Array.h | 12 +- .../core/knowhere/knowhere/common/BinarySet.h | 27 +- .../core/knowhere/knowhere/common/Buffer.h | 21 +- .../core/knowhere/knowhere/common/Config.h | 17 +- .../core/knowhere/knowhere/common/Dataset.h | 66 ++-- .../knowhere/knowhere/common/Exception.cpp | 30 +- .../core/knowhere/knowhere/common/Exception.h | 43 +-- cpp/src/core/knowhere/knowhere/common/Id.h | 25 +- cpp/src/core/knowhere/knowhere/common/Log.h | 5 +- .../core/knowhere/knowhere/common/Schema.h | 9 +- .../core/knowhere/knowhere/common/Tensor.h | 8 +- .../core/knowhere/knowhere/common/Timer.cpp | 71 ++-- cpp/src/core/knowhere/knowhere/common/Timer.h | 24 +- cpp/src/core/knowhere/knowhere/index/Index.h | 31 +- .../core/knowhere/knowhere/index/IndexModel.h | 10 +- .../core/knowhere/knowhere/index/IndexType.h | 8 +- .../knowhere/index/preprocessor/Normalize.cpp | 31 +- .../knowhere/index/preprocessor/Normalize.h | 25 +- .../index/preprocessor/Preprocessor.h | 11 +- .../index/vector_index/FaissBaseIndex.cpp | 42 ++- .../index/vector_index/FaissBaseIndex.h | 11 +- .../index/vector_index/IndexGPUIVF.cpp | 90 ++--- .../knowhere/index/vector_index/IndexGPUIVF.h | 98 ++--- .../index/vector_index/IndexGPUIVFPQ.cpp | 38 +- .../index/vector_index/IndexGPUIVFPQ.h | 25 +- .../index/vector_index/IndexGPUIVFSQ.cpp | 86 ++--- .../index/vector_index/IndexGPUIVFSQ.h | 24 +- .../index/vector_index/IndexIDMAP.cpp | 137 +++---- .../knowhere/index/vector_index/IndexIDMAP.h | 95 +++-- .../knowhere/index/vector_index/IndexIVF.cpp | 132 +++---- .../knowhere/index/vector_index/IndexIVF.h | 50 ++- .../index/vector_index/IndexIVFPQ.cpp | 36 +- .../knowhere/index/vector_index/IndexIVFPQ.h | 24 +- .../index/vector_index/IndexIVFSQ.cpp | 36 +- .../knowhere/index/vector_index/IndexIVFSQ.h | 22 +- .../knowhere/index/vector_index/IndexKDT.cpp | 75 ++-- .../knowhere/index/vector_index/IndexKDT.h | 41 ++- .../knowhere/index/vector_index/IndexNSG.cpp | 73 ++-- .../knowhere/index/vector_index/IndexNSG.h | 41 ++- .../knowhere/index/vector_index/VectorIndex.h | 22 +- .../index/vector_index/helpers/Cloner.cpp | 22 +- .../index/vector_index/helpers/Cloner.h | 12 +- .../index/vector_index/helpers/Definitions.h | 8 +- .../helpers/FaissGpuResourceMgr.cpp | 58 +-- .../helpers/FaissGpuResourceMgr.h | 58 +-- .../index/vector_index/helpers/FaissIO.cpp | 32 +- .../index/vector_index/helpers/FaissIO.h | 16 +- .../vector_index/helpers/IndexParameter.cpp | 11 +- .../vector_index/helpers/IndexParameter.h | 58 +-- .../vector_index/helpers/KDTParameterMgr.cpp | 10 +- .../vector_index/helpers/KDTParameterMgr.h | 18 +- .../knowhere/index/vector_index/nsg/NSG.cpp | 341 ++++++++++-------- .../knowhere/index/vector_index/nsg/NSG.h | 117 +++--- .../index/vector_index/nsg/NSGHelper.cpp | 117 +++--- .../index/vector_index/nsg/NSGHelper.h | 14 +- .../knowhere/index/vector_index/nsg/NSGIO.cpp | 24 +- .../knowhere/index/vector_index/nsg/NSGIO.h | 16 +- .../index/vector_index/nsg/Neighbor.h | 23 +- cpp/src/core/test/CMakeLists.txt | 1 + cpp/src/core/test/SPTAG.cpp | 86 +++-- .../core/test/faiss_ori/gpuresource_test.cpp | 104 +++--- cpp/src/core/test/kdtree.cpp | 283 ++++++++------- cpp/src/core/test/test_idmap.cpp | 76 ++-- cpp/src/core/test/test_ivf.cpp | 311 ++++++++-------- cpp/src/core/test/test_json.cpp | 12 +- cpp/src/core/test/test_kdt.cpp | 57 ++- cpp/src/core/test/test_nsg.cpp | 51 ++- cpp/src/core/test/test_nsg/test_nsg.cpp | 75 ++-- cpp/src/core/test/utils.cpp | 106 +++--- cpp/src/core/test/utils.h | 59 ++- 78 files changed, 2057 insertions(+), 1976 deletions(-) diff --git a/cpp/build-support/lint_exclusions.txt b/cpp/build-support/lint_exclusions.txt index b9c1eaa760..6888dfa32c 100644 --- a/cpp/build-support/lint_exclusions.txt +++ b/cpp/build-support/lint_exclusions.txt @@ -5,5 +5,4 @@ *thirdparty* *easylogging++* *SqliteMetaImpl.cpp -*src/grpc* -*src/core* \ No newline at end of file +*src/grpc* \ No newline at end of file diff --git a/cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.cpp b/cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.cpp index c07cb16fca..bc1dfbf13d 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.cpp @@ -15,42 +15,41 @@ // specific language governing permissions and limitations // under the License. - -#include "ArrowAdapter.h" +#include "knowhere/adapter/ArrowAdapter.h" namespace zilliz { namespace knowhere { ArrayPtr -CopyArray(const ArrayPtr &origin) { +CopyArray(const ArrayPtr& origin) { ArrayPtr copy = nullptr; auto copy_data = origin->data()->Copy(); switch (origin->type_id()) { -#define DEFINE_TYPE(type, clazz) \ - case arrow::Type::type: { \ - copy = std::make_shared(copy_data); \ - } +#define DEFINE_TYPE(type, clazz) \ + case arrow::Type::type: { \ + copy = std::make_shared(copy_data); \ + } DEFINE_TYPE(BOOL, BooleanArray) DEFINE_TYPE(BINARY, BinaryArray) DEFINE_TYPE(FIXED_SIZE_BINARY, FixedSizeBinaryArray) DEFINE_TYPE(DECIMAL, Decimal128Array) DEFINE_TYPE(FLOAT, NumericArray) DEFINE_TYPE(INT64, NumericArray) - default:break; + default: + break; } return copy; } SchemaPtr -CopySchema(const SchemaPtr &origin) { +CopySchema(const SchemaPtr& origin) { std::vector> fields; - for (auto &field : origin->fields()) { - auto copy = std::make_shared(field->name(), field->type(),field->nullable(), nullptr); + for (auto& field : origin->fields()) { + auto copy = std::make_shared(field->name(), field->type(), field->nullable(), nullptr); fields.emplace_back(copy); } return std::make_shared(std::move(fields)); } - -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.h b/cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.h index d19f8f3ae5..f4d0b348cf 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.h +++ b/cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.h @@ -15,22 +15,22 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include +#include +#include #include "knowhere/common/Array.h" - namespace zilliz { namespace knowhere { ArrayPtr -CopyArray(const ArrayPtr &origin); +CopyArray(const ArrayPtr& origin); SchemaPtr -CopySchema(const SchemaPtr &origin); +CopySchema(const SchemaPtr& origin); -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp index 056f4e2cad..8a0ad69591 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp @@ -15,36 +15,31 @@ // specific language governing permissions and limitations // under the License. +#include "knowhere/adapter/SptagAdapter.h" +#include "knowhere/adapter/Structure.h" #include "knowhere/index/vector_index/helpers/Definitions.h" -#include "SptagAdapter.h" -#include "Structure.h" - namespace zilliz { namespace knowhere { - std::shared_ptr -ConvertToMetadataSet(const DatasetPtr &dataset) { +ConvertToMetadataSet(const DatasetPtr& dataset) { auto array = dataset->array()[0]; auto elems = array->length(); auto p_data = array->data()->GetValues(1, 0); - auto p_offset = (int64_t *) malloc(sizeof(int64_t) * elems); - for (auto i = 0; i <= elems; ++i) - p_offset[i] = i * 8; - - std::shared_ptr metaset(new SPTAG::MemMetadataSet( - SPTAG::ByteArray((std::uint8_t *) p_data, elems * sizeof(int64_t), false), - SPTAG::ByteArray((std::uint8_t *) p_offset, elems * sizeof(int64_t), true), - elems)); + auto p_offset = (int64_t*)malloc(sizeof(int64_t) * elems); + for (auto i = 0; i <= elems; ++i) p_offset[i] = i * 8; + std::shared_ptr metaset( + new SPTAG::MemMetadataSet(SPTAG::ByteArray((std::uint8_t*)p_data, elems * sizeof(int64_t), false), + SPTAG::ByteArray((std::uint8_t*)p_offset, elems * sizeof(int64_t), true), elems)); return metaset; } std::shared_ptr -ConvertToVectorSet(const DatasetPtr &dataset) { +ConvertToVectorSet(const DatasetPtr& dataset) { auto tensor = dataset->tensor()[0]; auto p_data = tensor->raw_mutable_data(); @@ -54,18 +49,16 @@ ConvertToVectorSet(const DatasetPtr &dataset) { SPTAG::ByteArray byte_array(p_data, num_bytes, false); - auto vectorset = std::make_shared(byte_array, - SPTAG::VectorValueType::Float, - dimension, - rows); + auto vectorset = + std::make_shared(byte_array, SPTAG::VectorValueType::Float, dimension, rows); return vectorset; } std::vector -ConvertToQueryResult(const DatasetPtr &dataset, const Config &config) { +ConvertToQueryResult(const DatasetPtr& dataset, const Config& config) { auto tensor = dataset->tensor()[0]; - auto p_data = (float *) tensor->raw_mutable_data(); + auto p_data = (float*)tensor->raw_mutable_data(); auto dimension = tensor->shape()[1]; auto rows = tensor->shape()[0]; @@ -82,23 +75,23 @@ ConvertToDataset(std::vector query_results) { auto k = query_results[0].GetResultNum(); auto elems = query_results.size() * k; - auto p_id = (int64_t *) malloc(sizeof(int64_t) * elems); - auto p_dist = (float *) malloc(sizeof(float) * elems); - // TODO: throw if malloc failed. + auto p_id = (int64_t*)malloc(sizeof(int64_t) * elems); + auto p_dist = (float*)malloc(sizeof(float) * elems); +// TODO: throw if malloc failed. #pragma omp parallel for for (auto i = 0; i < query_results.size(); ++i) { auto results = query_results[i].GetResults(); auto num_result = query_results[i].GetResultNum(); for (auto j = 0; j < num_result; ++j) { -// p_id[i * k + j] = results[j].VID; - p_id[i * k + j] = *(int64_t *) query_results[i].GetMetadata(j).Data(); + // p_id[i * k + j] = results[j].VID; + p_id[i * k + j] = *(int64_t*)query_results[i].GetMetadata(j).Data(); p_dist[i * k + j] = results[j].Dist; } } - auto id_buf = MakeMutableBufferSmart((uint8_t *) p_id, sizeof(int64_t) * elems); - auto dist_buf = MakeMutableBufferSmart((uint8_t *) p_dist, sizeof(float) * elems); + auto id_buf = MakeMutableBufferSmart((uint8_t*)p_id, sizeof(int64_t) * elems); + auto dist_buf = MakeMutableBufferSmart((uint8_t*)p_dist, sizeof(float) * elems); // TODO: magic std::vector id_bufs{nullptr, id_buf}; @@ -109,11 +102,11 @@ ConvertToDataset(std::vector query_results) { auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); -// auto id_array_data = std::make_shared(int64_type, sizeof(int64_t) * elems, id_bufs); -// auto dist_array_data = std::make_shared(float_type, sizeof(float) * elems, dist_bufs); + // auto id_array_data = std::make_shared(int64_type, sizeof(int64_t) * elems, id_bufs); + // auto dist_array_data = std::make_shared(float_type, sizeof(float) * elems, dist_bufs); -// auto ids = ConstructInt64Array((uint8_t*)p_id, sizeof(int64_t) * elems); -// auto dists = ConstructFloatArray((uint8_t*)p_dist, sizeof(float) * elems); + // auto ids = ConstructInt64Array((uint8_t*)p_id, sizeof(int64_t) * elems); + // auto dists = ConstructFloatArray((uint8_t*)p_dist, sizeof(float) * elems); auto ids = std::make_shared>(id_array_data); auto dists = std::make_shared>(dist_array_data); @@ -127,5 +120,5 @@ ConvertToDataset(std::vector query_results) { return std::make_shared(array, schema); } -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.h b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.h index f47ffdc3b5..0889657cf9 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.h +++ b/cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.h @@ -15,12 +15,11 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include - #include +#include +#include #include "knowhere/common/Dataset.h" @@ -28,16 +27,16 @@ namespace zilliz { namespace knowhere { std::shared_ptr -ConvertToVectorSet(const DatasetPtr &dataset); +ConvertToVectorSet(const DatasetPtr& dataset); std::shared_ptr -ConvertToMetadataSet(const DatasetPtr &dataset); +ConvertToMetadataSet(const DatasetPtr& dataset); std::vector -ConvertToQueryResult(const DatasetPtr &dataset, const Config &config); +ConvertToQueryResult(const DatasetPtr& dataset, const Config& config); DatasetPtr ConvertToDataset(std::vector query_results); -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/adapter/Structure.cpp b/cpp/src/core/knowhere/knowhere/adapter/Structure.cpp index 18833b5d36..2cd69f3d10 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/Structure.cpp +++ b/cpp/src/core/knowhere/knowhere/adapter/Structure.cpp @@ -15,15 +15,16 @@ // specific language governing permissions and limitations // under the License. +#include "knowhere/adapter/Structure.h" -#include "Structure.h" - +#include +#include namespace zilliz { namespace knowhere { ArrayPtr -ConstructInt64ArraySmart(uint8_t *data, int64_t size) { +ConstructInt64ArraySmart(uint8_t* data, int64_t size) { // TODO: magic std::vector id_buf{nullptr, MakeMutableBufferSmart(data, size)}; auto type = std::make_shared(); @@ -32,7 +33,7 @@ ConstructInt64ArraySmart(uint8_t *data, int64_t size) { } ArrayPtr -ConstructFloatArraySmart(uint8_t *data, int64_t size) { +ConstructFloatArraySmart(uint8_t* data, int64_t size) { // TODO: magic std::vector id_buf{nullptr, MakeMutableBufferSmart(data, size)}; auto type = std::make_shared(); @@ -41,14 +42,14 @@ ConstructFloatArraySmart(uint8_t *data, int64_t size) { } TensorPtr -ConstructFloatTensorSmart(uint8_t *data, int64_t size, std::vector shape) { +ConstructFloatTensorSmart(uint8_t* data, int64_t size, std::vector shape) { auto buffer = MakeMutableBufferSmart(data, size); auto float_type = std::make_shared(); return std::make_shared(float_type, buffer, shape); } ArrayPtr -ConstructInt64Array(uint8_t *data, int64_t size) { +ConstructInt64Array(uint8_t* data, int64_t size) { // TODO: magic std::vector id_buf{nullptr, MakeMutableBuffer(data, size)}; auto type = std::make_shared(); @@ -57,7 +58,7 @@ ConstructInt64Array(uint8_t *data, int64_t size) { } ArrayPtr -ConstructFloatArray(uint8_t *data, int64_t size) { +ConstructFloatArray(uint8_t* data, int64_t size) { // TODO: magic std::vector id_buf{nullptr, MakeMutableBuffer(data, size)}; auto type = std::make_shared(); @@ -66,23 +67,23 @@ ConstructFloatArray(uint8_t *data, int64_t size) { } TensorPtr -ConstructFloatTensor(uint8_t *data, int64_t size, std::vector shape) { +ConstructFloatTensor(uint8_t* data, int64_t size, std::vector shape) { auto buffer = MakeMutableBuffer(data, size); auto float_type = std::make_shared(); return std::make_shared(float_type, buffer, shape); } FieldPtr -ConstructInt64Field(const std::string &name) { +ConstructInt64Field(const std::string& name) { auto type = std::make_shared(); return std::make_shared(name, type); } - FieldPtr -ConstructFloatField(const std::string &name) { +ConstructFloatField(const std::string& name) { auto type = std::make_shared(); return std::make_shared(name, type); } -} -} + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/adapter/Structure.h b/cpp/src/core/knowhere/knowhere/adapter/Structure.h index 7539dce2da..526dc91b73 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/Structure.h +++ b/cpp/src/core/knowhere/knowhere/adapter/Structure.h @@ -15,40 +15,40 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include -#include "knowhere/common/Dataset.h" +#include +#include +#include "knowhere/common/Dataset.h" namespace zilliz { namespace knowhere { extern ArrayPtr -ConstructInt64ArraySmart(uint8_t *data, int64_t size); +ConstructInt64ArraySmart(uint8_t* data, int64_t size); extern ArrayPtr -ConstructFloatArraySmart(uint8_t *data, int64_t size); +ConstructFloatArraySmart(uint8_t* data, int64_t size); extern TensorPtr -ConstructFloatTensorSmart(uint8_t *data, int64_t size, std::vector shape); +ConstructFloatTensorSmart(uint8_t* data, int64_t size, std::vector shape); extern ArrayPtr -ConstructInt64Array(uint8_t *data, int64_t size); +ConstructInt64Array(uint8_t* data, int64_t size); extern ArrayPtr -ConstructFloatArray(uint8_t *data, int64_t size); +ConstructFloatArray(uint8_t* data, int64_t size); extern TensorPtr -ConstructFloatTensor(uint8_t *data, int64_t size, std::vector shape); +ConstructFloatTensor(uint8_t* data, int64_t size, std::vector shape); extern FieldPtr -ConstructInt64Field(const std::string &name); +ConstructInt64Field(const std::string& name); extern FieldPtr -ConstructFloatField(const std::string &name); +ConstructFloatField(const std::string& name); - -} -} +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/adapter/VectorAdapter.h b/cpp/src/core/knowhere/knowhere/adapter/VectorAdapter.h index bdace40a74..ae4ef998cd 100644 --- a/cpp/src/core/knowhere/knowhere/adapter/VectorAdapter.h +++ b/cpp/src/core/knowhere/knowhere/adapter/VectorAdapter.h @@ -15,18 +15,16 @@ // specific language governing permissions and limitations // under the License. - #pragma once namespace zilliz { namespace knowhere { -#define GETTENSOR(dataset) \ - auto tensor = dataset->tensor()[0]; \ - auto p_data = tensor->raw_data(); \ - auto dim = tensor->shape()[1]; \ - auto rows = tensor->shape()[0]; \ +#define GETTENSOR(dataset) \ + auto tensor = dataset->tensor()[0]; \ + auto p_data = tensor->raw_data(); \ + auto dim = tensor->shape()[1]; \ + auto rows = tensor->shape()[0]; - -} -} \ No newline at end of file +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/common/Array.h b/cpp/src/core/knowhere/knowhere/common/Array.h index 94a5470029..89bbf6ccd7 100644 --- a/cpp/src/core/knowhere/knowhere/common/Array.h +++ b/cpp/src/core/knowhere/knowhere/common/Array.h @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include +#include #include "Schema.h" - namespace zilliz { namespace knowhere { @@ -35,9 +34,9 @@ using ArrayPtr = std::shared_ptr; using BooleanArray = arrow::BooleanArray; using BooleanArrayPtr = std::shared_ptr; -template +template using NumericArray = arrow::NumericArray; -template +template using NumericArrayPtr = std::shared_ptr>; using BinaryArray = arrow::BinaryArray; @@ -49,6 +48,5 @@ using FixedSizeBinaryArrayPtr = std::shared_ptr; using Decimal128Array = arrow::Decimal128Array; using Decimal128ArrayPtr = std::shared_ptr; - -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/common/BinarySet.h b/cpp/src/core/knowhere/knowhere/common/BinarySet.h index 509db68e2d..6d8580e071 100644 --- a/cpp/src/core/knowhere/knowhere/common/BinarySet.h +++ b/cpp/src/core/knowhere/knowhere/common/BinarySet.h @@ -15,21 +15,19 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include -#include -#include #include +#include +#include +#include #include "Id.h" - namespace zilliz { namespace knowhere { - struct Binary { ID id; std::shared_ptr data; @@ -37,29 +35,28 @@ struct Binary { }; using BinaryPtr = std::shared_ptr; - class BinarySet { public: BinaryPtr - GetByName(const std::string &name) const { + GetByName(const std::string& name) const { return binary_map_.at(name); } void - Append(const std::string &name, BinaryPtr binary) { + Append(const std::string& name, BinaryPtr binary) { binary_map_[name] = std::move(binary); } void - Append(const std::string &name, std::shared_ptr data, int64_t size) { + Append(const std::string& name, std::shared_ptr data, int64_t size) { auto binary = std::make_shared(); binary->data = data; binary->size = size; binary_map_[name] = std::move(binary); } - //void - //Append(const std::string &name, void *data, int64_t size, ID id) { + // void + // Append(const std::string &name, void *data, int64_t size, ID id) { // Binary binary; // binary.data = data; // binary.size = size; @@ -67,7 +64,8 @@ class BinarySet { // binary_map_[name] = binary; //} - void clear() { + void + clear() { binary_map_.clear(); } @@ -75,6 +73,5 @@ class BinarySet { std::map binary_map_; }; - -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/common/Buffer.h b/cpp/src/core/knowhere/knowhere/common/Buffer.h index 4468e6ec01..42af87a09b 100644 --- a/cpp/src/core/knowhere/knowhere/common/Buffer.h +++ b/cpp/src/core/knowhere/knowhere/common/Buffer.h @@ -15,14 +15,12 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include #include - namespace zilliz { namespace knowhere { @@ -34,31 +32,32 @@ using MutableBufferPtr = std::shared_ptr; namespace internal { struct BufferDeleter { - void operator()(Buffer *buffer) { - free((void *) buffer->data()); + void + operator()(Buffer* buffer) { + free((void*)buffer->data()); } }; - } + inline BufferPtr -MakeBufferSmart(uint8_t *data, const int64_t size) { +MakeBufferSmart(uint8_t* data, const int64_t size) { return BufferPtr(new Buffer(data, size), internal::BufferDeleter()); } inline MutableBufferPtr -MakeMutableBufferSmart(uint8_t *data, const int64_t size) { +MakeMutableBufferSmart(uint8_t* data, const int64_t size) { return MutableBufferPtr(new MutableBuffer(data, size), internal::BufferDeleter()); } inline BufferPtr -MakeBuffer(uint8_t *data, const int64_t size) { +MakeBuffer(uint8_t* data, const int64_t size) { return std::make_shared(data, size); } inline MutableBufferPtr -MakeMutableBuffer(uint8_t *data, const int64_t size) { +MakeMutableBuffer(uint8_t* data, const int64_t size) { return std::make_shared(data, size); } -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/common/Config.h b/cpp/src/core/knowhere/knowhere/common/Config.h index cfee35c35b..afb621d5e1 100644 --- a/cpp/src/core/knowhere/knowhere/common/Config.h +++ b/cpp/src/core/knowhere/knowhere/common/Config.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include @@ -42,20 +41,18 @@ struct Cfg { int64_t gpu_id = DEFAULT_GPUID; int64_t d = DEFAULT_DIM; - Cfg(const int64_t &dim, - const int64_t &k, - const int64_t &gpu_id, - METRICTYPE type) - : metric_type(type), k(k), gpu_id(gpu_id), d(dim) {} + Cfg(const int64_t& dim, const int64_t& k, const int64_t& gpu_id, METRICTYPE type) + : metric_type(type), k(k), gpu_id(gpu_id), d(dim) { + } Cfg() = default; virtual bool - CheckValid(){ + CheckValid() { return true; - }; + } }; using Config = std::shared_ptr; -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/common/Dataset.h b/cpp/src/core/knowhere/knowhere/common/Dataset.h index 91fb165811..4a4f4bdd80 100644 --- a/cpp/src/core/knowhere/knowhere/common/Dataset.h +++ b/cpp/src/core/knowhere/knowhere/common/Dataset.h @@ -15,20 +15,19 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include #include +#include +#include #include "Array.h" #include "Buffer.h" -#include "Tensor.h" -#include "Schema.h" #include "Config.h" +#include "Schema.h" +#include "Tensor.h" #include "knowhere/adapter/ArrowAdapter.h" - namespace zilliz { namespace knowhere { @@ -40,34 +39,38 @@ class Dataset { public: Dataset() = default; - Dataset(std::vector &&array, SchemaPtr array_schema, - std::vector &&tensor, SchemaPtr tensor_schema) + Dataset(std::vector&& array, SchemaPtr array_schema, std::vector&& tensor, + SchemaPtr tensor_schema) : array_(std::move(array)), array_schema_(std::move(array_schema)), tensor_(std::move(tensor)), - tensor_schema_(std::move(tensor_schema)) {} + tensor_schema_(std::move(tensor_schema)) { + } Dataset(std::vector array, SchemaPtr array_schema) - : array_(std::move(array)), array_schema_(std::move(array_schema)) {} + : array_(std::move(array)), array_schema_(std::move(array_schema)) { + } Dataset(std::vector tensor, SchemaPtr tensor_schema) - : tensor_(std::move(tensor)), tensor_schema_(std::move(tensor_schema)) {} + : tensor_(std::move(tensor)), tensor_schema_(std::move(tensor_schema)) { + } - Dataset(const Dataset &) = delete; - Dataset &operator=(const Dataset &) = delete; + Dataset(const Dataset&) = delete; + Dataset& + operator=(const Dataset&) = delete; DatasetPtr Clone() { auto dataset = std::make_shared(); std::vector clone_array; - for (auto &array : array_) { + for (auto& array : array_) { clone_array.emplace_back(CopyArray(array)); } dataset->set_array(clone_array); std::vector clone_tensor; - for (auto &tensor : tensor_) { + for (auto& tensor : tensor_) { auto buffer = tensor->data(); std::shared_ptr copy_buffer; // TODO: checkout copy success; @@ -86,16 +89,20 @@ class Dataset { } public: - const std::vector & - array() const { return array_; } + const std::vector& + array() const { + return array_; + } void set_array(std::vector array) { array_ = std::move(array); } - const std::vector & - tensor() const { return tensor_; } + const std::vector& + tensor() const { + return tensor_; + } void set_tensor(std::vector tensor) { @@ -103,7 +110,9 @@ class Dataset { } SchemaConstPtr - array_schema() const { return array_schema_; } + array_schema() const { + return array_schema_; + } void set_array_schema(SchemaPtr array_schema) { @@ -111,18 +120,20 @@ class Dataset { } SchemaConstPtr - tensor_schema() const { return tensor_schema_; } + tensor_schema() const { + return tensor_schema_; + } void set_tensor_schema(SchemaPtr tensor_schema) { tensor_schema_ = std::move(tensor_schema); } - //const Config & - //meta() const { return meta_; } + // const Config & + // meta() const { return meta_; } - //void - //set_meta(Config meta) { + // void + // set_meta(Config meta) { // meta_ = std::move(meta); //} @@ -131,11 +142,10 @@ class Dataset { SchemaPtr array_schema_; std::vector tensor_; SchemaPtr tensor_schema_; - //Config meta_; + // Config meta_; }; using DatasetPtr = std::shared_ptr; - -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/common/Exception.cpp b/cpp/src/core/knowhere/knowhere/common/Exception.cpp index a77e85ee41..75d9ce39bb 100644 --- a/cpp/src/core/knowhere/knowhere/common/Exception.cpp +++ b/cpp/src/core/knowhere/knowhere/common/Exception.cpp @@ -15,41 +15,37 @@ // specific language governing permissions and limitations // under the License. - #include -#include "Exception.h" #include "Log.h" +#include "knowhere/common/Exception.h" namespace zilliz { namespace knowhere { +KnowhereException::KnowhereException(const std::string& msg) : msg(msg) { +} -KnowhereException::KnowhereException(const std::string &msg):msg(msg) {} - -KnowhereException::KnowhereException(const std::string &m, const char *funcName, const char *file, int line) { +KnowhereException::KnowhereException(const std::string& m, const char* funcName, const char* file, int line) { #ifdef DEBUG - int size = snprintf(nullptr, 0, "Error in %s at %s:%d: %s", - funcName, file, line, m.c_str()); + int size = snprintf(nullptr, 0, "Error in %s at %s:%d: %s", funcName, file, line, m.c_str()); msg.resize(size + 1); - snprintf(&msg[0], msg.size(), "Error in %s at %s:%d: %s", - funcName, file, line, m.c_str()); + snprintf(&msg[0], msg.size(), "Error in %s at %s:%d: %s", funcName, file, line, m.c_str()); #else std::string file_path(file); auto const pos = file_path.find_last_of('/'); - auto filename = file_path.substr(pos+1).c_str(); + auto filename = file_path.substr(pos + 1).c_str(); - int size = snprintf(nullptr, 0, "Error in %s at %s:%d: %s", - funcName, filename, line, m.c_str()); + int size = snprintf(nullptr, 0, "Error in %s at %s:%d: %s", funcName, filename, line, m.c_str()); msg.resize(size + 1); - snprintf(&msg[0], msg.size(), "Error in %s at %s:%d: %s", - funcName, filename, line, m.c_str()); + snprintf(&msg[0], msg.size(), "Error in %s at %s:%d: %s", funcName, filename, line, m.c_str()); #endif } -const char *KnowhereException::what() const noexcept { +const char* +KnowhereException::what() const noexcept { return msg.c_str(); } -} -} \ No newline at end of file +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/common/Exception.h b/cpp/src/core/knowhere/knowhere/common/Exception.h index d357f0501e..ec89c4d616 100644 --- a/cpp/src/core/knowhere/knowhere/common/Exception.h +++ b/cpp/src/core/knowhere/knowhere/common/Exception.h @@ -15,46 +15,41 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include #include - namespace zilliz { namespace knowhere { class KnowhereException : public std::exception { public: - explicit KnowhereException(const std::string &msg); + explicit KnowhereException(const std::string& msg); - KnowhereException(const std::string &msg, const char *funName, - const char *file, int line); + KnowhereException(const std::string& msg, const char* funName, const char* file, int line); - const char *what() const noexcept override; + const char* + what() const noexcept override; std::string msg; }; +#define KNOHWERE_ERROR_MSG(MSG) printf("%s", KnowhereException(MSG, __PRETTY_FUNCTION__, __FILE__, __LINE__).what()) -#define KNOHWERE_ERROR_MSG(MSG)\ -printf("%s", KnowhereException(MSG, __PRETTY_FUNCTION__, __FILE__, __LINE__).what()) +#define KNOWHERE_THROW_MSG(MSG) \ + do { \ + throw KnowhereException(MSG, __PRETTY_FUNCTION__, __FILE__, __LINE__); \ + } while (false) -#define KNOWHERE_THROW_MSG(MSG)\ -do {\ - throw KnowhereException(MSG, __PRETTY_FUNCTION__, __FILE__, __LINE__);\ -} while (false) +#define KNOHERE_THROW_FORMAT(FMT, ...) \ + do { \ + std::string __s; \ + int __size = snprintf(nullptr, 0, FMT, __VA_ARGS__); \ + __s.resize(__size + 1); \ + snprintf(&__s[0], __s.size(), FMT, __VA_ARGS__); \ + throw faiss::FaissException(__s, __PRETTY_FUNCTION__, __FILE__, __LINE__); \ + } while (false) -#define KNOHERE_THROW_FORMAT(FMT, ...)\ - do { \ - std::string __s;\ - int __size = snprintf(nullptr, 0, FMT, __VA_ARGS__);\ - __s.resize(__size + 1);\ - snprintf(&__s[0], __s.size(), FMT, __VA_ARGS__);\ - throw faiss::FaissException(__s, __PRETTY_FUNCTION__, __FILE__, __LINE__);\ - } while (false) - - -} -} \ No newline at end of file +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/common/Id.h b/cpp/src/core/knowhere/knowhere/common/Id.h index 87e1467c44..9686075874 100644 --- a/cpp/src/core/knowhere/knowhere/common/Id.h +++ b/cpp/src/core/knowhere/knowhere/common/Id.h @@ -15,11 +15,10 @@ // specific language governing permissions and limitations // under the License. - #pragma once //#include "zcommon/id/id.h" -//using ID = zilliz::common::ID; +// using ID = zilliz::common::ID; #include #include @@ -27,18 +26,20 @@ namespace zilliz { namespace knowhere { - - class ID { public: constexpr static int64_t kIDSize = 20; public: - const int32_t * - data() const { return content_; } + const int32_t* + data() const { + return content_; + } - int32_t * - mutable_data() { return content_; } + int32_t* + mutable_data() { + return content_; + } bool IsValid() const; @@ -47,14 +48,14 @@ class ID { ToString() const; bool - operator==(const ID &that) const; + operator==(const ID& that) const; bool - operator<(const ID &that) const; + operator<(const ID& that) const; protected: int32_t content_[5] = {}; }; -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/common/Log.h b/cpp/src/core/knowhere/knowhere/common/Log.h index 1e390b3c1e..1ca78b155a 100644 --- a/cpp/src/core/knowhere/knowhere/common/Log.h +++ b/cpp/src/core/knowhere/knowhere/common/Log.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "utils/easylogging++.h" @@ -33,5 +32,5 @@ namespace knowhere { #define KNOWHERE_LOG_ERROR LOG(ERROR) << KNOWHERE_DOMAIN_NAME #define KNOWHERE_LOG_FATAL LOG(FATAL) << KNOWHERE_DOMAIN_NAME -} // namespace knowhere -} // namespace zilliz \ No newline at end of file +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/common/Schema.h b/cpp/src/core/knowhere/knowhere/common/Schema.h index b43b7eb875..6dc4784e53 100644 --- a/cpp/src/core/knowhere/knowhere/common/Schema.h +++ b/cpp/src/core/knowhere/knowhere/common/Schema.h @@ -15,18 +15,15 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include #include - namespace zilliz { namespace knowhere { - using DataType = arrow::DataType; using Field = arrow::Field; using FieldPtr = std::shared_ptr; @@ -34,7 +31,5 @@ using Schema = arrow::Schema; using SchemaPtr = std::shared_ptr; using SchemaConstPtr = std::shared_ptr; - - -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/common/Tensor.h b/cpp/src/core/knowhere/knowhere/common/Tensor.h index 42e86dc4d1..cd23933e96 100644 --- a/cpp/src/core/knowhere/knowhere/common/Tensor.h +++ b/cpp/src/core/knowhere/knowhere/common/Tensor.h @@ -15,21 +15,17 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include #include - namespace zilliz { namespace knowhere { - using Tensor = arrow::Tensor; using TensorPtr = std::shared_ptr; - -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/common/Timer.cpp b/cpp/src/core/knowhere/knowhere/common/Timer.cpp index 7f2cb71514..f4db63203c 100644 --- a/cpp/src/core/knowhere/knowhere/common/Timer.cpp +++ b/cpp/src/core/knowhere/knowhere/common/Timer.cpp @@ -15,18 +15,14 @@ // specific language governing permissions and limitations // under the License. +#include // TODO(linxj): using Log instead -#include // TODO(linxj): using Log instead - -#include "Timer.h" +#include "knowhere/common/Timer.h" namespace zilliz { namespace knowhere { -TimeRecorder::TimeRecorder(const std::string &header, - int64_t log_level) : - header_(header), - log_level_(log_level) { +TimeRecorder::TimeRecorder(const std::string& header, int64_t log_level) : header_(header), log_level_(log_level) { start_ = last_ = stdclock::now(); } @@ -42,9 +38,10 @@ TimeRecorder::GetTimeSpanStr(double span) { } void -TimeRecorder::PrintTimeRecord(const std::string &msg, double span) { +TimeRecorder::PrintTimeRecord(const std::string& msg, double span) { std::string str_log; - if (!header_.empty()) str_log += header_ + ": "; + if (!header_.empty()) + str_log += header_ + ": "; str_log += msg; str_log += " ("; str_log += TimeRecorder::GetTimeSpanStr(span); @@ -55,35 +52,35 @@ TimeRecorder::PrintTimeRecord(const std::string &msg, double span) { std::cout << str_log << std::endl; break; } - //case 1: { - // SERVER_LOG_DEBUG << str_log; - // break; - //} - //case 2: { - // SERVER_LOG_INFO << str_log; - // break; - //} - //case 3: { - // SERVER_LOG_WARNING << str_log; - // break; - //} - //case 4: { - // SERVER_LOG_ERROR << str_log; - // break; - //} - //case 5: { - // SERVER_LOG_FATAL << str_log; - // break; - //} - //default: { - // SERVER_LOG_INFO << str_log; - // break; - //} + // case 1: { + // SERVER_LOG_DEBUG << str_log; + // break; + //} + // case 2: { + // SERVER_LOG_INFO << str_log; + // break; + //} + // case 3: { + // SERVER_LOG_WARNING << str_log; + // break; + //} + // case 4: { + // SERVER_LOG_ERROR << str_log; + // break; + //} + // case 5: { + // SERVER_LOG_FATAL << str_log; + // break; + //} + // default: { + // SERVER_LOG_INFO << str_log; + // break; + //} } } double -TimeRecorder::RecordSection(const std::string &msg) { +TimeRecorder::RecordSection(const std::string& msg) { stdclock::time_point curr = stdclock::now(); double span = (std::chrono::duration(curr - last_)).count(); last_ = curr; @@ -93,7 +90,7 @@ TimeRecorder::RecordSection(const std::string &msg) { } double -TimeRecorder::ElapseFromBegin(const std::string &msg) { +TimeRecorder::ElapseFromBegin(const std::string& msg) { stdclock::time_point curr = stdclock::now(); double span = (std::chrono::duration(curr - start_)).count(); @@ -101,5 +98,5 @@ TimeRecorder::ElapseFromBegin(const std::string &msg) { return span; } -} -} \ No newline at end of file +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/common/Timer.h b/cpp/src/core/knowhere/knowhere/common/Timer.h index 8ecd60df99..b1557e4c20 100644 --- a/cpp/src/core/knowhere/knowhere/common/Timer.h +++ b/cpp/src/core/knowhere/knowhere/common/Timer.h @@ -15,11 +15,10 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include #include +#include namespace zilliz { namespace knowhere { @@ -28,19 +27,22 @@ class TimeRecorder { using stdclock = std::chrono::high_resolution_clock; public: - TimeRecorder(const std::string &header, - int64_t log_level = 0); + explicit TimeRecorder(const std::string& header, int64_t log_level = 0); - ~TimeRecorder();//trace = 0, debug = 1, info = 2, warn = 3, error = 4, critical = 5 + ~TimeRecorder(); // trace = 0, debug = 1, info = 2, warn = 3, error = 4, critical = 5 - double RecordSection(const std::string &msg); + double + RecordSection(const std::string& msg); - double ElapseFromBegin(const std::string &msg); + double + ElapseFromBegin(const std::string& msg); - static std::string GetTimeSpanStr(double span); + static std::string + GetTimeSpanStr(double span); private: - void PrintTimeRecord(const std::string &msg, double span); + void + PrintTimeRecord(const std::string& msg, double span); private: std::string header_; @@ -49,5 +51,5 @@ class TimeRecorder { int64_t log_level_; }; -} -} +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/Index.h b/cpp/src/core/knowhere/knowhere/index/Index.h index 4ccbef394c..613a8d049a 100644 --- a/cpp/src/core/knowhere/knowhere/index/Index.h +++ b/cpp/src/core/knowhere/knowhere/index/Index.h @@ -15,54 +15,55 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include +#include "IndexModel.h" +#include "IndexType.h" #include "knowhere/common/BinarySet.h" #include "knowhere/common/Dataset.h" -#include "IndexType.h" -#include "IndexModel.h" #include "knowhere/index/preprocessor/Preprocessor.h" - namespace zilliz { namespace knowhere { - class Index { public: virtual BinarySet Serialize() = 0; virtual void - Load(const BinarySet &index_binary) = 0; + Load(const BinarySet& index_binary) = 0; // @throw virtual DatasetPtr - Search(const DatasetPtr &dataset, const Config &config) = 0; + Search(const DatasetPtr& dataset, const Config& config) = 0; public: IndexType - idx_type() const { return idx_type_; } + idx_type() const { + return idx_type_; + } void - set_idx_type(IndexType idx_type) { idx_type_ = idx_type; } + set_idx_type(IndexType idx_type) { + idx_type_ = idx_type; + } virtual void - set_preprocessor(PreprocessorPtr preprocessor) {} + set_preprocessor(PreprocessorPtr preprocessor) { + } virtual void - set_index_model(IndexModelPtr model) {} + set_index_model(IndexModelPtr model) { + } private: IndexType idx_type_; }; - using IndexPtr = std::shared_ptr; - -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/IndexModel.h b/cpp/src/core/knowhere/knowhere/index/IndexModel.h index 7557363f65..64acc5ad22 100644 --- a/cpp/src/core/knowhere/knowhere/index/IndexModel.h +++ b/cpp/src/core/knowhere/knowhere/index/IndexModel.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include @@ -24,19 +23,16 @@ namespace zilliz { namespace knowhere { - class IndexModel { public: virtual BinarySet Serialize() = 0; virtual void - Load(const BinarySet &binary) = 0; + Load(const BinarySet& binary) = 0; }; using IndexModelPtr = std::shared_ptr; - - -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/IndexType.h b/cpp/src/core/knowhere/knowhere/index/IndexType.h index 3ece6287f5..64a2188790 100644 --- a/cpp/src/core/knowhere/knowhere/index/IndexType.h +++ b/cpp/src/core/knowhere/knowhere/index/IndexType.h @@ -15,14 +15,11 @@ // specific language governing permissions and limitations // under the License. - #pragma once - namespace zilliz { namespace knowhere { - enum class IndexType { kUnknown = 0, kVecIdxBegin = 100, @@ -30,6 +27,5 @@ enum class IndexType { kVecIdxEnd, }; - -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/preprocessor/Normalize.cpp b/cpp/src/core/knowhere/knowhere/index/preprocessor/Normalize.cpp index 41f6ad3715..2a31a22f16 100644 --- a/cpp/src/core/knowhere/knowhere/index/preprocessor/Normalize.cpp +++ b/cpp/src/core/knowhere/knowhere/index/preprocessor/Normalize.cpp @@ -1,14 +1,30 @@ +//// Licensed to the Apache Software Foundation (ASF) under one +//// or more contributor license agreements. See the NOTICE file +//// distributed with this work for additional information +//// regarding copyright ownership. The ASF licenses this file +//// to you under the Apache License, Version 2.0 (the +//// "License"); you may not use this file except in compliance +//// with the License. You may obtain a copy of the License at +//// +//// http://www.apache.org/licenses/LICENSE-2.0 +//// +//// Unless required by applicable law or agreed to in writing, +//// software distributed under the License is distributed on an +//// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//// KIND, either express or implied. See the License for the +//// specific language governing permissions and limitations +//// under the License. // //#include "knowhere/index/vector_index/definitions.h" //#include "knowhere/common/config.h" -//#include "knowhere/index/preprocessor/normalize.h" +#include "knowhere/index/preprocessor/Normalize.h" // // -//namespace zilliz { -//namespace knowhere { +// namespace zilliz { +// namespace knowhere { // -//DatasetPtr -//NormalizePreprocessor::Preprocess(const DatasetPtr &dataset) { +// DatasetPtr +// NormalizePreprocessor::Preprocess(const DatasetPtr &dataset) { // // TODO: wrap dataset->tensor // auto tensor = dataset->tensor()[0]; // auto p_data = (float *)tensor->raw_mutable_data(); @@ -21,8 +37,8 @@ // } //} // -//void -//NormalizePreprocessor::Normalize(float *arr, int64_t dimension) { +// void +// NormalizePreprocessor::Normalize(float *arr, int64_t dimension) { // double vector_length = 0; // for (auto j = 0; j < dimension; j++) { // double val = arr[j]; @@ -39,4 +55,3 @@ // //} // namespace knowhere //} // namespace zilliz - diff --git a/cpp/src/core/knowhere/knowhere/index/preprocessor/Normalize.h b/cpp/src/core/knowhere/knowhere/index/preprocessor/Normalize.h index 399a5d3490..9e918d56b8 100644 --- a/cpp/src/core/knowhere/knowhere/index/preprocessor/Normalize.h +++ b/cpp/src/core/knowhere/knowhere/index/preprocessor/Normalize.h @@ -1,13 +1,30 @@ +//// Licensed to the Apache Software Foundation (ASF) under one +//// or more contributor license agreements. See the NOTICE file +//// distributed with this work for additional information +//// regarding copyright ownership. The ASF licenses this file +//// to you under the Apache License, Version 2.0 (the +//// "License"); you may not use this file except in compliance +//// with the License. You may obtain a copy of the License at +//// +//// http://www.apache.org/licenses/LICENSE-2.0 +//// +//// Unless required by applicable law or agreed to in writing, +//// software distributed under the License is distributed on an +//// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//// KIND, either express or implied. See the License for the +//// specific language governing permissions and limitations +//// under the License. +// //#pragma once // //#include //#include "preprocessor.h" // // -//namespace zilliz { -//namespace knowhere { +// namespace zilliz { +// namespace knowhere { // -//class NormalizePreprocessor : public Preprocessor { +// class NormalizePreprocessor : public Preprocessor { // public: // DatasetPtr // Preprocess(const DatasetPtr &input) override; @@ -19,7 +36,7 @@ //}; // // -//using NormalizePreprocessorPtr = std::shared_ptr; +// using NormalizePreprocessorPtr = std::shared_ptr; // // //} // namespace knowhere diff --git a/cpp/src/core/knowhere/knowhere/index/preprocessor/Preprocessor.h b/cpp/src/core/knowhere/knowhere/index/preprocessor/Preprocessor.h index e1c01d2085..c04dbdf3dc 100644 --- a/cpp/src/core/knowhere/knowhere/index/preprocessor/Preprocessor.h +++ b/cpp/src/core/knowhere/knowhere/index/preprocessor/Preprocessor.h @@ -15,27 +15,22 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include #include "knowhere/common/Dataset.h" - namespace zilliz { namespace knowhere { - class Preprocessor { public: virtual DatasetPtr - Preprocess(const DatasetPtr &input) = 0; + Preprocess(const DatasetPtr& input) = 0; }; - using PreprocessorPtr = std::shared_ptr; - -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp index a3ab7956cb..5e90da9d56 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp @@ -15,23 +15,24 @@ // specific language governing permissions and limitations // under the License. - -#include #include +#include +#include #include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/FaissBaseIndex.h" #include "knowhere/index/vector_index/helpers/FaissIO.h" -#include "FaissBaseIndex.h" - namespace zilliz { namespace knowhere { -FaissBaseIndex::FaissBaseIndex(std::shared_ptr index) : index_(std::move(index)) {} +FaissBaseIndex::FaissBaseIndex(std::shared_ptr index) : index_(std::move(index)) { +} -BinarySet FaissBaseIndex::SerializeImpl() { +BinarySet +FaissBaseIndex::SerializeImpl() { try { - faiss::Index *index = index_.get(); + faiss::Index* index = index_.get(); SealImpl(); @@ -44,37 +45,38 @@ BinarySet FaissBaseIndex::SerializeImpl() { // TODO(linxj): use virtual func Name() instead of raw string. res_set.Append("IVF", data, writer.rp); return res_set; - } catch (std::exception &e) { + } catch (std::exception& e) { KNOWHERE_THROW_MSG(e.what()); } } -void FaissBaseIndex::LoadImpl(const BinarySet &index_binary) { +void +FaissBaseIndex::LoadImpl(const BinarySet& index_binary) { auto binary = index_binary.GetByName("IVF"); MemoryIOReader reader; reader.total = binary->size; reader.data_ = binary->data.get(); - faiss::Index *index = faiss::read_index(&reader); + faiss::Index* index = faiss::read_index(&reader); index_.reset(index); } -void FaissBaseIndex::SealImpl() { -// TODO(linxj): enable -//#ifdef ZILLIZ_FAISS - faiss::Index *index = index_.get(); - auto idx = dynamic_cast(index); +void +FaissBaseIndex::SealImpl() { + // TODO(linxj): enable + //#ifdef ZILLIZ_FAISS + faiss::Index* index = index_.get(); + auto idx = dynamic_cast(index); if (idx != nullptr) { idx->to_readonly(); } - //else { + // else { // KNOHWERE_ERROR_MSG("Seal failed"); //} -//#endif + //#endif } -} // knowhere -} // zilliz - +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.h b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.h index 29edbbd61e..bac4a2bf29 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include @@ -24,7 +23,6 @@ #include "knowhere/common/BinarySet.h" - namespace zilliz { namespace knowhere { @@ -36,7 +34,7 @@ class FaissBaseIndex { SerializeImpl(); virtual void - LoadImpl(const BinarySet &index_binary); + LoadImpl(const BinarySet& index_binary); virtual void SealImpl(); @@ -45,8 +43,5 @@ class FaissBaseIndex { std::shared_ptr index_ = nullptr; }; -} // knowhere -} // zilliz - - - +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp index ba41859135..332d31bce0 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp @@ -15,30 +15,28 @@ // specific language governing permissions and limitations // under the License. - - +#include +#include #include #include #include -#include -#include #include +#include - -#include "knowhere/common/Exception.h" -#include "knowhere/index/vector_index/helpers/Cloner.h" #include "knowhere/adapter/VectorAdapter.h" -#include "IndexGPUIVF.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/helpers/Cloner.h" #include "knowhere/index/vector_index/helpers/FaissIO.h" - namespace zilliz { namespace knowhere { -IndexModelPtr GPUIVF::Train(const DatasetPtr &dataset, const Config &config) { +IndexModelPtr +GPUIVF::Train(const DatasetPtr& dataset, const Config& config) { auto build_cfg = std::dynamic_pointer_cast(config); if (build_cfg != nullptr) { - build_cfg->CheckValid(); // throw exception + build_cfg->CheckValid(); // throw exception } gpu_id_ = build_cfg->gpu_id; @@ -49,10 +47,9 @@ IndexModelPtr GPUIVF::Train(const DatasetPtr &dataset, const Config &config) { ResScope rs(temp_resource, gpu_id_, true); faiss::gpu::GpuIndexIVFFlatConfig idx_config; idx_config.device = gpu_id_; - faiss::gpu::GpuIndexIVFFlat device_index(temp_resource->faiss_res.get(), dim, - build_cfg->nlist, GetMetricType(build_cfg->metric_type), - idx_config); - device_index.train(rows, (float *) p_data); + faiss::gpu::GpuIndexIVFFlat device_index(temp_resource->faiss_res.get(), dim, build_cfg->nlist, + GetMetricType(build_cfg->metric_type), idx_config); + device_index.train(rows, (float*)p_data); std::shared_ptr host_index = nullptr; host_index.reset(faiss::gpu::index_gpu_to_cpu(&device_index)); @@ -63,7 +60,8 @@ IndexModelPtr GPUIVF::Train(const DatasetPtr &dataset, const Config &config) { } } -void GPUIVF::set_index_model(IndexModelPtr model) { +void +GPUIVF::set_index_model(IndexModelPtr model) { std::lock_guard lk(mutex_); auto host_index = std::static_pointer_cast(model); @@ -77,7 +75,8 @@ void GPUIVF::set_index_model(IndexModelPtr model) { } } -BinarySet GPUIVF::SerializeImpl() { +BinarySet +GPUIVF::SerializeImpl() { if (!index_ || !index_->is_trained) { KNOWHERE_THROW_MSG("index not initialize or trained"); } @@ -85,8 +84,8 @@ BinarySet GPUIVF::SerializeImpl() { try { MemoryIOWriter writer; { - faiss::Index *index = index_.get(); - faiss::Index *host_index = faiss::gpu::index_gpu_to_cpu(index); + faiss::Index* index = index_.get(); + faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(index); SealImpl(); @@ -100,19 +99,20 @@ BinarySet GPUIVF::SerializeImpl() { res_set.Append("IVF", data, writer.rp); return res_set; - } catch (std::exception &e) { + } catch (std::exception& e) { KNOWHERE_THROW_MSG(e.what()); } } -void GPUIVF::LoadImpl(const BinarySet &index_binary) { +void +GPUIVF::LoadImpl(const BinarySet& index_binary) { auto binary = index_binary.GetByName("IVF"); MemoryIOReader reader; { reader.total = binary->size; reader.data_ = binary->data.get(); - faiss::Index *index = faiss::read_index(&reader); + faiss::Index* index = faiss::read_index(&reader); if (auto temp_res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_)) { ResScope rs(temp_res, gpu_id_, false); @@ -127,23 +127,20 @@ void GPUIVF::LoadImpl(const BinarySet &index_binary) { } } -IVFIndexPtr GPUIVF::Copy_index_gpu_to_cpu() { +IVFIndexPtr +GPUIVF::Copy_index_gpu_to_cpu() { std::lock_guard lk(mutex_); - faiss::Index *device_index = index_.get(); - faiss::Index *host_index = faiss::gpu::index_gpu_to_cpu(device_index); + faiss::Index* device_index = index_.get(); + faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(device_index); std::shared_ptr new_index; new_index.reset(host_index); return std::make_shared(new_index); } -void GPUIVF::search_impl(int64_t n, - const float *data, - int64_t k, - float *distances, - int64_t *labels, - const Config &cfg) { +void +GPUIVF::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) { std::lock_guard lk(mutex_); // TODO(linxj): gpu index support GenParams @@ -154,49 +151,54 @@ void GPUIVF::search_impl(int64_t n, { // TODO(linxj): allocate mem ResScope rs(res_, gpu_id_); - device_index->search(n, (float *) data, k, distances, labels); + device_index->search(n, (float*)data, k, distances, labels); } } } -VectorIndexPtr GPUIVF::CopyGpuToCpu(const Config &config) { +VectorIndexPtr +GPUIVF::CopyGpuToCpu(const Config& config) { std::lock_guard lk(mutex_); - faiss::Index *device_index = index_.get(); - faiss::Index *host_index = faiss::gpu::index_gpu_to_cpu(device_index); + faiss::Index* device_index = index_.get(); + faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(device_index); std::shared_ptr new_index; new_index.reset(host_index); return std::make_shared(new_index); } -VectorIndexPtr GPUIVF::Clone() { +VectorIndexPtr +GPUIVF::Clone() { auto cpu_idx = CopyGpuToCpu(Config()); return ::zilliz::knowhere::cloner::CopyCpuToGpu(cpu_idx, gpu_id_, Config()); } -VectorIndexPtr GPUIVF::CopyGpuToGpu(const int64_t &device_id, const Config &config) { +VectorIndexPtr +GPUIVF::CopyGpuToGpu(const int64_t& device_id, const Config& config) { auto host_index = CopyGpuToCpu(config); return std::static_pointer_cast(host_index)->CopyCpuToGpu(device_id, config); } -void GPUIVF::Add(const DatasetPtr &dataset, const Config &config) { +void +GPUIVF::Add(const DatasetPtr& dataset, const Config& config) { if (auto spt = res_.lock()) { ResScope rs(res_, gpu_id_); IVF::Add(dataset, config); - } - else { + } else { KNOWHERE_THROW_MSG("Add IVF can't get gpu resource"); } } -void GPUIndex::SetGpuDevice(const int &gpu_id) { +void +GPUIndex::SetGpuDevice(const int& gpu_id) { gpu_id_ = gpu_id; } -const int64_t &GPUIndex::GetGpuDevice() { +const int64_t& +GPUIndex::GetGpuDevice() { return gpu_id_; } -} -} +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.h index cd04fe4c54..347e255b87 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.h @@ -15,84 +15,84 @@ // specific language governing permissions and limitations // under the License. - #pragma once +#include +#include #include "IndexIVF.h" #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" - namespace zilliz { namespace knowhere { class GPUIndex { -public: - explicit GPUIndex(const int &device_id) : gpu_id_(device_id) {} + public: + explicit GPUIndex(const int& device_id) : gpu_id_(device_id) { + } - GPUIndex(const int& device_id, const ResPtr& resource): gpu_id_(device_id), res_(resource) {} + GPUIndex(const int& device_id, const ResPtr& resource) : gpu_id_(device_id), res_(resource) { + } - virtual VectorIndexPtr - CopyGpuToCpu(const Config &config) = 0; + virtual VectorIndexPtr + CopyGpuToCpu(const Config& config) = 0; - virtual VectorIndexPtr - CopyGpuToGpu(const int64_t &device_id, const Config &config) = 0; + virtual VectorIndexPtr + CopyGpuToGpu(const int64_t& device_id, const Config& config) = 0; - void - SetGpuDevice(const int &gpu_id); + void + SetGpuDevice(const int& gpu_id); - const int64_t & - GetGpuDevice(); + const int64_t& + GetGpuDevice(); -protected: - int64_t gpu_id_; - ResWPtr res_; + protected: + int64_t gpu_id_; + ResWPtr res_; }; class GPUIVF : public IVF, public GPUIndex { -public: - explicit GPUIVF(const int &device_id) : IVF(), GPUIndex(device_id) {} + public: + explicit GPUIVF(const int& device_id) : IVF(), GPUIndex(device_id) { + } - explicit GPUIVF(std::shared_ptr index, const int64_t &device_id, ResPtr &resource) - : IVF(std::move(index)), GPUIndex(device_id, resource) {}; + explicit GPUIVF(std::shared_ptr index, const int64_t& device_id, ResPtr& resource) + : IVF(std::move(index)), GPUIndex(device_id, resource) { + } - IndexModelPtr - Train(const DatasetPtr &dataset, const Config &config) override; + IndexModelPtr + Train(const DatasetPtr& dataset, const Config& config) override; - void - Add(const DatasetPtr &dataset, const Config &config) override; + void + Add(const DatasetPtr& dataset, const Config& config) override; - void - set_index_model(IndexModelPtr model) override; + void + set_index_model(IndexModelPtr model) override; - //DatasetPtr Search(const DatasetPtr &dataset, const Config &config) override; - VectorIndexPtr - CopyGpuToCpu(const Config &config) override; + // DatasetPtr Search(const DatasetPtr &dataset, const Config &config) override; + VectorIndexPtr + CopyGpuToCpu(const Config& config) override; - VectorIndexPtr - CopyGpuToGpu(const int64_t &device_id, const Config &config) override; + VectorIndexPtr + CopyGpuToGpu(const int64_t& device_id, const Config& config) override; - VectorIndexPtr - Clone() final; + VectorIndexPtr + Clone() final; - // TODO(linxj): Deprecated - virtual IVFIndexPtr Copy_index_gpu_to_cpu(); + // TODO(linxj): Deprecated + virtual IVFIndexPtr + Copy_index_gpu_to_cpu(); -protected: - void - search_impl(int64_t n, - const float *data, - int64_t k, - float *distances, - int64_t *labels, - const Config &cfg) override; + protected: + void + search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) override; - BinarySet - SerializeImpl() override; + BinarySet + SerializeImpl() override; - void - LoadImpl(const BinarySet &index_binary) override; + void + LoadImpl(const BinarySet& index_binary) override; }; -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp index 03193ea604..7aa8f7db2a 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp @@ -15,23 +15,23 @@ // specific language governing permissions and limitations // under the License. - -#include -#include #include +#include +#include +#include -#include "IndexGPUIVFPQ.h" -#include "knowhere/common/Exception.h" #include "knowhere/adapter/VectorAdapter.h" - +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/IndexGPUIVFPQ.h" namespace zilliz { namespace knowhere { -IndexModelPtr GPUIVFPQ::Train(const DatasetPtr &dataset, const Config &config) { +IndexModelPtr +GPUIVFPQ::Train(const DatasetPtr& dataset, const Config& config) { auto build_cfg = std::dynamic_pointer_cast(config); if (build_cfg != nullptr) { - build_cfg->CheckValid(); // throw exception + build_cfg->CheckValid(); // throw exception } gpu_id_ = build_cfg->gpu_id; @@ -40,9 +40,9 @@ IndexModelPtr GPUIVFPQ::Train(const DatasetPtr &dataset, const Config &config) { // TODO(linxj): set device here. // TODO(linxj): set gpu resource here. faiss::gpu::StandardGpuResources res; - faiss::gpu::GpuIndexIVFPQ device_index(&res, dim, build_cfg->nlist, build_cfg->m, - build_cfg->nbits, GetMetricType(build_cfg->metric_type)); // IP not support - device_index.train(rows, (float *) p_data); + faiss::gpu::GpuIndexIVFPQ device_index(&res, dim, build_cfg->nlist, build_cfg->m, build_cfg->nbits, + GetMetricType(build_cfg->metric_type)); // IP not support + device_index.train(rows, (float*)p_data); std::shared_ptr host_index = nullptr; host_index.reset(faiss::gpu::index_gpu_to_cpu(&device_index)); @@ -50,20 +50,22 @@ IndexModelPtr GPUIVFPQ::Train(const DatasetPtr &dataset, const Config &config) { return std::make_shared(host_index); } -std::shared_ptr GPUIVFPQ::GenParams(const Config &config) { +std::shared_ptr +GPUIVFPQ::GenParams(const Config& config) { auto params = std::make_shared(); auto search_cfg = std::dynamic_pointer_cast(config); params->nprobe = search_cfg->nprobe; -// params->scan_table_threshold = conf->scan_table_threhold; -// params->polysemous_ht = conf->polysemous_ht; -// params->max_codes = conf->max_codes; + // params->scan_table_threshold = conf->scan_table_threhold; + // params->polysemous_ht = conf->polysemous_ht; + // params->max_codes = conf->max_codes; return params; } -VectorIndexPtr GPUIVFPQ::CopyGpuToCpu(const Config &config) { +VectorIndexPtr +GPUIVFPQ::CopyGpuToCpu(const Config& config) { KNOWHERE_THROW_MSG("not support yet"); } -} // knowhere -} // zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.h index c407ee0cc7..6d298e77e6 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.h @@ -15,33 +15,32 @@ // specific language governing permissions and limitations // under the License. - #pragma once +#include + #include "IndexGPUIVF.h" namespace zilliz { namespace knowhere { class GPUIVFPQ : public GPUIVF { -public: - explicit GPUIVFPQ(const int &device_id) : GPUIVF(device_id) {} + public: + explicit GPUIVFPQ(const int& device_id) : GPUIVF(device_id) { + } IndexModelPtr - Train(const DatasetPtr &dataset, const Config &config) override; + Train(const DatasetPtr& dataset, const Config& config) override; -public: + public: VectorIndexPtr - CopyGpuToCpu(const Config &config) override; + CopyGpuToCpu(const Config& config) override; -protected: + protected: // TODO(linxj): remove GenParams. std::shared_ptr - GenParams(const Config &config) override; + GenParams(const Config& config) override; }; -} // knowhere -} // zilliz - - - +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp index af23267404..bf2aa22474 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp @@ -15,60 +15,62 @@ // specific language governing permissions and limitations // under the License. - #include +#include +#include #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" -#include "IndexGPUIVFSQ.h" -#include "IndexIVFSQ.h" - +#include "knowhere/index/vector_index/IndexGPUIVFSQ.h" +#include "knowhere/index/vector_index/IndexIVFSQ.h" namespace zilliz { namespace knowhere { - IndexModelPtr GPUIVFSQ::Train(const DatasetPtr &dataset, const Config &config) { - auto build_cfg = std::dynamic_pointer_cast(config); - if (build_cfg != nullptr) { - build_cfg->CheckValid(); // throw exception - } - gpu_id_ = build_cfg->gpu_id; - - GETTENSOR(dataset) - - std::stringstream index_type; - index_type << "IVF" << build_cfg->nlist << "," << "SQ" << build_cfg->nbits; - auto build_index = faiss::index_factory(dim, index_type.str().c_str(), GetMetricType(build_cfg->metric_type)); - - auto temp_resource = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_); - if (temp_resource != nullptr) { - ResScope rs(temp_resource, gpu_id_, true); - auto device_index = faiss::gpu::index_cpu_to_gpu(temp_resource->faiss_res.get(), gpu_id_, build_index); - device_index->train(rows, (float *) p_data); - - std::shared_ptr host_index = nullptr; - host_index.reset(faiss::gpu::index_gpu_to_cpu(device_index)); - - delete device_index; - delete build_index; - - return std::make_shared(host_index); - } else { - KNOWHERE_THROW_MSG("Build IVFSQ can't get gpu resource"); - } +IndexModelPtr +GPUIVFSQ::Train(const DatasetPtr& dataset, const Config& config) { + auto build_cfg = std::dynamic_pointer_cast(config); + if (build_cfg != nullptr) { + build_cfg->CheckValid(); // throw exception } + gpu_id_ = build_cfg->gpu_id; - VectorIndexPtr GPUIVFSQ::CopyGpuToCpu(const Config &config) { - std::lock_guard lk(mutex_); + GETTENSOR(dataset) - faiss::Index *device_index = index_.get(); - faiss::Index *host_index = faiss::gpu::index_gpu_to_cpu(device_index); + std::stringstream index_type; + index_type << "IVF" << build_cfg->nlist << "," + << "SQ" << build_cfg->nbits; + auto build_index = faiss::index_factory(dim, index_type.str().c_str(), GetMetricType(build_cfg->metric_type)); - std::shared_ptr new_index; - new_index.reset(host_index); - return std::make_shared(new_index); + auto temp_resource = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_); + if (temp_resource != nullptr) { + ResScope rs(temp_resource, gpu_id_, true); + auto device_index = faiss::gpu::index_cpu_to_gpu(temp_resource->faiss_res.get(), gpu_id_, build_index); + device_index->train(rows, (float*)p_data); + + std::shared_ptr host_index = nullptr; + host_index.reset(faiss::gpu::index_gpu_to_cpu(device_index)); + + delete device_index; + delete build_index; + + return std::make_shared(host_index); + } else { + KNOWHERE_THROW_MSG("Build IVFSQ can't get gpu resource"); } +} -} // knowhere -} // zilliz +VectorIndexPtr +GPUIVFSQ::CopyGpuToCpu(const Config& config) { + std::lock_guard lk(mutex_); + faiss::Index* device_index = index_.get(); + faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(device_index); + + std::shared_ptr new_index; + new_index.reset(host_index); + return std::make_shared(new_index); +} + +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.h index 677907964b..5c6b4a038b 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.h @@ -15,29 +15,31 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include "IndexGPUIVF.h" +#include +#include +#include "IndexGPUIVF.h" namespace zilliz { namespace knowhere { class GPUIVFSQ : public GPUIVF { -public: - explicit GPUIVFSQ(const int &device_id) : GPUIVF(device_id) {} + public: + explicit GPUIVFSQ(const int& device_id) : GPUIVF(device_id) { + } - explicit GPUIVFSQ(std::shared_ptr index, const int64_t &device_id, ResPtr &resource) - : GPUIVF(std::move(index), device_id, resource) {}; + explicit GPUIVFSQ(std::shared_ptr index, const int64_t& device_id, ResPtr& resource) + : GPUIVF(std::move(index), device_id, resource) { + } IndexModelPtr - Train(const DatasetPtr &dataset, const Config &config) override; + Train(const DatasetPtr& dataset, const Config& config) override; VectorIndexPtr - CopyGpuToCpu(const Config &config) override; + CopyGpuToCpu(const Config& config) override; }; -} // knowhere -} // zilliz - +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp index f5e3fd8f40..8d4ff16a1f 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp @@ -15,24 +15,23 @@ // specific language governing permissions and limitations // under the License. - -#include #include +#include #include -#include #include +#include +#include - -#include "knowhere/common/Exception.h" #include "knowhere/adapter/VectorAdapter.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/IndexIDMAP.h" #include "knowhere/index/vector_index/helpers/FaissIO.h" -#include "IndexIDMAP.h" - namespace zilliz { namespace knowhere { -BinarySet IDMAP::Serialize() { +BinarySet +IDMAP::Serialize() { if (!index_) { KNOWHERE_THROW_MSG("index not initialize"); } @@ -41,31 +40,33 @@ BinarySet IDMAP::Serialize() { return SerializeImpl(); } -void IDMAP::Load(const BinarySet &index_binary) { +void +IDMAP::Load(const BinarySet& index_binary) { std::lock_guard lk(mutex_); LoadImpl(index_binary); } -DatasetPtr IDMAP::Search(const DatasetPtr &dataset, const Config &config) { +DatasetPtr +IDMAP::Search(const DatasetPtr& dataset, const Config& config) { if (!index_) { KNOWHERE_THROW_MSG("index not initialize"); } config->CheckValid(); - //auto metric_type = config["metric_type"].as_string() == "L2" ? + // auto metric_type = config["metric_type"].as_string() == "L2" ? // faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; - //index_->metric_type = metric_type; + // index_->metric_type = metric_type; GETTENSOR(dataset) auto elems = rows * config->k; - auto res_ids = (int64_t *) malloc(sizeof(int64_t) * elems); - auto res_dis = (float *) malloc(sizeof(float) * elems); + auto res_ids = (int64_t*)malloc(sizeof(int64_t) * elems); + auto res_dis = (float*)malloc(sizeof(float) * elems); - search_impl(rows, (float *) p_data, config->k, res_dis, res_ids, Config()); + search_impl(rows, (float*)p_data, config->k, res_dis, res_ids, Config()); - auto id_buf = MakeMutableBufferSmart((uint8_t *) res_ids, sizeof(int64_t) * elems); - auto dist_buf = MakeMutableBufferSmart((uint8_t *) res_dis, sizeof(float) * elems); + auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); + auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); std::vector id_bufs{nullptr, id_buf}; std::vector dist_bufs{nullptr, dist_buf}; @@ -83,12 +84,13 @@ DatasetPtr IDMAP::Search(const DatasetPtr &dataset, const Config &config) { return std::make_shared(array, nullptr); } -void IDMAP::search_impl(int64_t n, const float *data, int64_t k, float *distances, int64_t *labels, const Config &cfg) { - index_->search(n, (float *) data, k, distances, labels); - +void +IDMAP::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) { + index_->search(n, (float*)data, k, distances, labels); } -void IDMAP::Add(const DatasetPtr &dataset, const Config &config) { +void +IDMAP::Add(const DatasetPtr& dataset, const Config& config) { if (!index_) { KNOWHERE_THROW_MSG("index not initialize"); } @@ -98,49 +100,56 @@ void IDMAP::Add(const DatasetPtr &dataset, const Config &config) { // TODO: magic here. auto array = dataset->array()[0]; - auto p_ids = array->data()->GetValues(1, 0); + auto p_ids = array->data()->GetValues(1, 0); - index_->add_with_ids(rows, (float *) p_data, p_ids); + index_->add_with_ids(rows, (float*)p_data, p_ids); } -int64_t IDMAP::Count() { +int64_t +IDMAP::Count() { return index_->ntotal; } -int64_t IDMAP::Dimension() { +int64_t +IDMAP::Dimension() { return index_->d; } // TODO(linxj): return const pointer -float *IDMAP::GetRawVectors() { +float* +IDMAP::GetRawVectors() { try { - auto file_index = dynamic_cast(index_.get()); + auto file_index = dynamic_cast(index_.get()); auto flat_index = dynamic_cast(file_index->index); return flat_index->xb.data(); - } catch (std::exception &e) { + } catch (std::exception& e) { KNOWHERE_THROW_MSG(e.what()); } } // TODO(linxj): return const pointer -int64_t *IDMAP::GetRawIds() { +int64_t* +IDMAP::GetRawIds() { try { - auto file_index = dynamic_cast(index_.get()); + auto file_index = dynamic_cast(index_.get()); return file_index->id_map.data(); - } catch (std::exception &e) { + } catch (std::exception& e) { KNOWHERE_THROW_MSG(e.what()); } } const char* type = "IDMap,Flat"; -void IDMAP::Train(const Config &config) { + +void +IDMAP::Train(const Config& config) { config->CheckValid(); auto index = faiss::index_factory(config->d, type, GetMetricType(config->metric_type)); index_.reset(index); } -VectorIndexPtr IDMAP::Clone() { +VectorIndexPtr +IDMAP::Clone() { std::lock_guard lk(mutex_); auto clone_index = faiss::clone_index(index_.get()); @@ -149,8 +158,9 @@ VectorIndexPtr IDMAP::Clone() { return std::make_shared(new_index); } -VectorIndexPtr IDMAP::CopyCpuToGpu(const int64_t &device_id, const Config &config) { - if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)){ +VectorIndexPtr +IDMAP::CopyCpuToGpu(const int64_t& device_id, const Config& config) { + if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) { ResScope rs(res, device_id, false); auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get()); @@ -162,38 +172,41 @@ VectorIndexPtr IDMAP::CopyCpuToGpu(const int64_t &device_id, const Config &confi } } -void IDMAP::Seal() { +void +IDMAP::Seal() { // do nothing } -VectorIndexPtr GPUIDMAP::CopyGpuToCpu(const Config &config) { +VectorIndexPtr +GPUIDMAP::CopyGpuToCpu(const Config& config) { std::lock_guard lk(mutex_); - faiss::Index *device_index = index_.get(); - faiss::Index *host_index = faiss::gpu::index_gpu_to_cpu(device_index); + faiss::Index* device_index = index_.get(); + faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(device_index); std::shared_ptr new_index; new_index.reset(host_index); return std::make_shared(new_index); } -VectorIndexPtr GPUIDMAP::Clone() { +VectorIndexPtr +GPUIDMAP::Clone() { auto cpu_idx = CopyGpuToCpu(Config()); - if (auto idmap = std::dynamic_pointer_cast(cpu_idx)){ + if (auto idmap = std::dynamic_pointer_cast(cpu_idx)) { return idmap->CopyCpuToGpu(gpu_id_, Config()); - } - else { + } else { KNOWHERE_THROW_MSG("IndexType not Support GpuClone"); } } -BinarySet GPUIDMAP::SerializeImpl() { +BinarySet +GPUIDMAP::SerializeImpl() { try { MemoryIOWriter writer; { - faiss::Index *index = index_.get(); - faiss::Index *host_index = faiss::gpu::index_gpu_to_cpu(index); + faiss::Index* index = index_.get(); + faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(index); faiss::write_index(host_index, &writer); delete host_index; @@ -205,21 +218,22 @@ BinarySet GPUIDMAP::SerializeImpl() { res_set.Append("IVF", data, writer.rp); return res_set; - } catch (std::exception &e) { + } catch (std::exception& e) { KNOWHERE_THROW_MSG(e.what()); } } -void GPUIDMAP::LoadImpl(const BinarySet &index_binary) { +void +GPUIDMAP::LoadImpl(const BinarySet& index_binary) { auto binary = index_binary.GetByName("IVF"); MemoryIOReader reader; { reader.total = binary->size; reader.data_ = binary->data.get(); - faiss::Index *index = faiss::read_index(&reader); + faiss::Index* index = faiss::read_index(&reader); - if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_) ){ + if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_)) { ResScope rs(res, gpu_id_, false); auto device_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), gpu_id_, index); index_.reset(device_index); @@ -232,28 +246,27 @@ void GPUIDMAP::LoadImpl(const BinarySet &index_binary) { } } -VectorIndexPtr GPUIDMAP::CopyGpuToGpu(const int64_t &device_id, const Config &config) { +VectorIndexPtr +GPUIDMAP::CopyGpuToGpu(const int64_t& device_id, const Config& config) { auto cpu_index = CopyGpuToCpu(config); return std::static_pointer_cast(cpu_index)->CopyCpuToGpu(device_id, config); } -float *GPUIDMAP::GetRawVectors() { +float* +GPUIDMAP::GetRawVectors() { KNOWHERE_THROW_MSG("Not support"); } -int64_t *GPUIDMAP::GetRawIds() { +int64_t* +GPUIDMAP::GetRawIds() { KNOWHERE_THROW_MSG("Not support"); } -void GPUIDMAP::search_impl(int64_t n, - const float *data, - int64_t k, - float *distances, - int64_t *labels, - const Config &cfg) { +void +GPUIDMAP::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) { ResScope rs(res_, gpu_id_); - index_->search(n, (float *) data, k, distances, labels); + index_->search(n, (float*)data, k, distances, labels); } -} -} +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.h index 106759faf1..fd13b87e34 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.h @@ -15,41 +15,54 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include "IndexIVF.h" #include "IndexGPUIVF.h" +#include "IndexIVF.h" +#include +#include namespace zilliz { namespace knowhere { class IDMAP : public VectorIndex, public FaissBaseIndex { public: - IDMAP() : FaissBaseIndex(nullptr) {}; - explicit IDMAP(std::shared_ptr index) : FaissBaseIndex(std::move(index)) {}; - BinarySet Serialize() override; - void Load(const BinarySet &index_binary) override; - void Train(const Config &config); - DatasetPtr Search(const DatasetPtr &dataset, const Config &config) override; - int64_t Count() override; - VectorIndexPtr Clone() override; - int64_t Dimension() override; - void Add(const DatasetPtr &dataset, const Config &config) override; - VectorIndexPtr CopyCpuToGpu(const int64_t &device_id, const Config &config); - void Seal() override; + IDMAP() : FaissBaseIndex(nullptr) { + } - virtual float *GetRawVectors(); - virtual int64_t *GetRawIds(); + explicit IDMAP(std::shared_ptr index) : FaissBaseIndex(std::move(index)) { + } + + BinarySet + Serialize() override; + void + Load(const BinarySet& index_binary) override; + void + Train(const Config& config); + DatasetPtr + Search(const DatasetPtr& dataset, const Config& config) override; + int64_t + Count() override; + VectorIndexPtr + Clone() override; + int64_t + Dimension() override; + void + Add(const DatasetPtr& dataset, const Config& config) override; + VectorIndexPtr + CopyCpuToGpu(const int64_t& device_id, const Config& config); + void + Seal() override; + + virtual float* + GetRawVectors(); + virtual int64_t* + GetRawIds(); protected: - virtual void search_impl(int64_t n, - const float *data, - int64_t k, - float *distances, - int64_t *labels, - const Config &cfg); + virtual void + search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg); std::mutex mutex_; }; @@ -57,27 +70,31 @@ using IDMAPPtr = std::shared_ptr; class GPUIDMAP : public IDMAP, public GPUIndex { public: - explicit GPUIDMAP(std::shared_ptr index, const int64_t &device_id, ResPtr& res) - : IDMAP(std::move(index)), GPUIndex(device_id, res) {} + explicit GPUIDMAP(std::shared_ptr index, const int64_t& device_id, ResPtr& res) + : IDMAP(std::move(index)), GPUIndex(device_id, res) { + } - VectorIndexPtr CopyGpuToCpu(const Config &config) override; - float *GetRawVectors() override; - int64_t *GetRawIds() override; - VectorIndexPtr Clone() override; - VectorIndexPtr CopyGpuToGpu(const int64_t &device_id, const Config &config) override; + VectorIndexPtr + CopyGpuToCpu(const Config& config) override; + float* + GetRawVectors() override; + int64_t* + GetRawIds() override; + VectorIndexPtr + Clone() override; + VectorIndexPtr + CopyGpuToGpu(const int64_t& device_id, const Config& config) override; protected: - void search_impl(int64_t n, - const float *data, - int64_t k, - float *distances, - int64_t *labels, - const Config &cfg) override; - BinarySet SerializeImpl() override; - void LoadImpl(const BinarySet &index_binary) override; + void + search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) override; + BinarySet + SerializeImpl() override; + void + LoadImpl(const BinarySet& index_binary) override; }; using GPUIDMAPPtr = std::shared_ptr; -} -} +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp index 2e6d0e9763..13805ce14d 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -15,47 +15,47 @@ // specific language governing permissions and limitations // under the License. +#include +#include +#include #include #include #include #include -#include -#include -#include -#include #include +#include +#include +#include +#include - -#include "knowhere/common/Exception.h" #include "knowhere/adapter/VectorAdapter.h" -#include "IndexIVF.h" -#include "IndexGPUIVF.h" - +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexIVF.h" namespace zilliz { namespace knowhere { - -IndexModelPtr IVF::Train(const DatasetPtr &dataset, const Config &config) { +IndexModelPtr +IVF::Train(const DatasetPtr& dataset, const Config& config) { auto build_cfg = std::dynamic_pointer_cast(config); if (build_cfg != nullptr) { - build_cfg->CheckValid(); // throw exception + build_cfg->CheckValid(); // throw exception } GETTENSOR(dataset) - faiss::Index *coarse_quantizer = new faiss::IndexFlatL2(dim); - auto index = std::make_shared(coarse_quantizer, dim, - build_cfg->nlist, + faiss::Index* coarse_quantizer = new faiss::IndexFlatL2(dim); + auto index = std::make_shared(coarse_quantizer, dim, build_cfg->nlist, GetMetricType(build_cfg->metric_type)); - index->train(rows, (float *) p_data); + index->train(rows, (float*)p_data); // TODO(linxj): override here. train return model or not. return std::make_shared(index); } - -void IVF::Add(const DatasetPtr &dataset, const Config &config) { +void +IVF::Add(const DatasetPtr& dataset, const Config& config) { if (!index_ || !index_->is_trained) { KNOWHERE_THROW_MSG("index not initialize or trained"); } @@ -64,11 +64,12 @@ void IVF::Add(const DatasetPtr &dataset, const Config &config) { GETTENSOR(dataset) auto array = dataset->array()[0]; - auto p_ids = array->data()->GetValues(1, 0); - index_->add_with_ids(rows, (float *) p_data, p_ids); + auto p_ids = array->data()->GetValues(1, 0); + index_->add_with_ids(rows, (float*)p_data, p_ids); } -void IVF::AddWithoutIds(const DatasetPtr &dataset, const Config &config) { +void +IVF::AddWithoutIds(const DatasetPtr& dataset, const Config& config) { if (!index_ || !index_->is_trained) { KNOWHERE_THROW_MSG("index not initialize or trained"); } @@ -76,10 +77,11 @@ void IVF::AddWithoutIds(const DatasetPtr &dataset, const Config &config) { std::lock_guard lk(mutex_); GETTENSOR(dataset) - index_->add(rows, (float *) p_data); + index_->add(rows, (float*)p_data); } -BinarySet IVF::Serialize() { +BinarySet +IVF::Serialize() { if (!index_ || !index_->is_trained) { KNOWHERE_THROW_MSG("index not initialize or trained"); } @@ -89,31 +91,33 @@ BinarySet IVF::Serialize() { return SerializeImpl(); } -void IVF::Load(const BinarySet &index_binary) { +void +IVF::Load(const BinarySet& index_binary) { std::lock_guard lk(mutex_); LoadImpl(index_binary); } -DatasetPtr IVF::Search(const DatasetPtr &dataset, const Config &config) { +DatasetPtr +IVF::Search(const DatasetPtr& dataset, const Config& config) { if (!index_ || !index_->is_trained) { KNOWHERE_THROW_MSG("index not initialize or trained"); } auto search_cfg = std::dynamic_pointer_cast(config); if (search_cfg != nullptr) { - search_cfg->CheckValid(); // throw exception + search_cfg->CheckValid(); // throw exception } GETTENSOR(dataset) auto elems = rows * search_cfg->k; - auto res_ids = (int64_t *) malloc(sizeof(int64_t) * elems); - auto res_dis = (float *) malloc(sizeof(float) * elems); + auto res_ids = (int64_t*)malloc(sizeof(int64_t) * elems); + auto res_dis = (float*)malloc(sizeof(float) * elems); - search_impl(rows, (float*) p_data, search_cfg->k, res_dis, res_ids, config); + search_impl(rows, (float*)p_data, search_cfg->k, res_dis, res_ids, config); - auto id_buf = MakeMutableBufferSmart((uint8_t *) res_ids, sizeof(int64_t) * elems); - auto dist_buf = MakeMutableBufferSmart((uint8_t *) res_dis, sizeof(float) * elems); + auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); + auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); std::vector id_bufs{nullptr, id_buf}; std::vector dist_bufs{nullptr, dist_buf}; @@ -131,7 +135,8 @@ DatasetPtr IVF::Search(const DatasetPtr &dataset, const Config &config) { return std::make_shared(array, nullptr); } -void IVF::set_index_model(IndexModelPtr model) { +void +IVF::set_index_model(IndexModelPtr model) { std::lock_guard lk(mutex_); auto rel_model = std::static_pointer_cast(model); @@ -140,25 +145,29 @@ void IVF::set_index_model(IndexModelPtr model) { index_.reset(faiss::clone_index(rel_model->index_.get())); } -std::shared_ptr IVF::GenParams(const Config &config) { +std::shared_ptr +IVF::GenParams(const Config& config) { auto params = std::make_shared(); auto search_cfg = std::dynamic_pointer_cast(config); params->nprobe = search_cfg->nprobe; - //params->max_codes = config.get_with_default("max_codes", size_t(0)); + // params->max_codes = config.get_with_default("max_codes", size_t(0)); return params; } -int64_t IVF::Count() { +int64_t +IVF::Count() { return index_->ntotal; } -int64_t IVF::Dimension() { +int64_t +IVF::Dimension() { return index_->d; } -void IVF::GenGraph(const int64_t &k, Graph &graph, const DatasetPtr &dataset, const Config &config) { +void +IVF::GenGraph(const int64_t& k, Graph& graph, const DatasetPtr& dataset, const Config& config) { GETTENSOR(dataset) auto ntotal = Count(); @@ -174,7 +183,7 @@ void IVF::GenGraph(const int64_t &k, Graph &graph, const DatasetPtr &dataset, co for (int i = 0; i < total_search_count; ++i) { auto b_size = i == total_search_count - 1 && tail_batch_size != 0 ? tail_batch_size : batch_size; - auto &res = res_vec[i]; + auto& res = res_vec[i]; res.resize(k * b_size); auto xq = p_data + batch_size * dim * i; @@ -182,7 +191,7 @@ void IVF::GenGraph(const int64_t &k, Graph &graph, const DatasetPtr &dataset, co int tmp = 0; for (int j = 0; j < b_size; ++j) { - auto &node = graph[batch_size * i + j]; + auto& node = graph[batch_size * i + j]; node.resize(k); for (int m = 0; m < k && tmp < k * b_size; ++m, ++tmp) { // TODO(linxj): avoid memcopy here. @@ -192,18 +201,15 @@ void IVF::GenGraph(const int64_t &k, Graph &graph, const DatasetPtr &dataset, co } } -void IVF::search_impl(int64_t n, - const float *data, - int64_t k, - float *distances, - int64_t *labels, - const Config &cfg) { +void +IVF::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) { auto params = GenParams(cfg); - faiss::ivflib::search_with_parameters(index_.get(), n, (float *) data, k, distances, labels, params.get()); + faiss::ivflib::search_with_parameters(index_.get(), n, (float*)data, k, distances, labels, params.get()); } -VectorIndexPtr IVF::CopyCpuToGpu(const int64_t& device_id, const Config &config) { - if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)){ +VectorIndexPtr +IVF::CopyCpuToGpu(const int64_t& device_id, const Config& config) { + if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) { ResScope rs(res, device_id, false); auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get()); @@ -215,7 +221,8 @@ VectorIndexPtr IVF::CopyCpuToGpu(const int64_t& device_id, const Config &config) } } -VectorIndexPtr IVF::Clone() { +VectorIndexPtr +IVF::Clone() { std::lock_guard lk(mutex_); auto clone_index = faiss::clone_index(index_.get()); @@ -224,21 +231,24 @@ VectorIndexPtr IVF::Clone() { return Clone_impl(new_index); } -VectorIndexPtr IVF::Clone_impl(const std::shared_ptr &index) { +VectorIndexPtr +IVF::Clone_impl(const std::shared_ptr& index) { return std::make_shared(index); } -void IVF::Seal() { +void +IVF::Seal() { if (!index_ || !index_->is_trained) { KNOWHERE_THROW_MSG("index not initialize or trained"); } SealImpl(); } +IVFIndexModel::IVFIndexModel(std::shared_ptr index) : FaissBaseIndex(std::move(index)) { +} -IVFIndexModel::IVFIndexModel(std::shared_ptr index) : FaissBaseIndex(std::move(index)) {} - -BinarySet IVFIndexModel::Serialize() { +BinarySet +IVFIndexModel::Serialize() { if (!index_ || !index_->is_trained) { KNOWHERE_THROW_MSG("indexmodel not initialize or trained"); } @@ -246,18 +256,16 @@ BinarySet IVFIndexModel::Serialize() { return SerializeImpl(); } -void IVFIndexModel::Load(const BinarySet &binary_set) { +void +IVFIndexModel::Load(const BinarySet& binary_set) { std::lock_guard lk(mutex_); LoadImpl(binary_set); } -void IVFIndexModel::SealImpl() { +void +IVFIndexModel::SealImpl() { // do nothing } - - - - -} -} +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.h index 72f98a5afa..088849c22a 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.h @@ -15,17 +15,17 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include #include +#include +#include -#include "VectorIndex.h" #include "FaissBaseIndex.h" +#include "VectorIndex.h" #include "faiss/IndexIVF.h" - namespace zilliz { namespace knowhere { @@ -33,36 +33,38 @@ using Graph = std::vector>; class IVF : public VectorIndex, protected FaissBaseIndex { public: - IVF() : FaissBaseIndex(nullptr) {}; + IVF() : FaissBaseIndex(nullptr) { + } - explicit IVF(std::shared_ptr index) : FaissBaseIndex(std::move(index)) {} + explicit IVF(std::shared_ptr index) : FaissBaseIndex(std::move(index)) { + } VectorIndexPtr - Clone() override;; + Clone() override; IndexModelPtr - Train(const DatasetPtr &dataset, const Config &config) override; + Train(const DatasetPtr& dataset, const Config& config) override; void set_index_model(IndexModelPtr model) override; void - Add(const DatasetPtr &dataset, const Config &config) override; + Add(const DatasetPtr& dataset, const Config& config) override; void - AddWithoutIds(const DatasetPtr &dataset, const Config &config); + AddWithoutIds(const DatasetPtr& dataset, const Config& config); DatasetPtr - Search(const DatasetPtr &dataset, const Config &config) override; + Search(const DatasetPtr& dataset, const Config& config) override; void - GenGraph(const int64_t &k, Graph &graph, const DatasetPtr &dataset, const Config &config); + GenGraph(const int64_t& k, Graph& graph, const DatasetPtr& dataset, const Config& config); BinarySet Serialize() override; void - Load(const BinarySet &index_binary) override; + Load(const BinarySet& index_binary) override; int64_t Count() override; @@ -74,23 +76,17 @@ class IVF : public VectorIndex, protected FaissBaseIndex { Seal() override; virtual VectorIndexPtr - CopyCpuToGpu(const int64_t &device_id, const Config &config); - + CopyCpuToGpu(const int64_t& device_id, const Config& config); protected: virtual std::shared_ptr - GenParams(const Config &config); + GenParams(const Config& config); virtual VectorIndexPtr - Clone_impl(const std::shared_ptr &index); + Clone_impl(const std::shared_ptr& index); virtual void - search_impl(int64_t n, - const float *data, - int64_t k, - float *distances, - int64_t *labels, - const Config &cfg); + search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg); protected: std::mutex mutex_; @@ -106,13 +102,14 @@ class IVFIndexModel : public IndexModel, public FaissBaseIndex { public: explicit IVFIndexModel(std::shared_ptr index); - IVFIndexModel() : FaissBaseIndex(nullptr) {}; + IVFIndexModel() : FaissBaseIndex(nullptr) { + } BinarySet Serialize() override; void - Load(const BinarySet &binary) override; + Load(const BinarySet& binary) override; protected: void @@ -121,7 +118,8 @@ class IVFIndexModel : public IndexModel, public FaissBaseIndex { protected: std::mutex mutex_; }; + using IVFIndexModelPtr = std::shared_ptr; -} -} \ No newline at end of file +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp index b5418b77cf..c0eda94310 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp @@ -15,47 +15,51 @@ // specific language governing permissions and limitations // under the License. - #include #include +#include +#include -#include "IndexIVFPQ.h" -#include "knowhere/common/Exception.h" #include "knowhere/adapter/VectorAdapter.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/IndexIVFPQ.h" namespace zilliz { namespace knowhere { -IndexModelPtr IVFPQ::Train(const DatasetPtr &dataset, const Config &config) { +IndexModelPtr +IVFPQ::Train(const DatasetPtr& dataset, const Config& config) { auto build_cfg = std::dynamic_pointer_cast(config); if (build_cfg != nullptr) { - build_cfg->CheckValid(); // throw exception + build_cfg->CheckValid(); // throw exception } GETTENSOR(dataset) - faiss::Index *coarse_quantizer = new faiss::IndexFlat(dim, GetMetricType(build_cfg->metric_type)); - auto index = std::make_shared(coarse_quantizer, dim, - build_cfg->nlist, build_cfg->m, build_cfg->nbits); - index->train(rows, (float *) p_data); + faiss::Index* coarse_quantizer = new faiss::IndexFlat(dim, GetMetricType(build_cfg->metric_type)); + auto index = + std::make_shared(coarse_quantizer, dim, build_cfg->nlist, build_cfg->m, build_cfg->nbits); + index->train(rows, (float*)p_data); return std::make_shared(index); } -std::shared_ptr IVFPQ::GenParams(const Config &config) { +std::shared_ptr +IVFPQ::GenParams(const Config& config) { auto params = std::make_shared(); auto search_cfg = std::dynamic_pointer_cast(config); params->nprobe = search_cfg->nprobe; -// params->scan_table_threshold = conf->scan_table_threhold; -// params->polysemous_ht = conf->polysemous_ht; -// params->max_codes = conf->max_codes; + // params->scan_table_threshold = conf->scan_table_threhold; + // params->polysemous_ht = conf->polysemous_ht; + // params->max_codes = conf->max_codes; return params; } -VectorIndexPtr IVFPQ::Clone_impl(const std::shared_ptr &index) { +VectorIndexPtr +IVFPQ::Clone_impl(const std::shared_ptr& index) { return std::make_shared(index); } -} // knowhere -} // zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.h index 427f76fe3e..f071c1e0fa 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.h @@ -15,33 +15,33 @@ // specific language governing permissions and limitations // under the License. - #pragma once +#include +#include + #include "IndexIVF.h" namespace zilliz { namespace knowhere { class IVFPQ : public IVF { -public: - explicit IVFPQ(std::shared_ptr index) : IVF(std::move(index)) {} + public: + explicit IVFPQ(std::shared_ptr index) : IVF(std::move(index)) { + } IVFPQ() = default; IndexModelPtr - Train(const DatasetPtr &dataset, const Config &config) override; + Train(const DatasetPtr& dataset, const Config& config) override; -protected: + protected: std::shared_ptr - GenParams(const Config &config) override; + GenParams(const Config& config) override; VectorIndexPtr - Clone_impl(const std::shared_ptr &index) override; + Clone_impl(const std::shared_ptr& index) override; }; -} // knowhere -} // zilliz - - - +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp index ec29627ebc..5406211fa3 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp @@ -15,44 +15,46 @@ // specific language governing permissions and limitations // under the License. - #include +#include -#include "knowhere/common/Exception.h" -#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #include "knowhere/adapter/VectorAdapter.h" -#include "IndexIVFSQ.h" -#include "IndexGPUIVFSQ.h" - +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/IndexGPUIVFSQ.h" +#include "knowhere/index/vector_index/IndexIVFSQ.h" +#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" namespace zilliz { namespace knowhere { -IndexModelPtr IVFSQ::Train(const DatasetPtr &dataset, const Config &config) { +IndexModelPtr +IVFSQ::Train(const DatasetPtr& dataset, const Config& config) { auto build_cfg = std::dynamic_pointer_cast(config); if (build_cfg != nullptr) { - build_cfg->CheckValid(); // throw exception + build_cfg->CheckValid(); // throw exception } GETTENSOR(dataset) std::stringstream index_type; - index_type << "IVF" << build_cfg->nlist << "," << "SQ" << build_cfg->nbits; - auto build_index = faiss::index_factory(dim, index_type.str().c_str(), - GetMetricType(build_cfg->metric_type)); - build_index->train(rows, (float *) p_data); + index_type << "IVF" << build_cfg->nlist << "," + << "SQ" << build_cfg->nbits; + auto build_index = faiss::index_factory(dim, index_type.str().c_str(), GetMetricType(build_cfg->metric_type)); + build_index->train(rows, (float*)p_data); std::shared_ptr ret_index; ret_index.reset(build_index); return std::make_shared(ret_index); } -VectorIndexPtr IVFSQ::Clone_impl(const std::shared_ptr &index) { +VectorIndexPtr +IVFSQ::Clone_impl(const std::shared_ptr& index) { return std::make_shared(index); } -VectorIndexPtr IVFSQ::CopyCpuToGpu(const int64_t &device_id, const Config &config) { - if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)){ +VectorIndexPtr +IVFSQ::CopyCpuToGpu(const int64_t& device_id, const Config& config) { + if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) { ResScope rs(res, device_id, false); faiss::gpu::GpuClonerOptions option; option.allInGpu = true; @@ -67,5 +69,5 @@ VectorIndexPtr IVFSQ::CopyCpuToGpu(const int64_t &device_id, const Config &confi } } -} // knowhere -} // zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.h index 5ee771f951..3a05ca9826 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.h @@ -15,31 +15,33 @@ // specific language governing permissions and limitations // under the License. - #pragma once +#include +#include + #include "IndexIVF.h" namespace zilliz { namespace knowhere { class IVFSQ : public IVF { -public: - explicit IVFSQ(std::shared_ptr index) : IVF(std::move(index)) {} + public: + explicit IVFSQ(std::shared_ptr index) : IVF(std::move(index)) { + } IVFSQ() = default; IndexModelPtr - Train(const DatasetPtr &dataset, const Config &config) override; + Train(const DatasetPtr& dataset, const Config& config) override; VectorIndexPtr - CopyCpuToGpu(const int64_t &device_id, const Config &config) override; + CopyCpuToGpu(const int64_t& device_id, const Config& config) override; -protected: + protected: VectorIndexPtr - Clone_impl(const std::shared_ptr &index) override; + Clone_impl(const std::shared_ptr& index) override; }; -} // knowhere -} // zilliz - +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp index cceb88516a..6efb839384 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp @@ -15,41 +15,39 @@ // specific language governing permissions and limitations // under the License. - -#include -#include -#include #include - +#include +#include +#include +#include #undef mkdir -#include "IndexKDT.h" +#include "knowhere/index/vector_index/IndexKDT.h" #include "knowhere/index/vector_index/helpers/Definitions.h" //#include "knowhere/index/preprocessor/normalize.h" -#include "knowhere/index/vector_index/helpers/KDTParameterMgr.h" #include "knowhere/adapter/SptagAdapter.h" #include "knowhere/common/Exception.h" - +#include "knowhere/index/vector_index/helpers/KDTParameterMgr.h" namespace zilliz { namespace knowhere { BinarySet CPUKDTRNG::Serialize() { - std::vector index_blobs; + std::vector index_blobs; std::vector index_len; index_ptr_->SaveIndexToMemory(index_blobs, index_len); BinarySet binary_set; auto sample = std::make_shared(); - sample.reset(static_cast(index_blobs[0])); + sample.reset(static_cast(index_blobs[0])); auto tree = std::make_shared(); - tree.reset(static_cast(index_blobs[1])); + tree.reset(static_cast(index_blobs[1])); auto graph = std::make_shared(); - graph.reset(static_cast(index_blobs[2])); + graph.reset(static_cast(index_blobs[2])); auto metadata = std::make_shared(); - metadata.reset(static_cast(index_blobs[3])); + metadata.reset(static_cast(index_blobs[3])); binary_set.Append("samples", sample, index_len[0]); binary_set.Append("tree", tree, index_len[1]); @@ -59,8 +57,8 @@ CPUKDTRNG::Serialize() { } void -CPUKDTRNG::Load(const BinarySet &binary_set) { - std::vector index_blobs; +CPUKDTRNG::Load(const BinarySet& binary_set) { + std::vector index_blobs; auto samples = binary_set.GetByName("samples"); index_blobs.push_back(samples->data.get()); @@ -77,17 +75,17 @@ CPUKDTRNG::Load(const BinarySet &binary_set) { index_ptr_->LoadIndexFromMemory(index_blobs); } -//PreprocessorPtr -//CPUKDTRNG::BuildPreprocessor(const DatasetPtr &dataset, const Config &config) { +// PreprocessorPtr +// CPUKDTRNG::BuildPreprocessor(const DatasetPtr &dataset, const Config &config) { // return std::make_shared(); //} IndexModelPtr -CPUKDTRNG::Train(const DatasetPtr &origin, const Config &train_config) { +CPUKDTRNG::Train(const DatasetPtr& origin, const Config& train_config) { SetParameters(train_config); DatasetPtr dataset = origin->Clone(); - //if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine + // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine // && preprocessor_) { // preprocessor_->Preprocess(dataset); //} @@ -101,11 +99,11 @@ CPUKDTRNG::Train(const DatasetPtr &origin, const Config &train_config) { } void -CPUKDTRNG::Add(const DatasetPtr &origin, const Config &add_config) { +CPUKDTRNG::Add(const DatasetPtr& origin, const Config& add_config) { SetParameters(add_config); DatasetPtr dataset = origin->Clone(); - //if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine + // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine // && preprocessor_) { // preprocessor_->Preprocess(dataset); //} @@ -116,18 +114,18 @@ CPUKDTRNG::Add(const DatasetPtr &origin, const Config &add_config) { } void -CPUKDTRNG::SetParameters(const Config &config) { - for (auto ¶ : KDTParameterMgr::GetInstance().GetKDTParameters()) { -// auto value = config.get_with_default(para.first, para.second); +CPUKDTRNG::SetParameters(const Config& config) { + for (auto& para : KDTParameterMgr::GetInstance().GetKDTParameters()) { + // auto value = config.get_with_default(para.first, para.second); index_ptr_->SetParameter(para.first, para.second); } } DatasetPtr -CPUKDTRNG::Search(const DatasetPtr &dataset, const Config &config) { +CPUKDTRNG::Search(const DatasetPtr& dataset, const Config& config) { SetParameters(config); auto tensor = dataset->tensor()[0]; - auto p = (float *) tensor->raw_mutable_data(); + auto p = (float*)tensor->raw_mutable_data(); for (auto i = 0; i < 10; ++i) { for (auto j = 0; j < 10; ++j) { std::cout << p[i * 10 + j] << " "; @@ -138,7 +136,7 @@ CPUKDTRNG::Search(const DatasetPtr &dataset, const Config &config) { #pragma omp parallel for for (auto i = 0; i < query_results.size(); ++i) { - auto target = (float *) query_results[i].GetTarget(); + auto target = (float*)query_results[i].GetTarget(); std::cout << target[0] << ", " << target[1] << ", " << target[2] << std::endl; index_ptr_->SearchIndex(query_results[i]); } @@ -146,27 +144,34 @@ CPUKDTRNG::Search(const DatasetPtr &dataset, const Config &config) { return ConvertToDataset(query_results); } -int64_t CPUKDTRNG::Count() { +int64_t +CPUKDTRNG::Count() { index_ptr_->GetNumSamples(); } -int64_t CPUKDTRNG::Dimension() { + +int64_t +CPUKDTRNG::Dimension() { index_ptr_->GetFeatureDim(); } -VectorIndexPtr CPUKDTRNG::Clone() { +VectorIndexPtr +CPUKDTRNG::Clone() { KNOWHERE_THROW_MSG("not support"); } -void CPUKDTRNG::Seal() { +void +CPUKDTRNG::Seal() { // do nothing } // TODO(linxj): BinarySet -CPUKDTRNGIndexModel::Serialize() {} +CPUKDTRNGIndexModel::Serialize() { +} void -CPUKDTRNGIndexModel::Load(const BinarySet &binary) {} +CPUKDTRNGIndexModel::Load(const BinarySet& binary) { +} -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.h index 7fd41e9bad..86737ac3dd 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.h @@ -15,53 +15,54 @@ // specific language governing permissions and limitations // under the License. - #pragma once +#include #include #include #include "VectorIndex.h" #include "knowhere/index/IndexModel.h" -#include - namespace zilliz { namespace knowhere { - class CPUKDTRNG : public VectorIndex { public: CPUKDTRNG() { - index_ptr_ = SPTAG::VectorIndex::CreateInstance(SPTAG::IndexAlgoType::KDT, - SPTAG::VectorValueType::Float); + index_ptr_ = SPTAG::VectorIndex::CreateInstance(SPTAG::IndexAlgoType::KDT, SPTAG::VectorValueType::Float); index_ptr_->SetParameter("DistCalcMethod", "L2"); } public: BinarySet Serialize() override; - VectorIndexPtr Clone() override; + VectorIndexPtr + Clone() override; void - Load(const BinarySet &index_array) override; + Load(const BinarySet& index_array) override; public: - //PreprocessorPtr - //BuildPreprocessor(const DatasetPtr &dataset, const Config &config) override; - int64_t Count() override; - int64_t Dimension() override; + // PreprocessorPtr + // BuildPreprocessor(const DatasetPtr &dataset, const Config &config) override; + int64_t + Count() override; + int64_t + Dimension() override; IndexModelPtr - Train(const DatasetPtr &dataset, const Config &config) override; + Train(const DatasetPtr& dataset, const Config& config) override; void - Add(const DatasetPtr &dataset, const Config &config) override; + Add(const DatasetPtr& dataset, const Config& config) override; DatasetPtr - Search(const DatasetPtr &dataset, const Config &config) override; - void Seal() override; + Search(const DatasetPtr& dataset, const Config& config) override; + void + Seal() override; + private: void - SetParameters(const Config &config); + SetParameters(const Config& config); private: PreprocessorPtr preprocessor_; @@ -76,7 +77,7 @@ class CPUKDTRNGIndexModel : public IndexModel { Serialize() override; void - Load(const BinarySet &binary) override; + Load(const BinarySet& binary) override; private: std::shared_ptr index_; @@ -84,5 +85,5 @@ class CPUKDTRNGIndexModel : public IndexModel { using CPUKDTRNGIndexModelPtr = std::shared_ptr; -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.cpp index 38a7fed23b..b731f801b3 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.cpp @@ -15,28 +15,27 @@ // specific language governing permissions and limitations // under the License. - -#include "IndexNSG.h" -#include "knowhere/index/vector_index/nsg/NSG.h" -#include "knowhere/index/vector_index/nsg/NSGIO.h" -#include "IndexIDMAP.h" -#include "IndexIVF.h" -#include "IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexNSG.h" #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" #include "knowhere/common/Timer.h" - +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexIDMAP.h" +#include "knowhere/index/vector_index/IndexIVF.h" +#include "knowhere/index/vector_index/nsg/NSG.h" +#include "knowhere/index/vector_index/nsg/NSGIO.h" namespace zilliz { namespace knowhere { -BinarySet NSG::Serialize() { +BinarySet +NSG::Serialize() { if (!index_ || !index_->is_trained) { KNOWHERE_THROW_MSG("index not initialize or trained"); } try { - algo::NsgIndex *index = index_.get(); + algo::NsgIndex* index = index_.get(); MemoryIOWriter writer; algo::write_index(index, writer); @@ -46,12 +45,13 @@ BinarySet NSG::Serialize() { BinarySet res_set; res_set.Append("NSG", data, writer.total); return res_set; - } catch (std::exception &e) { + } catch (std::exception& e) { KNOWHERE_THROW_MSG(e.what()); } } -void NSG::Load(const BinarySet &index_binary) { +void +NSG::Load(const BinarySet& index_binary) { try { auto binary = index_binary.GetByName("NSG"); @@ -61,15 +61,16 @@ void NSG::Load(const BinarySet &index_binary) { auto index = algo::read_index(reader); index_.reset(index); - } catch (std::exception &e) { + } catch (std::exception& e) { KNOWHERE_THROW_MSG(e.what()); } } -DatasetPtr NSG::Search(const DatasetPtr &dataset, const Config &config) { +DatasetPtr +NSG::Search(const DatasetPtr& dataset, const Config& config) { auto build_cfg = std::dynamic_pointer_cast(config); if (build_cfg != nullptr) { - build_cfg->CheckValid(); // throw exception + build_cfg->CheckValid(); // throw exception } if (!index_ || !index_->is_trained) { @@ -79,16 +80,15 @@ DatasetPtr NSG::Search(const DatasetPtr &dataset, const Config &config) { GETTENSOR(dataset) auto elems = rows * build_cfg->k; - auto res_ids = (int64_t *) malloc(sizeof(int64_t) * elems); - auto res_dis = (float *) malloc(sizeof(float) * elems); + auto res_ids = (int64_t*)malloc(sizeof(int64_t) * elems); + auto res_dis = (float*)malloc(sizeof(float) * elems); algo::SearchParams s_params; s_params.search_length = build_cfg->search_length; - index_->Search((float *) p_data, rows, dim, - build_cfg->k, res_dis, res_ids, s_params); + index_->Search((float*)p_data, rows, dim, build_cfg->k, res_dis, res_ids, s_params); - auto id_buf = MakeMutableBufferSmart((uint8_t *) res_ids, sizeof(int64_t) * elems); - auto dist_buf = MakeMutableBufferSmart((uint8_t *) res_dis, sizeof(float) * elems); + auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); + auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); std::vector id_bufs{nullptr, id_buf}; std::vector dist_bufs{nullptr, dist_buf}; @@ -106,10 +106,11 @@ DatasetPtr NSG::Search(const DatasetPtr &dataset, const Config &config) { return std::make_shared(array, nullptr); } -IndexModelPtr NSG::Train(const DatasetPtr &dataset, const Config &config) { +IndexModelPtr +NSG::Train(const DatasetPtr& dataset, const Config& config) { auto build_cfg = std::dynamic_pointer_cast(config); if (build_cfg != nullptr) { - build_cfg->CheckValid(); // throw exception + build_cfg->CheckValid(); // throw exception } if (build_cfg->metric_type != METRICTYPE::L2) { @@ -132,34 +133,38 @@ IndexModelPtr NSG::Train(const DatasetPtr &dataset, const Config &config) { GETTENSOR(dataset) auto array = dataset->array()[0]; - auto p_ids = array->data()->GetValues(1, 0); + auto p_ids = array->data()->GetValues(1, 0); index_ = std::make_shared(dim, rows); index_->SetKnnGraph(knng); - index_->Build_with_ids(rows, (float *) p_data, (long *) p_ids, b_params); - return nullptr; // TODO(linxj): support serialize + index_->Build_with_ids(rows, (float*)p_data, (int64_t*)p_ids, b_params); + return nullptr; // TODO(linxj): support serialize } -void NSG::Add(const DatasetPtr &dataset, const Config &config) { +void +NSG::Add(const DatasetPtr& dataset, const Config& config) { // do nothing } -int64_t NSG::Count() { +int64_t +NSG::Count() { return index_->ntotal; } -int64_t NSG::Dimension() { +int64_t +NSG::Dimension() { return index_->dimension; } -VectorIndexPtr NSG::Clone() { +VectorIndexPtr +NSG::Clone() { KNOWHERE_THROW_MSG("not support"); } -void NSG::Seal() { +void +NSG::Seal() { // do nothing } -} -} - +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.h b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.h index 873bba2287..4371ddb65a 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.h @@ -15,11 +15,12 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include "VectorIndex.h" +#include +#include +#include "VectorIndex.h" namespace zilliz { namespace knowhere { @@ -30,18 +31,30 @@ class NsgIndex; class NSG : public VectorIndex { public: - explicit NSG(const int64_t& gpu_num):gpu_(gpu_num){} + explicit NSG(const int64_t& gpu_num) : gpu_(gpu_num) { + } + NSG() = default; - IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override; - DatasetPtr Search(const DatasetPtr &dataset, const Config &config) override; - void Add(const DatasetPtr &dataset, const Config &config) override; - BinarySet Serialize() override; - void Load(const BinarySet &index_binary) override; - int64_t Count() override; - int64_t Dimension() override; - VectorIndexPtr Clone() override; - void Seal() override; + IndexModelPtr + Train(const DatasetPtr& dataset, const Config& config) override; + DatasetPtr + Search(const DatasetPtr& dataset, const Config& config) override; + void + Add(const DatasetPtr& dataset, const Config& config) override; + BinarySet + Serialize() override; + void + Load(const BinarySet& index_binary) override; + int64_t + Count() override; + int64_t + Dimension() override; + VectorIndexPtr + Clone() override; + void + Seal() override; + private: std::shared_ptr index_; int64_t gpu_; @@ -49,5 +62,5 @@ class NSG : public VectorIndex { using NSGIndexPtr = std::shared_ptr(); -} -} +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/VectorIndex.h b/cpp/src/core/knowhere/knowhere/index/vector_index/VectorIndex.h index 908989d15d..e96de885af 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/VectorIndex.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/VectorIndex.h @@ -15,18 +15,15 @@ // specific language governing permissions and limitations // under the License. - #pragma once - #include #include "knowhere/common/Config.h" -#include "knowhere/index/vector_index/helpers/IndexParameter.h" #include "knowhere/common/Dataset.h" #include "knowhere/index/Index.h" #include "knowhere/index/preprocessor/Preprocessor.h" - +#include "knowhere/index/vector_index/helpers/IndexParameter.h" namespace zilliz { namespace knowhere { @@ -34,17 +31,20 @@ namespace knowhere { class VectorIndex; using VectorIndexPtr = std::shared_ptr; - class VectorIndex : public Index { public: virtual PreprocessorPtr - BuildPreprocessor(const DatasetPtr &dataset, const Config &config) { return nullptr; } + BuildPreprocessor(const DatasetPtr& dataset, const Config& config) { + return nullptr; + } virtual IndexModelPtr - Train(const DatasetPtr &dataset, const Config &config) { return nullptr; } + Train(const DatasetPtr& dataset, const Config& config) { + return nullptr; + } virtual void - Add(const DatasetPtr &dataset, const Config &config) = 0; + Add(const DatasetPtr& dataset, const Config& config) = 0; virtual void Seal() = 0; @@ -59,7 +59,5 @@ class VectorIndex : public Index { Dimension() = 0; }; - - -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp index d8ccdd1087..c6f3f81053 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp @@ -15,21 +15,20 @@ // specific language governing permissions and limitations // under the License. - +#include "knowhere/index/vector_index/helpers/Cloner.h" #include "knowhere/common/Exception.h" -#include "knowhere/index/vector_index/IndexIVF.h" -#include "knowhere/index/vector_index/IndexIVFSQ.h" -#include "knowhere/index/vector_index/IndexIVFPQ.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" #include "knowhere/index/vector_index/IndexIDMAP.h" -#include "Cloner.h" - +#include "knowhere/index/vector_index/IndexIVF.h" +#include "knowhere/index/vector_index/IndexIVFPQ.h" +#include "knowhere/index/vector_index/IndexIVFSQ.h" namespace zilliz { namespace knowhere { namespace cloner { -VectorIndexPtr CopyGpuToCpu(const VectorIndexPtr &index, const Config &config) { +VectorIndexPtr +CopyGpuToCpu(const VectorIndexPtr& index, const Config& config) { if (auto device_index = std::dynamic_pointer_cast(index)) { return device_index->CopyGpuToCpu(config); } else { @@ -37,7 +36,8 @@ VectorIndexPtr CopyGpuToCpu(const VectorIndexPtr &index, const Config &config) { } } -VectorIndexPtr CopyCpuToGpu(const VectorIndexPtr &index, const int64_t &device_id, const Config &config) { +VectorIndexPtr +CopyCpuToGpu(const VectorIndexPtr& index, const int64_t& device_id, const Config& config) { if (auto device_index = std::dynamic_pointer_cast(index)) { return device_index->CopyGpuToGpu(device_id, config); } @@ -55,6 +55,6 @@ VectorIndexPtr CopyCpuToGpu(const VectorIndexPtr &index, const int64_t &device_i } } -} // cloner -} -} +} // namespace cloner +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.h b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.h index a72e4df141..fb5d00d7df 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.h @@ -15,23 +15,21 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include "knowhere/index/vector_index/VectorIndex.h" - namespace zilliz { namespace knowhere { namespace cloner { // TODO(linxj): rename CopyToGpu extern VectorIndexPtr -CopyCpuToGpu(const VectorIndexPtr &index, const int64_t &device_id, const Config &config); +CopyCpuToGpu(const VectorIndexPtr& index, const int64_t& device_id, const Config& config); extern VectorIndexPtr -CopyGpuToCpu(const VectorIndexPtr &index, const Config &config); +CopyGpuToCpu(const VectorIndexPtr& index, const Config& config); -} // cloner -} // knowhere -} // zilliz \ No newline at end of file +} // namespace cloner +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Definitions.h b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Definitions.h index fef7b01221..d5959f7501 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Definitions.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Definitions.h @@ -15,10 +15,8 @@ // specific language governing permissions and limitations // under the License. - #pragma once - namespace zilliz { namespace knowhere { namespace definition { @@ -27,6 +25,6 @@ namespace definition { #define META_DIM ("dimension") #define META_K ("k") -} // definition -} // knowhere -} // zilliz +} // namespace definition +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp index c50594b4b0..70c879d03a 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp @@ -15,25 +15,24 @@ // specific language governing permissions and limitations // under the License. +#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" -#include "FaissGpuResourceMgr.h" - +#include namespace zilliz { namespace knowhere { -FaissGpuResourceMgr &FaissGpuResourceMgr::GetInstance() { +FaissGpuResourceMgr& +FaissGpuResourceMgr::GetInstance() { static FaissGpuResourceMgr instance; return instance; } -void FaissGpuResourceMgr::AllocateTempMem(ResPtr &resource, - const int64_t &device_id, - const int64_t &size) { +void +FaissGpuResourceMgr::AllocateTempMem(ResPtr& resource, const int64_t& device_id, const int64_t& size) { if (size) { resource->faiss_res->setTempMemory(size); - } - else { + } else { auto search = devices_params_.find(device_id); if (search != devices_params_.end()) { resource->faiss_res->setTempMemory(search->second.temp_mem_size); @@ -42,10 +41,8 @@ void FaissGpuResourceMgr::AllocateTempMem(ResPtr &resource, } } -void FaissGpuResourceMgr::InitDevice(int64_t device_id, - int64_t pin_mem_size, - int64_t temp_mem_size, - int64_t res_num) { +void +FaissGpuResourceMgr::InitDevice(int64_t device_id, int64_t pin_mem_size, int64_t temp_mem_size, int64_t res_num) { DeviceParams params; params.pinned_mem_size = pin_mem_size; params.temp_mem_size = temp_mem_size; @@ -54,23 +51,25 @@ void FaissGpuResourceMgr::InitDevice(int64_t device_id, devices_params_.emplace(device_id, params); } -void FaissGpuResourceMgr::InitResource() { - if(is_init) return ; +void +FaissGpuResourceMgr::InitResource() { + if (is_init) + return; is_init = true; - //std::cout << "InitResource" << std::endl; - for(auto& device : devices_params_) { + // std::cout << "InitResource" << std::endl; + for (auto& device : devices_params_) { auto& device_id = device.first; mutex_cache_.emplace(device_id, std::make_unique()); - //std::cout << "Device Id: " << device_id << std::endl; + // std::cout << "Device Id: " << device_id << std::endl; auto& device_param = device.second; auto& bq = idle_map_[device_id]; for (int64_t i = 0; i < device_param.resource_num; ++i) { - //std::cout << "Resource Id: " << i << std::endl; + // std::cout << "Resource Id: " << i << std::endl; auto raw_resource = std::make_shared(); // TODO(linxj): enable set pinned memory @@ -80,11 +79,11 @@ void FaissGpuResourceMgr::InitResource() { bq.Put(res_wrapper); } } - //std::cout << "End initResource" << std::endl; + // std::cout << "End initResource" << std::endl; } -ResPtr FaissGpuResourceMgr::GetRes(const int64_t &device_id, - const int64_t &alloc_size) { +ResPtr +FaissGpuResourceMgr::GetRes(const int64_t& device_id, const int64_t& alloc_size) { InitResource(); auto finder = idle_map_.find(device_id); @@ -97,7 +96,8 @@ ResPtr FaissGpuResourceMgr::GetRes(const int64_t &device_id, return nullptr; } -void FaissGpuResourceMgr::MoveToIdle(const int64_t &device_id, const ResPtr &res) { +void +FaissGpuResourceMgr::MoveToIdle(const int64_t& device_id, const ResPtr& res) { auto finder = idle_map_.find(device_id); if (finder != idle_map_.end()) { auto& bq = finder->second; @@ -105,8 +105,9 @@ void FaissGpuResourceMgr::MoveToIdle(const int64_t &device_id, const ResPtr &res } } -void FaissGpuResourceMgr::Free() { - for (auto &item : idle_map_) { +void +FaissGpuResourceMgr::Free() { + for (auto& item : idle_map_) { auto& bq = item.second; while (!bq.Empty()) { bq.Take(); @@ -117,12 +118,11 @@ void FaissGpuResourceMgr::Free() { void FaissGpuResourceMgr::Dump() { - for (auto &item : idle_map_) { + for (auto& item : idle_map_) { auto& bq = item.second; - std::cout << "device_id: " << item.first - << ", resource count:" << bq.Size(); + std::cout << "device_id: " << item.first << ", resource count:" << bq.Size(); } } -} // knowhere -} // zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h index 3b4786a2a5..926b8b4e00 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. - #pragma once +#include #include #include -#include +#include #include @@ -30,7 +30,7 @@ namespace zilliz { namespace knowhere { struct Resource { - explicit Resource(std::shared_ptr &r) : faiss_res(r) { + explicit Resource(std::shared_ptr& r) : faiss_res(r) { static int64_t global_id = 0; id = global_id++; } @@ -43,19 +43,19 @@ using ResPtr = std::shared_ptr; using ResWPtr = std::weak_ptr; class FaissGpuResourceMgr { -public: + public: friend class ResScope; using ResBQ = zilliz::milvus::server::BlockingQueue; -public: + public: struct DeviceParams { int64_t temp_mem_size = 0; int64_t pinned_mem_size = 0; int64_t resource_num = 2; }; -public: - static FaissGpuResourceMgr & + public: + static FaissGpuResourceMgr& GetInstance(); // Free gpu resource, avoid cudaGetDevice error when deallocate. @@ -64,67 +64,67 @@ public: Free(); void - AllocateTempMem(ResPtr &resource, const int64_t& device_id, const int64_t& size); + AllocateTempMem(ResPtr& resource, const int64_t& device_id, const int64_t& size); void - InitDevice(int64_t device_id, - int64_t pin_mem_size = 0, - int64_t temp_mem_size = 0, - int64_t res_num = 2); + InitDevice(int64_t device_id, int64_t pin_mem_size = 0, int64_t temp_mem_size = 0, int64_t res_num = 2); void InitResource(); // allocate gpu memory invoke by build or copy_to_gpu ResPtr - GetRes(const int64_t &device_id, const int64_t& alloc_size = 0); + GetRes(const int64_t& device_id, const int64_t& alloc_size = 0); void - MoveToIdle(const int64_t &device_id, const ResPtr& res); + MoveToIdle(const int64_t& device_id, const ResPtr& res); void Dump(); -protected: + protected: bool is_init = false; - std::map> mutex_cache_; + std::map> mutex_cache_; std::map devices_params_; std::map idle_map_; }; class ResScope { -public: - ResScope(ResPtr &res, const int64_t& device_id, const bool& isown) - : resource(res), device_id(device_id), move(true), own(isown) { + public: + ResScope(ResPtr& res, const int64_t& device_id, const bool& isown) + : resource(res), device_id(device_id), move(true), own(isown) { Lock(); } // specif for search // get the ownership of gpuresource and gpu - ResScope(ResWPtr &res, const int64_t &device_id) - :device_id(device_id),move(false),own(true) { + ResScope(ResWPtr& res, const int64_t& device_id) : device_id(device_id), move(false), own(true) { resource = res.lock(); Lock(); } - void Lock() { - if (own) FaissGpuResourceMgr::GetInstance().mutex_cache_[device_id]->lock(); + void + Lock() { + if (own) + FaissGpuResourceMgr::GetInstance().mutex_cache_[device_id]->lock(); resource->mutex.lock(); } ~ResScope() { - if (own) FaissGpuResourceMgr::GetInstance().mutex_cache_[device_id]->unlock(); - if (move) FaissGpuResourceMgr::GetInstance().MoveToIdle(device_id, resource); + if (own) + FaissGpuResourceMgr::GetInstance().mutex_cache_[device_id]->unlock(); + if (move) + FaissGpuResourceMgr::GetInstance().MoveToIdle(device_id, resource); resource->mutex.unlock(); } -private: - ResPtr resource; // hold resource until deconstruct + private: + ResPtr resource; // hold resource until deconstruct int64_t device_id; bool move = true; bool own = false; }; -} // knowhere -} // zilliz \ No newline at end of file +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp index 48167fe925..30a54b388d 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp @@ -15,51 +15,55 @@ // specific language governing permissions and limitations // under the License. - #include -#include "FaissIO.h" +#include "knowhere/index/vector_index/helpers/FaissIO.h" namespace zilliz { namespace knowhere { // TODO(linxj): Get From Config File static size_t magic_num = 2; -size_t MemoryIOWriter::operator()(const void *ptr, size_t size, size_t nitems) { + +size_t +MemoryIOWriter::operator()(const void* ptr, size_t size, size_t nitems) { auto total_need = size * nitems + rp; - if (!data_) { // data == nullptr + if (!data_) { // data == nullptr total = total_need * magic_num; rp = size * nitems; data_ = new uint8_t[total]; - memcpy((void *) (data_), ptr, rp); + memcpy((void*)(data_), ptr, rp); } if (total_need > total) { total = total_need * magic_num; auto new_data = new uint8_t[total]; - memcpy((void *) new_data, (void *) data_, rp); + memcpy((void*)new_data, (void*)data_, rp); delete data_; data_ = new_data; - memcpy((void *) (data_ + rp), ptr, size * nitems); + memcpy((void*)(data_ + rp), ptr, size * nitems); rp = total_need; } else { - memcpy((void *) (data_ + rp), ptr, size * nitems); + memcpy((void*)(data_ + rp), ptr, size * nitems); rp = total_need; } return nitems; } -size_t MemoryIOReader::operator()(void *ptr, size_t size, size_t nitems) { - if (rp >= total) return 0; +size_t +MemoryIOReader::operator()(void* ptr, size_t size, size_t nitems) { + if (rp >= total) + return 0; size_t nremain = (total - rp) / size; - if (nremain < nitems) nitems = nremain; - memcpy(ptr, (void *) (data_ + rp), size * nitems); + if (nremain < nitems) + nitems = nremain; + memcpy(ptr, (void*)(data_ + rp), size * nitems); rp += size * nitems; return nitems; } -} // knowhere -} // zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.h b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.h index cc089ec367..c850e89d9f 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include @@ -24,25 +23,22 @@ namespace zilliz { namespace knowhere { struct MemoryIOWriter : public faiss::IOWriter { - uint8_t *data_ = nullptr; + uint8_t* data_ = nullptr; size_t total = 0; size_t rp = 0; size_t - operator()(const void *ptr, size_t size, size_t nitems) override; + operator()(const void* ptr, size_t size, size_t nitems) override; }; struct MemoryIOReader : public faiss::IOReader { - uint8_t *data_; + uint8_t* data_; size_t rp = 0; size_t total = 0; size_t - operator()(void *ptr, size_t size, size_t nitems) override; + operator()(void* ptr, size_t size, size_t nitems) override; }; -} // knowhere -} // zilliz - - - +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp index 9999444368..c4a179bc6a 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp @@ -15,8 +15,7 @@ // specific language governing permissions and limitations // under the License. - -#include "IndexParameter.h" +#include "knowhere/index/vector_index/helpers/IndexParameter.h" #include "knowhere/common/Exception.h" #include @@ -24,7 +23,8 @@ namespace zilliz { namespace knowhere { -faiss::MetricType GetMetricType(METRICTYPE &type) { +faiss::MetricType +GetMetricType(METRICTYPE& type) { if (type == METRICTYPE::L2) { return faiss::METRIC_L2; } @@ -35,6 +35,5 @@ faiss::MetricType GetMetricType(METRICTYPE &type) { KNOWHERE_THROW_MSG("Metric type is invalid"); } - -} -} +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h index cd873ed251..d5056d7ee6 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h @@ -15,17 +15,18 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include "knowhere/common/Config.h" #include +#include +#include "knowhere/common/Config.h" namespace zilliz { namespace knowhere { -extern faiss::MetricType GetMetricType(METRICTYPE &type); +extern faiss::MetricType +GetMetricType(METRICTYPE& type); // IVF Config constexpr int64_t DEFAULT_NLIST = INVALID_VALUE; @@ -46,11 +47,7 @@ struct IVFCfg : public Cfg { int64_t nlist = DEFAULT_NLIST; int64_t nprobe = DEFAULT_NPROBE; - IVFCfg(const int64_t &dim, - const int64_t &k, - const int64_t &gpu_id, - const int64_t &nlist, - const int64_t &nprobe, + IVFCfg(const int64_t& dim, const int64_t& k, const int64_t& gpu_id, const int64_t& nlist, const int64_t& nprobe, METRICTYPE type) : Cfg(dim, k, gpu_id, type), nlist(nlist), nprobe(nprobe) { } @@ -68,13 +65,8 @@ struct IVFSQCfg : public IVFCfg { // TODO(linxj): cpu only support SQ4 SQ6 SQ8 SQ16, gpu only support SQ4, SQ8, SQ16 int64_t nbits = DEFAULT_NBITS; - IVFSQCfg(const int64_t &dim, - const int64_t &k, - const int64_t &gpu_id, - const int64_t &nlist, - const int64_t &nprobe, - const int64_t &nbits, - METRICTYPE type) + IVFSQCfg(const int64_t& dim, const int64_t& k, const int64_t& gpu_id, const int64_t& nlist, const int64_t& nprobe, + const int64_t& nbits, METRICTYPE type) : IVFCfg(dim, k, gpu_id, nlist, nprobe, type), nbits(nbits) { } @@ -88,22 +80,16 @@ struct IVFSQCfg : public IVFCfg { using IVFSQConfig = std::shared_ptr; struct IVFPQCfg : public IVFCfg { - int64_t m = DEFAULT_NSUBVECTORS; // number of subquantizers(subvector) - int64_t nbits = DEFAULT_NBITS; // number of bit per subvector index + int64_t m = DEFAULT_NSUBVECTORS; // number of subquantizers(subvector) + int64_t nbits = DEFAULT_NBITS; // number of bit per subvector index // TODO(linxj): not use yet int64_t scan_table_threhold = DEFAULT_SCAN_TABLE_THREHOLD; int64_t polysemous_ht = DEFAULT_POLYSEMOUS_HT; int64_t max_codes = DEFAULT_MAX_CODES; - IVFPQCfg(const int64_t &dim, - const int64_t &k, - const int64_t &gpu_id, - const int64_t &nlist, - const int64_t &nprobe, - const int64_t &nbits, - const int64_t &m, - METRICTYPE type) + IVFPQCfg(const int64_t& dim, const int64_t& k, const int64_t& gpu_id, const int64_t& nlist, const int64_t& nprobe, + const int64_t& nbits, const int64_t& m, METRICTYPE type) : IVFCfg(dim, k, gpu_id, nlist, nprobe, type), m(m), nbits(nbits) { } @@ -122,19 +108,14 @@ struct NSGCfg : public IVFCfg { int64_t out_degree = DEFAULT_OUT_DEGREE; int64_t candidate_pool_size = DEFAULT_CANDIDATE_SISE; - NSGCfg(const int64_t &dim, - const int64_t &k, - const int64_t &gpu_id, - const int64_t &nlist, - const int64_t &nprobe, - const int64_t &knng, - const int64_t &search_length, - const int64_t &out_degree, - const int64_t &candidate_size, + NSGCfg(const int64_t& dim, const int64_t& k, const int64_t& gpu_id, const int64_t& nlist, const int64_t& nprobe, + const int64_t& knng, const int64_t& search_length, const int64_t& out_degree, const int64_t& candidate_size, METRICTYPE type) : IVFCfg(dim, k, gpu_id, nlist, nprobe, type), - knng(knng), search_length(search_length), - out_degree(out_degree), candidate_pool_size(candidate_size) { + knng(knng), + search_length(search_length), + out_degree(out_degree), + candidate_pool_size(candidate_size) { } NSGCfg() = default; @@ -150,6 +131,5 @@ struct KDTCfg : public Cfg { int64_t tptnubmber = -1; }; -} // knowhere -} // zilliz - +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp index 3d9de271b7..015dc17fc5 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp @@ -15,16 +15,14 @@ // specific language governing permissions and limitations // under the License. - #include -#include "KDTParameterMgr.h" - +#include "knowhere/index/vector_index/helpers/KDTParameterMgr.h" namespace zilliz { namespace knowhere { -const std::vector & +const std::vector& KDTParameterMgr::GetKDTParameters() { return kdt_parameters_; } @@ -55,5 +53,5 @@ KDTParameterMgr::KDTParameterMgr() { }; } -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h index 4b43e642e2..ec2d73eebb 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h @@ -15,13 +15,13 @@ // specific language governing permissions and limitations // under the License. - #pragma once +#include #include +#include #include - namespace zilliz { namespace knowhere { @@ -29,18 +29,20 @@ using KDTParameter = std::pair; class KDTParameterMgr { public: - const std::vector & + const std::vector& GetKDTParameters(); public: - static KDTParameterMgr & + static KDTParameterMgr& GetInstance() { static KDTParameterMgr instance; return instance; } - KDTParameterMgr(const KDTParameterMgr &) = delete; - KDTParameterMgr &operator=(const KDTParameterMgr &) = delete; + KDTParameterMgr(const KDTParameterMgr&) = delete; + KDTParameterMgr& + operator=(const KDTParameterMgr&) = delete; + private: KDTParameterMgr(); @@ -48,5 +50,5 @@ class KDTParameterMgr { std::vector kdt_parameters_; }; -} // namespace knowhere -} // namespace zilliz +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp index fcf77625db..19bc825d81 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp @@ -15,29 +15,28 @@ // specific language governing permissions and limitations // under the License. -#include +#include #include -#include +#include #include +#include #include -#include +#include -#include "NSG.h" #include "knowhere/common/Exception.h" #include "knowhere/common/Log.h" #include "knowhere/common/Timer.h" -#include "NSGHelper.h" +#include "knowhere/index/vector_index/nsg/NSG.h" +#include "knowhere/index/vector_index/nsg/NSGHelper.h" // TODO: enable macro //#include - namespace zilliz { namespace knowhere { namespace algo { - -NsgIndex::NsgIndex(const size_t &dimension, const size_t &n, MetricType metric) +NsgIndex::NsgIndex(const size_t& dimension, const size_t& n, MetricType metric) : dimension(dimension), ntotal(n), metric_type(metric) { } @@ -46,16 +45,17 @@ NsgIndex::~NsgIndex() { delete[] ids_; } -//void NsgIndex::Build(size_t nb, const float *data, const BuildParam ¶meters) { +// void NsgIndex::Build(size_t nb, const float *data, const BuildParam ¶meters) { //} -void NsgIndex::Build_with_ids(size_t nb, const float *data, const long *ids, const BuildParams ¶meters) { +void +NsgIndex::Build_with_ids(size_t nb, const float* data, const int64_t* ids, const BuildParams& parameters) { TimeRecorder rc("NSG"); ntotal = nb; ori_data_ = new float[ntotal * dimension]; - ids_ = new long[ntotal]; - memcpy((void *) ori_data_, (void *) data, sizeof(float) * ntotal * dimension); - memcpy((void *) ids_, (void *) ids, sizeof(long) * ntotal); + ids_ = new int64_t[ntotal]; + memcpy((void*)ori_data_, (void*)data, sizeof(float) * ntotal * dimension); + memcpy((void*)ids_, (void*)ids, sizeof(int64_t) * ntotal); search_length = parameters.search_length; out_degree = parameters.out_degree; @@ -69,8 +69,8 @@ void NsgIndex::Build_with_ids(size_t nb, const float *data, const long *ids, con //>> Debug code ///// - //int count = 0; - //for (int i = 0; i < ntotal; ++i) { + // int count = 0; + // for (int i = 0; i < ntotal; ++i) { // count += nsg[i].size(); //} ///// @@ -92,7 +92,8 @@ void NsgIndex::Build_with_ids(size_t nb, const float *data, const long *ids, con is_trained = true; } -void NsgIndex::InitNavigationPoint() { +void +NsgIndex::InitNavigationPoint() { // calculate the center of vectors auto center = new float[dimension]; memset(center, 0, sizeof(float) * dimension); @@ -108,11 +109,12 @@ void NsgIndex::InitNavigationPoint() { // select navigation point std::vector resset, fullset; - navigation_point = rand() % ntotal; // random initialize navigating point + unsigned int seed = 100; + navigation_point = rand_r(&seed) % ntotal; // random initialize navigating point //>> Debug code ///// - //navigation_point = drand48(); + // navigation_point = drand48(); ///// GetNeighbors(center, resset, knng); @@ -120,22 +122,21 @@ void NsgIndex::InitNavigationPoint() { //>> Debug code ///// - //std::cout << "ep: " << navigation_point << std::endl; + // std::cout << "ep: " << navigation_point << std::endl; ///// //>> Debug code ///// - //float r1 = calculate(center, ori_data_ + navigation_point * dimension, dimension); - //assert(r1 == resset[0].distance); + // float r1 = calculate(center, ori_data_ + navigation_point * dimension, dimension); + // assert(r1 == resset[0].distance); ///// } // Specify Link -void NsgIndex::GetNeighbors(const float *query, - std::vector &resset, - std::vector &fullset, - boost::dynamic_bitset<> &has_calculated_dist) { - auto &graph = knng; +void +NsgIndex::GetNeighbors(const float* query, std::vector& resset, std::vector& fullset, + boost::dynamic_bitset<>& has_calculated_dist) { + auto& graph = knng; size_t buffer_size = search_length; if (buffer_size > ntotal) { @@ -156,9 +157,12 @@ void NsgIndex::GetNeighbors(const float *query, has_calculated_dist[init_ids[i]] = true; ++count; } + + unsigned int seed = 100; while (count < buffer_size) { - node_t id = rand() % ntotal; - if (has_calculated_dist[id]) continue; // duplicate id + node_t id = rand_r(&seed) % ntotal; + if (has_calculated_dist[id]) + continue; // duplicate id init_ids.push_back(id); ++count; has_calculated_dist[id] = true; @@ -184,9 +188,9 @@ void NsgIndex::GetNeighbors(const float *query, fullset.push_back(resset[i]); /////////////////////////////////////// } - std::sort(resset.begin(), resset.end()); // sort by distance + std::sort(resset.begin(), resset.end()); // sort by distance - //search nearest neighbor + // search nearest neighbor size_t cursor = 0; while (cursor < buffer_size) { size_t nearest_updated_pos = buffer_size; @@ -195,36 +199,42 @@ void NsgIndex::GetNeighbors(const float *query, resset[cursor].has_explored = true; node_t start_pos = resset[cursor].id; - auto &wait_for_search_node_vec = graph[start_pos]; + auto& wait_for_search_node_vec = graph[start_pos]; for (size_t i = 0; i < wait_for_search_node_vec.size(); ++i) { node_t id = wait_for_search_node_vec[i]; - if (has_calculated_dist[id]) continue; + if (has_calculated_dist[id]) + continue; has_calculated_dist[id] = true; - float - dist = calculate(query, ori_data_ + dimension * id, dimension); + float dist = calculate(query, ori_data_ + dimension * id, dimension); Neighbor nn(id, dist, false); fullset.push_back(nn); - if (dist >= resset[buffer_size - 1].distance) continue; + if (dist >= resset[buffer_size - 1].distance) + continue; - size_t pos = InsertIntoPool(resset.data(), buffer_size, nn); // replace with a closer node - if (pos < nearest_updated_pos) nearest_updated_pos = pos; + size_t pos = InsertIntoPool(resset.data(), buffer_size, nn); // replace with a closer node + if (pos < nearest_updated_pos) + nearest_updated_pos = pos; - //assert(buffer_size + 1 >= resset.size()); - if (buffer_size + 1 < resset.size()) ++buffer_size; + // assert(buffer_size + 1 >= resset.size()); + if (buffer_size + 1 < resset.size()) + ++buffer_size; } } if (cursor >= nearest_updated_pos) { - cursor = nearest_updated_pos; // re-search from new pos - } else ++cursor; + cursor = nearest_updated_pos; // re-search from new pos + } else { + ++cursor; + } } } } // FindUnconnectedNode -void NsgIndex::GetNeighbors(const float *query, std::vector &resset, std::vector &fullset) { - auto &graph = nsg; +void +NsgIndex::GetNeighbors(const float* query, std::vector& resset, std::vector& fullset) { + auto& graph = nsg; size_t buffer_size = search_length; if (buffer_size > ntotal) { @@ -232,7 +242,7 @@ void NsgIndex::GetNeighbors(const float *query, std::vector &resset, s } std::vector init_ids; - boost::dynamic_bitset<> has_calculated_dist{ntotal, 0}; // TODO: ? + boost::dynamic_bitset<> has_calculated_dist{ntotal, 0}; // TODO: ? { /* @@ -246,9 +256,11 @@ void NsgIndex::GetNeighbors(const float *query, std::vector &resset, s has_calculated_dist[init_ids[i]] = true; ++count; } + unsigned int seed = 100; while (count < buffer_size) { - node_t id = rand() % ntotal; - if (has_calculated_dist[id]) continue; // duplicate id + node_t id = rand_r(&seed) % ntotal; + if (has_calculated_dist[id]) + continue; // duplicate id init_ids.push_back(id); ++count; has_calculated_dist[id] = true; @@ -270,7 +282,7 @@ void NsgIndex::GetNeighbors(const float *query, std::vector &resset, s float dist = calculate(ori_data_ + id * dimension, query, dimension); resset[i] = Neighbor(id, dist, false); } - std::sort(resset.begin(), resset.end()); // sort by distance + std::sort(resset.begin(), resset.end()); // sort by distance // search nearest neighbor size_t cursor = 0; @@ -281,38 +293,41 @@ void NsgIndex::GetNeighbors(const float *query, std::vector &resset, s resset[cursor].has_explored = true; node_t start_pos = resset[cursor].id; - auto &wait_for_search_node_vec = graph[start_pos]; + auto& wait_for_search_node_vec = graph[start_pos]; for (size_t i = 0; i < wait_for_search_node_vec.size(); ++i) { node_t id = wait_for_search_node_vec[i]; - if (has_calculated_dist[id]) continue; + if (has_calculated_dist[id]) + continue; has_calculated_dist[id] = true; - float - dist = calculate(ori_data_ + dimension * id, query, dimension); + float dist = calculate(ori_data_ + dimension * id, query, dimension); Neighbor nn(id, dist, false); fullset.push_back(nn); - if (dist >= resset[buffer_size - 1].distance) continue; + if (dist >= resset[buffer_size - 1].distance) + continue; - size_t pos = InsertIntoPool(resset.data(), buffer_size, nn); // replace with a closer node - if (pos < nearest_updated_pos) nearest_updated_pos = pos; + size_t pos = InsertIntoPool(resset.data(), buffer_size, nn); // replace with a closer node + if (pos < nearest_updated_pos) + nearest_updated_pos = pos; - //assert(buffer_size + 1 >= resset.size()); - if (buffer_size + 1 < resset.size()) ++buffer_size; // trick + // assert(buffer_size + 1 >= resset.size()); + if (buffer_size + 1 < resset.size()) + ++buffer_size; // trick } } if (cursor >= nearest_updated_pos) { - cursor = nearest_updated_pos; // re-search from new pos - } else ++cursor; + cursor = nearest_updated_pos; // re-search from new pos + } else { + ++cursor; + } } } } -void NsgIndex::GetNeighbors(const float *query, - std::vector &resset, - Graph &graph, - SearchParams *params) { - size_t &buffer_size = params ? params->search_length : search_length; +void +NsgIndex::GetNeighbors(const float* query, std::vector& resset, Graph& graph, SearchParams* params) { + size_t& buffer_size = params ? params->search_length : search_length; if (buffer_size > ntotal) { // TODO: throw exception here. @@ -333,9 +348,11 @@ void NsgIndex::GetNeighbors(const float *query, has_calculated_dist[init_ids[i]] = true; ++count; } + unsigned int seed = 100; while (count < buffer_size) { - node_t id = rand() % ntotal; - if (has_calculated_dist[id]) continue; // duplicate id + node_t id = rand_r(&seed) % ntotal; + if (has_calculated_dist[id]) + continue; // duplicate id init_ids.push_back(id); ++count; has_calculated_dist[id] = true; @@ -349,7 +366,7 @@ void NsgIndex::GetNeighbors(const float *query, for (size_t i = 0; i < init_ids.size(); ++i) { node_t id = init_ids[i]; - //assert(id < ntotal); + // assert(id < ntotal); if (id >= static_cast(ntotal)) { KNOWHERE_THROW_MSG("Build Index Error, id > ntotal"); continue; @@ -358,11 +375,11 @@ void NsgIndex::GetNeighbors(const float *query, float dist = calculate(ori_data_ + id * dimension, query, dimension); resset[i] = Neighbor(id, dist, false); } - std::sort(resset.begin(), resset.end()); // sort by distance + std::sort(resset.begin(), resset.end()); // sort by distance //>> Debug code ///// - //for (int j = 0; j < buffer_size; ++j) { + // for (int j = 0; j < buffer_size; ++j) { // std::cout << "resset_id: " << resset[j].id << ", resset_dist: " << resset[j].distance << std::endl; //} ///// @@ -376,41 +393,47 @@ void NsgIndex::GetNeighbors(const float *query, resset[cursor].has_explored = true; node_t start_pos = resset[cursor].id; - auto &wait_for_search_node_vec = graph[start_pos]; + auto& wait_for_search_node_vec = graph[start_pos]; for (size_t i = 0; i < wait_for_search_node_vec.size(); ++i) { node_t id = wait_for_search_node_vec[i]; - if (has_calculated_dist[id]) continue; + if (has_calculated_dist[id]) + continue; has_calculated_dist[id] = true; - float - dist = calculate(query, ori_data_ + dimension * id, dimension); + float dist = calculate(query, ori_data_ + dimension * id, dimension); - if (dist >= resset[buffer_size - 1].distance) continue; + if (dist >= resset[buffer_size - 1].distance) + continue; ///////////// difference from other GetNeighbors /////////////// Neighbor nn(id, dist, false); /////////////////////////////////////// - size_t pos = InsertIntoPool(resset.data(), buffer_size, nn); // replace with a closer node - if (pos < nearest_updated_pos) nearest_updated_pos = pos; + size_t pos = InsertIntoPool(resset.data(), buffer_size, nn); // replace with a closer node + if (pos < nearest_updated_pos) + nearest_updated_pos = pos; //>> Debug code ///// - //std::cout << "pos: " << pos << ", nn: " << nn.id << ":" << nn.distance << ", nup: " << nearest_updated_pos << std::endl; + // std::cout << "pos: " << pos << ", nn: " << nn.id << ":" << nn.distance << ", nup: " << + // nearest_updated_pos << std::endl; ///// - // trick: avoid search query search_length < init_ids.size() ... - if (buffer_size + 1 < resset.size()) ++buffer_size; + if (buffer_size + 1 < resset.size()) + ++buffer_size; } } if (cursor >= nearest_updated_pos) { - cursor = nearest_updated_pos; // re-search from new pos - } else ++cursor; + cursor = nearest_updated_pos; // re-search from new pos + } else { + ++cursor; + } } } } -void NsgIndex::Link() { +void +NsgIndex::Link() { auto cut_graph_dist = new float[ntotal * out_degree]; nsg.resize(ntotal); @@ -418,7 +441,7 @@ void NsgIndex::Link() { { std::vector fullset; std::vector temp; - boost::dynamic_bitset<> flags{ntotal, 0}; // TODO: ? + boost::dynamic_bitset<> flags{ntotal, 0}; // TODO: ? #pragma omp for schedule(dynamic, 100) for (size_t n = 0; n < ntotal; ++n) { fullset.clear(); @@ -427,8 +450,8 @@ void NsgIndex::Link() { //>> Debug code ///// - //float r1 = calculate(ori_data_ + n * dimension, ori_data_ + temp[0].id * dimension, dimension); - //assert(r1 == temp[0].distance); + // float r1 = calculate(ori_data_ + n * dimension, ori_data_ + temp[0].id * dimension, dimension); + // assert(r1 == temp[0].distance); ///// SyncPrune(n, fullset, flags, cut_graph_dist); } @@ -436,7 +459,7 @@ void NsgIndex::Link() { //>> Debug code ///// - //auto bak_nsg = nsg; + // auto bak_nsg = nsg; ///// knng.clear(); @@ -452,8 +475,8 @@ void NsgIndex::Link() { //>> Debug code ///// - //int count = 0; - //for (int i = 0; i < ntotal; ++i) { + // int count = 0; + // for (int i = 0; i < ntotal; ++i) { // if (bak_nsg[i].size() != nsg[i].size()) { // //count += nsg[i].size() - bak_nsg[i].size(); // count += nsg[i].size(); @@ -466,16 +489,15 @@ void NsgIndex::Link() { } } -void NsgIndex::SyncPrune(size_t n, - std::vector &pool, - boost::dynamic_bitset<> &has_calculated, - float *cut_graph_dist) { +void +NsgIndex::SyncPrune(size_t n, std::vector& pool, boost::dynamic_bitset<>& has_calculated, + float* cut_graph_dist) { // avoid lose nearest neighbor in knng for (size_t i = 0; i < knng[n].size(); ++i) { auto id = knng[n][i]; - if (has_calculated[id]) continue; - float dist = calculate(ori_data_ + dimension * n, - ori_data_ + dimension * id, dimension); + if (has_calculated[id]) + continue; + float dist = calculate(ori_data_ + dimension * n, ori_data_ + dimension * id, dimension); pool.emplace_back(Neighbor(id, dist, true)); } @@ -486,13 +508,13 @@ void NsgIndex::SyncPrune(size_t n, if (pool[cursor].id == static_cast(n)) { cursor++; } - result.push_back(pool[cursor]); // init result with nearest neighbor + result.push_back(pool[cursor]); // init result with nearest neighbor SelectEdge(cursor, pool, result, true); // filling the cut_graph - auto &des_id_pool = nsg[n]; - float *des_dist_pool = cut_graph_dist + n * out_degree; + auto& des_id_pool = nsg[n]; + float* des_dist_pool = cut_graph_dist + n * out_degree; for (size_t i = 0; i < result.size(); ++i) { des_id_pool.push_back(result[i].id); des_dist_pool[i] = result[i].distance; @@ -504,24 +526,27 @@ void NsgIndex::SyncPrune(size_t n, } //>> Optimize: remove read-lock -void NsgIndex::InterInsert(unsigned n, std::vector &mutex_vec, float *cut_graph_dist) { - auto ¤t = n; +void +NsgIndex::InterInsert(unsigned n, std::vector& mutex_vec, float* cut_graph_dist) { + auto& current = n; - auto &neighbor_id_pool = nsg[current]; - float *neighbor_dist_pool = cut_graph_dist + current * out_degree; + auto& neighbor_id_pool = nsg[current]; + float* neighbor_dist_pool = cut_graph_dist + current * out_degree; for (size_t i = 0; i < out_degree; ++i) { - if (neighbor_dist_pool[i] == -1) break; + if (neighbor_dist_pool[i] == -1) + break; - size_t current_neighbor = neighbor_id_pool[i]; // center's neighbor id - auto &nsn_id_pool = nsg[current_neighbor]; // nsn => neighbor's neighbor - float *nsn_dist_pool = cut_graph_dist + current_neighbor * out_degree; + size_t current_neighbor = neighbor_id_pool[i]; // center's neighbor id + auto& nsn_id_pool = nsg[current_neighbor]; // nsn => neighbor's neighbor + float* nsn_dist_pool = cut_graph_dist + current_neighbor * out_degree; - std::vector wait_for_link_pool; // maintain candidate neighbor of the current neighbor. + std::vector wait_for_link_pool; // maintain candidate neighbor of the current neighbor. int duplicate = false; { LockGuard lk(mutex_vec[current_neighbor]); for (size_t j = 0; j < out_degree; ++j) { - if (nsn_dist_pool[j] == -1) break; + if (nsn_dist_pool[j] == -1) + break; // 保证至少有一条边能连回来 if (n == nsn_id_pool[j]) { @@ -533,7 +558,8 @@ void NsgIndex::InterInsert(unsigned n, std::vector &mutex_vec, float wait_for_link_pool.push_back(nsn); } } - if (duplicate) continue; + if (duplicate) + continue; // original: (neighbor) <------- (current) // after: (neighbor) -------> (current) @@ -564,20 +590,18 @@ void NsgIndex::InterInsert(unsigned n, std::vector &mutex_vec, float if (nsn_dist_pool[j] == -1) { nsn_id_pool.push_back(current_as_neighbor.id); nsn_dist_pool[j] = current_as_neighbor.distance; - if (j + 1 < out_degree) nsn_dist_pool[j + 1] = -1; + if (j + 1 < out_degree) + nsn_dist_pool[j + 1] = -1; break; } } } - } } -void NsgIndex::SelectEdge(unsigned &cursor, - std::vector &sort_pool, - std::vector &result, - bool limit) { - auto &pool = sort_pool; +void +NsgIndex::SelectEdge(unsigned& cursor, std::vector& sort_pool, std::vector& result, bool limit) { + auto& pool = sort_pool; /* * edge selection @@ -587,22 +611,23 @@ void NsgIndex::SelectEdge(unsigned &cursor, */ size_t search_deepth = limit ? candidate_pool_size : pool.size(); while (result.size() < out_degree && cursor < search_deepth && (++cursor) < pool.size()) { - auto &p = pool[cursor]; + auto& p = pool[cursor]; bool should_link = true; for (size_t t = 0; t < result.size(); ++t) { - float dist = calculate(ori_data_ + dimension * result[t].id, - ori_data_ + dimension * p.id, dimension); + float dist = calculate(ori_data_ + dimension * result[t].id, ori_data_ + dimension * p.id, dimension); if (dist < p.distance) { should_link = false; break; } } - if (should_link) result.push_back(p); + if (should_link) + result.push_back(p); } } -void NsgIndex::CheckConnectivity() { +void +NsgIndex::CheckConnectivity() { auto root = navigation_point; boost::dynamic_bitset<> has_linked{ntotal, 0}; int64_t linked_count = 0; @@ -616,28 +641,29 @@ void NsgIndex::CheckConnectivity() { } } -void NsgIndex::DFS(size_t root, boost::dynamic_bitset<> &has_linked, int64_t &linked_count) { +void +NsgIndex::DFS(size_t root, boost::dynamic_bitset<>& has_linked, int64_t& linked_count) { size_t start = root; std::stack s; s.push(root); if (!has_linked[root]) { - linked_count++; // not link - has_linked[root] = true; // link start... + linked_count++; // not link + has_linked[root] = true; // link start... } while (!s.empty()) { size_t next = ntotal + 1; for (unsigned i = 0; i < nsg[start].size(); i++) { - if (has_linked[nsg[start][i]] == false) // if not link - { + if (has_linked[nsg[start][i]] == false) { // if not link next = nsg[start][i]; break; } } if (next == (ntotal + 1)) { s.pop(); - if (s.empty()) break; + if (s.empty()) + break; start = s.top(); continue; } @@ -648,17 +674,19 @@ void NsgIndex::DFS(size_t root, boost::dynamic_bitset<> &has_linked, int64_t &li } } -void NsgIndex::FindUnconnectedNode(boost::dynamic_bitset<> &has_linked, int64_t &root) { +void +NsgIndex::FindUnconnectedNode(boost::dynamic_bitset<>& has_linked, int64_t& root) { // find any of unlinked-node size_t id = ntotal; - for (size_t i = 0; i < ntotal; i++) { // find not link + for (size_t i = 0; i < ntotal; i++) { // find not link if (has_linked[i] == false) { id = i; break; } } - if (id == ntotal) return; // No Unlinked Node + if (id == ntotal) + return; // No Unlinked Node // search unlinked-node's neighbor std::vector tmp, pool; @@ -666,7 +694,7 @@ void NsgIndex::FindUnconnectedNode(boost::dynamic_bitset<> &has_linked, int64_t std::sort(pool.begin(), pool.end()); size_t found = 0; - for (size_t i = 0; i < pool.size(); i++) { // find nearest neighbor and add unlinked-node as its neighbor + for (size_t i = 0; i < pool.size(); i++) { // find nearest neighbor and add unlinked-node as its neighbor if (has_linked[pool[i].id]) { root = pool[i].id; found = 1; @@ -674,8 +702,9 @@ void NsgIndex::FindUnconnectedNode(boost::dynamic_bitset<> &has_linked, int64_t } } if (found == 0) { - while (true) { // random a linked-node and add unlinked-node as its neighbor - size_t rid = rand() % ntotal; + unsigned int seed = 100; + while (true) { // random a linked-node and add unlinked-node as its neighbor + size_t rid = rand_r(&seed) % ntotal; if (has_linked[rid]) { root = rid; break; @@ -685,22 +714,17 @@ void NsgIndex::FindUnconnectedNode(boost::dynamic_bitset<> &has_linked, int64_t nsg[root].push_back(id); } - -void NsgIndex::Search(const float *query, - const unsigned &nq, - const unsigned &dim, - const unsigned &k, - float *dist, - long *ids, - SearchParams ¶ms) { +void +NsgIndex::Search(const float* query, const unsigned& nq, const unsigned& dim, const unsigned& k, float* dist, + int64_t* ids, SearchParams& params) { std::vector> resset(nq); TimeRecorder rc("search"); if (nq == 1) { GetNeighbors(query, resset[0], nsg, ¶ms); - } else{ - //#pragma omp parallel for schedule(dynamic, 50) - #pragma omp parallel for + } else { +//#pragma omp parallel for schedule(dynamic, 50) +#pragma omp parallel for for (unsigned int i = 0; i < nq; ++i) { // TODO(linxj): when to use openmp auto single_query = query + i * dim; @@ -711,7 +735,7 @@ void NsgIndex::Search(const float *query, for (unsigned int i = 0; i < nq; ++i) { for (unsigned int j = 0; j < k; ++j) { - //ids[i * k + j] = resset[i][j].id; + // ids[i * k + j] = resset[i][j].id; // Fix(linxj): bug, reset[i][j] out of range ids[i * k + j] = ids_[resset[i][j].id]; @@ -720,27 +744,28 @@ void NsgIndex::Search(const float *query, } //>> Debug: test single insert - //int x_0 = resset[0].size(); - //for (int l = 0; l < resset[0].size(); ++l) { + // int x_0 = resset[0].size(); + // for (int l = 0; l < resset[0].size(); ++l) { // resset[0].pop_back(); //} - //resset.clear(); + // resset.clear(); - //ProfilerStart("xx.prof"); - //std::vector resset; - //GetNeighbors(query, resset, nsg, ¶ms); - //for (int i = 0; i < k; ++i) { + // ProfilerStart("xx.prof"); + // std::vector resset; + // GetNeighbors(query, resset, nsg, ¶ms); + // for (int i = 0; i < k; ++i) { // ids[i] = resset[i].id; - //dist[i] = resset[i].distance; + // dist[i] = resset[i].distance; //} - //ProfilerStop(); + // ProfilerStop(); } -void NsgIndex::SetKnnGraph(Graph &g) { +void +NsgIndex::SetKnnGraph(Graph& g) { knng = std::move(g); } -//void NsgIndex::GetKnnGraphFromFile() { +// void NsgIndex::GetKnnGraphFromFile() { // //std::string filename = "/home/zilliz/opt/workspace/wook/efanna_graph/tests/sift.1M.50NN.graph"; // std::string filename = "/home/zilliz/opt/workspace/wook/efanna_graph/tests/sift.50NN.graph"; // @@ -765,6 +790,6 @@ void NsgIndex::SetKnnGraph(Graph &g) { // in.close(); //} -} -} -} +} // namespace algo +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.h index 5291c17280..e075260398 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.h @@ -15,22 +15,19 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include -#include #include +#include #include #include "Neighbor.h" - namespace zilliz { namespace knowhere { namespace algo { - using node_t = int64_t; enum class MetricType { @@ -53,15 +50,15 @@ using Graph = std::vector>; class NsgIndex { public: size_t dimension; - size_t ntotal; // totabl nb of indexed vectors - MetricType metric_type; // L2 | IP + size_t ntotal; // totabl nb of indexed vectors + MetricType metric_type; // L2 | IP - float *ori_data_; - long *ids_; // TODO: support different type - Graph nsg; // final graph - Graph knng; // reset after build + float* ori_data_; + int64_t* ids_; // TODO: support different type + Graph nsg; // final graph + Graph knng; // reset after build - node_t navigation_point; // offset of node in origin data + node_t navigation_point; // offset of node in origin data bool is_trained = false; @@ -69,91 +66,81 @@ class NsgIndex { * build and search parameter */ size_t search_length; - size_t candidate_pool_size; // search deepth in fullset + size_t candidate_pool_size; // search deepth in fullset size_t out_degree; public: - explicit NsgIndex(const size_t &dimension, - const size_t &n, - MetricType metric = MetricType::METRIC_L2); + explicit NsgIndex(const size_t& dimension, const size_t& n, MetricType metric = MetricType::METRIC_L2); NsgIndex() = default; virtual ~NsgIndex(); - void SetKnnGraph(Graph &knng); + void + SetKnnGraph(Graph& knng); - virtual void Build_with_ids(size_t nb, - const float *data, - const long *ids, - const BuildParams ¶meters); + virtual void + Build_with_ids(size_t nb, const float* data, const int64_t* ids, const BuildParams& parameters); - void Search(const float *query, - const unsigned &nq, - const unsigned &dim, - const unsigned &k, - float *dist, - long *ids, - SearchParams ¶ms); + void + Search(const float* query, const unsigned& nq, const unsigned& dim, const unsigned& k, float* dist, int64_t* ids, + SearchParams& params); // Not support yet. - //virtual void Add() = 0; - //virtual void Add_with_ids() = 0; - //virtual void Delete() = 0; - //virtual void Delete_with_ids() = 0; - //virtual void Rebuild(size_t nb, + // virtual void Add() = 0; + // virtual void Add_with_ids() = 0; + // virtual void Delete() = 0; + // virtual void Delete_with_ids() = 0; + // virtual void Rebuild(size_t nb, // const float *data, - // const long *ids, + // const int64_t *ids, // const Parameters ¶meters) = 0; - //virtual void Build(size_t nb, + // virtual void Build(size_t nb, // const float *data, // const BuildParam ¶meters); protected: - virtual void InitNavigationPoint(); + virtual void + InitNavigationPoint(); // link specify - void GetNeighbors(const float *query, - std::vector &resset, - std::vector &fullset, - boost::dynamic_bitset<> &has_calculated_dist); + void + GetNeighbors(const float* query, std::vector& resset, std::vector& fullset, + boost::dynamic_bitset<>& has_calculated_dist); // FindUnconnectedNode - void GetNeighbors(const float *query, - std::vector &resset, - std::vector &fullset); + void + GetNeighbors(const float* query, std::vector& resset, std::vector& fullset); // search and navigation-point - void GetNeighbors(const float *query, - std::vector &resset, - Graph &graph, - SearchParams *param = nullptr); + void + GetNeighbors(const float* query, std::vector& resset, Graph& graph, SearchParams* param = nullptr); - void Link(); + void + Link(); - void SyncPrune(size_t q, - std::vector &pool, - boost::dynamic_bitset<> &has_calculated, - float *cut_graph_dist - ); + void + SyncPrune(size_t q, std::vector& pool, boost::dynamic_bitset<>& has_calculated, float* cut_graph_dist); - void SelectEdge(unsigned &cursor, - std::vector &sort_pool, - std::vector &result, - bool limit = false); + void + SelectEdge(unsigned& cursor, std::vector& sort_pool, std::vector& result, bool limit = false); - void InterInsert(unsigned n, std::vector &mutex_vec, float *dist); + void + InterInsert(unsigned n, std::vector& mutex_vec, float* dist); - void CheckConnectivity(); + void + CheckConnectivity(); - void DFS(size_t root, boost::dynamic_bitset<> &flags, int64_t &count); + void + DFS(size_t root, boost::dynamic_bitset<>& flags, int64_t& count); - void FindUnconnectedNode(boost::dynamic_bitset<> &flags, int64_t &root); + void + FindUnconnectedNode(boost::dynamic_bitset<>& flags, int64_t& root); - //private: - // void GetKnnGraphFromFile(); + // private: + // void GetKnnGraphFromFile(); }; -} -} -} +} // namespace algo +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.cpp index 77cf0e9464..1dd306bf77 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.cpp @@ -15,19 +15,18 @@ // specific language governing permissions and limitations // under the License. - #include #include -#include "NSGHelper.h" - +#include "knowhere/index/vector_index/nsg/NSGHelper.h" namespace zilliz { namespace knowhere { namespace algo { // TODO: impl search && insert && return insert pos. why not just find and swap? -int InsertIntoPool(Neighbor *addr, unsigned K, Neighbor nn) { +int +InsertIntoPool(Neighbor* addr, unsigned K, Neighbor nn) { //>> Fix: Add assert for (unsigned int i = 0; i < K; ++i) { assert(addr[i].id != nn.id); @@ -37,7 +36,7 @@ int InsertIntoPool(Neighbor *addr, unsigned K, Neighbor nn) { int left = 0, right = K - 1; if (addr[left].distance > nn.distance) { //>> Fix: memmove overflow, dump when vector deconstruct - memmove((char *) &addr[left + 1], &addr[left], (K - 1) * sizeof(Neighbor)); + memmove((char*)&addr[left + 1], &addr[left], (K - 1) * sizeof(Neighbor)); addr[left] = nn; return left; } @@ -52,10 +51,10 @@ int InsertIntoPool(Neighbor *addr, unsigned K, Neighbor nn) { else left = mid; } - //check equal ID + // check equal ID while (left > 0) { - if (addr[left].distance < nn.distance) // pos is right + if (addr[left].distance < nn.distance) // pos is right break; if (addr[left].id == nn.id) return K + 1; @@ -65,24 +64,25 @@ int InsertIntoPool(Neighbor *addr, unsigned K, Neighbor nn) { return K + 1; //>> Fix: memmove overflow, dump when vector deconstruct - memmove((char *) &addr[right + 1], &addr[right], (K - 1 - right) * sizeof(Neighbor)); + memmove((char*)&addr[right + 1], &addr[right], (K - 1 - right) * sizeof(Neighbor)); addr[right] = nn; return right; } // TODO: support L2 / IP -float calculate(const float *a, const float *b, unsigned size) { +float +calculate(const float* a, const float* b, unsigned size) { float result = 0; #ifdef __GNUC__ #ifdef __AVX__ #define AVX_L2SQR(addr1, addr2, dest, tmp1, tmp2) \ - tmp1 = _mm256_loadu_ps(addr1);\ - tmp2 = _mm256_loadu_ps(addr2);\ - tmp1 = _mm256_sub_ps(tmp1, tmp2); \ - tmp1 = _mm256_mul_ps(tmp1, tmp1); \ - dest = _mm256_add_ps(dest, tmp1); + tmp1 = _mm256_loadu_ps(addr1); \ + tmp2 = _mm256_loadu_ps(addr2); \ + tmp1 = _mm256_sub_ps(tmp1, tmp2); \ + tmp1 = _mm256_mul_ps(tmp1, tmp1); \ + dest = _mm256_add_ps(dest, tmp1); __m256 sum; __m256 l0, l1; @@ -90,14 +90,16 @@ float calculate(const float *a, const float *b, unsigned size) { unsigned D = (size + 7) & ~7U; unsigned DR = D % 16; unsigned DD = D - DR; - const float *l = a; - const float *r = b; - const float *e_l = l + DD; - const float *e_r = r + DD; - float unpack[8] __attribute__ ((aligned (32))) = {0, 0, 0, 0, 0, 0, 0, 0}; + const float* l = a; + const float* r = b; + const float* e_l = l + DD; + const float* e_r = r + DD; + float unpack[8] __attribute__((aligned(32))) = {0, 0, 0, 0, 0, 0, 0, 0}; sum = _mm256_loadu_ps(unpack); - if (DR) { AVX_L2SQR(e_l, e_r, sum, l0, r0); } + if (DR) { + AVX_L2SQR(e_l, e_r, sum, l0, r0); + } for (unsigned i = 0; i < DD; i += 16, l += 16, r += 16) { AVX_L2SQR(l, r, sum, l0, r0); @@ -109,11 +111,11 @@ float calculate(const float *a, const float *b, unsigned size) { #else #ifdef __SSE2__ #define SSE_L2SQR(addr1, addr2, dest, tmp1, tmp2) \ - tmp1 = _mm_load_ps(addr1);\ - tmp2 = _mm_load_ps(addr2);\ - tmp1 = _mm_sub_ps(tmp1, tmp2); \ - tmp1 = _mm_mul_ps(tmp1, tmp1); \ - dest = _mm_add_ps(dest, tmp1); + tmp1 = _mm_load_ps(addr1); \ + tmp2 = _mm_load_ps(addr2); \ + tmp1 = _mm_sub_ps(tmp1, tmp2); \ + tmp1 = _mm_mul_ps(tmp1, tmp1); \ + dest = _mm_add_ps(dest, tmp1); __m128 sum; __m128 l0, l1, l2, l3; @@ -121,18 +123,22 @@ float calculate(const float *a, const float *b, unsigned size) { unsigned D = (size + 3) & ~3U; unsigned DR = D % 16; unsigned DD = D - DR; - const float *l = a; - const float *r = b; - const float *e_l = l + DD; - const float *e_r = r + DD; - float unpack[4] __attribute__ ((aligned (16))) = {0, 0, 0, 0}; + const float* l = a; + const float* r = b; + const float* e_l = l + DD; + const float* e_r = r + DD; + float unpack[4] __attribute__((aligned(16))) = {0, 0, 0, 0}; sum = _mm_load_ps(unpack); switch (DR) { - case 12:SSE_L2SQR(e_l + 8, e_r + 8, sum, l2, r2); - case 8:SSE_L2SQR(e_l + 4, e_r + 4, sum, l1, r1); - case 4:SSE_L2SQR(e_l, e_r, sum, l0, r0); - default:break; + case 12: + SSE_L2SQR(e_l + 8, e_r + 8, sum, l2, r2); + case 8: + SSE_L2SQR(e_l + 4, e_r + 4, sum, l1, r1); + case 4: + SSE_L2SQR(e_l, e_r, sum, l0, r0); + default: + break; } for (unsigned i = 0; i < DD; i += 16, l += 16, r += 16) { SSE_L2SQR(l, r, sum, l0, r0); @@ -143,28 +149,28 @@ float calculate(const float *a, const float *b, unsigned size) { _mm_storeu_ps(unpack, sum); result += unpack[0] + unpack[1] + unpack[2] + unpack[3]; -//nomal distance +// nomal distance #else float diff0, diff1, diff2, diff3; - const float* last = a + size; - const float* unroll_group = last - 3; + const float* last = a + size; + const float* unroll_group = last - 3; - /* Process 4 items with each loop for efficiency. */ - while (a < unroll_group) { - diff0 = a[0] - b[0]; - diff1 = a[1] - b[1]; - diff2 = a[2] - b[2]; - diff3 = a[3] - b[3]; - result += diff0 * diff0 + diff1 * diff1 + diff2 * diff2 + diff3 * diff3; - a += 4; - b += 4; - } - /* Process last 0-3 pixels. Not needed for standard vector lengths. */ - while (a < last) { - diff0 = *a++ - *b++; - result += diff0 * diff0; - } + /* Process 4 items with each loop for efficiency. */ + while (a < unroll_group) { + diff0 = a[0] - b[0]; + diff1 = a[1] - b[1]; + diff2 = a[2] - b[2]; + diff3 = a[3] - b[3]; + result += diff0 * diff0 + diff1 * diff1 + diff2 * diff2 + diff3 * diff3; + a += 4; + b += 4; + } + /* Process last 0-3 pixels. Not needed for standard vector lengths. */ + while (a < last) { + diff0 = *a++ - *b++; + result += diff0 * diff0; + } #endif #endif #endif @@ -172,7 +178,6 @@ float calculate(const float *a, const float *b, unsigned size) { return result; } - -} -} -} \ No newline at end of file +} // namespace algo +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.h index f5c13194c4..cb8007f2b1 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.h @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include @@ -26,14 +25,15 @@ #include "NSG.h" #include "knowhere/common/Config.h" - namespace zilliz { namespace knowhere { namespace algo { -extern int InsertIntoPool(Neighbor *addr, unsigned K, Neighbor nn); -extern float calculate(const float *a, const float *b, unsigned size); +extern int +InsertIntoPool(Neighbor* addr, unsigned K, Neighbor nn); +extern float +calculate(const float* a, const float* b, unsigned size); -} -} -} +} // namespace algo +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.cpp b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.cpp index bcdde8052c..db3113da2f 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.cpp +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.cpp @@ -15,31 +15,31 @@ // specific language governing permissions and limitations // under the License. - #include -#include "NSGIO.h" - +#include "knowhere/index/vector_index/nsg/NSGIO.h" namespace zilliz { namespace knowhere { namespace algo { -void write_index(NsgIndex *index, MemoryIOWriter &writer) { +void +write_index(NsgIndex* index, MemoryIOWriter& writer) { writer(&index->ntotal, sizeof(index->ntotal), 1); writer(&index->dimension, sizeof(index->dimension), 1); writer(&index->navigation_point, sizeof(index->navigation_point), 1); writer(index->ori_data_, sizeof(float) * index->ntotal * index->dimension, 1); - writer(index->ids_, sizeof(long) * index->ntotal, 1); + writer(index->ids_, sizeof(int64_t) * index->ntotal, 1); for (unsigned i = 0; i < index->ntotal; ++i) { - auto neighbor_num = (node_t) index->nsg[i].size(); + auto neighbor_num = (node_t)index->nsg[i].size(); writer(&neighbor_num, sizeof(node_t), 1); writer(index->nsg[i].data(), neighbor_num * sizeof(node_t), 1); } } -NsgIndex *read_index(MemoryIOReader &reader) { +NsgIndex* +read_index(MemoryIOReader& reader) { size_t ntotal; size_t dimension; reader(&ntotal, sizeof(size_t), 1); @@ -48,9 +48,9 @@ NsgIndex *read_index(MemoryIOReader &reader) { reader(&index->navigation_point, sizeof(index->navigation_point), 1); index->ori_data_ = new float[index->ntotal * index->dimension]; - index->ids_ = new long[index->ntotal]; + index->ids_ = new int64_t[index->ntotal]; reader(index->ori_data_, sizeof(float) * index->ntotal * index->dimension, 1); - reader(index->ids_, sizeof(long) * index->ntotal, 1); + reader(index->ids_, sizeof(int64_t) * index->ntotal, 1); index->nsg.reserve(index->ntotal); index->nsg.resize(index->ntotal); @@ -66,6 +66,6 @@ NsgIndex *read_index(MemoryIOReader &reader) { return index; } -} -} -} +} // namespace algo +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.h index 3d6786c6c2..fd740a991e 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.h @@ -15,21 +15,21 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include "knowhere/index/vector_index/helpers/FaissIO.h" #include "NSG.h" #include "knowhere/index/vector_index/IndexIVF.h" - +#include "knowhere/index/vector_index/helpers/FaissIO.h" namespace zilliz { namespace knowhere { namespace algo { -extern void write_index(NsgIndex* index, MemoryIOWriter& writer); -extern NsgIndex* read_index(MemoryIOReader& reader); +extern void +write_index(NsgIndex* index, MemoryIOWriter& writer); +extern NsgIndex* +read_index(MemoryIOReader& reader); -} -} -} +} // namespace algo +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/Neighbor.h b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/Neighbor.h index 9aceb62692..ee71a3f66e 100644 --- a/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/Neighbor.h +++ b/cpp/src/core/knowhere/knowhere/index/vector_index/nsg/Neighbor.h @@ -15,12 +15,10 @@ // specific language governing permissions and limitations // under the License. - #pragma once #include - namespace zilliz { namespace knowhere { namespace algo { @@ -29,21 +27,25 @@ using node_t = int64_t; // TODO: search use simple neighbor struct Neighbor { - node_t id; // offset of node in origin data + node_t id; // offset of node in origin data float distance; bool has_explored; Neighbor() = default; - explicit Neighbor(node_t id, float distance, bool f) : id{id}, distance{distance}, has_explored(f) {} - explicit Neighbor(node_t id, float distance) : id{id}, distance{distance}, has_explored(false) {} + explicit Neighbor(node_t id, float distance, bool f) : id{id}, distance{distance}, has_explored(f) { + } - inline bool operator<(const Neighbor &other) const { + explicit Neighbor(node_t id, float distance) : id{id}, distance{distance}, has_explored(false) { + } + + inline bool + operator<(const Neighbor& other) const { return distance < other.distance; } }; -//struct SimpleNeighbor { +// struct SimpleNeighbor { // node_t id; // offset of node in origin data // float distance; // @@ -57,7 +59,6 @@ struct Neighbor { typedef std::lock_guard LockGuard; - -} -} -} \ No newline at end of file +} // namespace algo +} // namespace knowhere +} // namespace zilliz diff --git a/cpp/src/core/test/CMakeLists.txt b/cpp/src/core/test/CMakeLists.txt index 5ad1c9e00e..ecb52695bc 100644 --- a/cpp/src/core/test/CMakeLists.txt +++ b/cpp/src/core/test/CMakeLists.txt @@ -1,6 +1,7 @@ include_directories(${CORE_SOURCE_DIR}/thirdparty) include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService) include_directories(${CORE_SOURCE_DIR}/knowhere) +include_directories(${CORE_SOURCE_DIR}) include_directories(/usr/local/cuda/include) link_directories(/usr/local/cuda/lib64) link_directories(${CORE_SOURCE_DIR}/thirdparty/tbb) diff --git a/cpp/src/core/test/SPTAG.cpp b/cpp/src/core/test/SPTAG.cpp index 3dfea0f088..62198d5495 100644 --- a/cpp/src/core/test/SPTAG.cpp +++ b/cpp/src/core/test/SPTAG.cpp @@ -1,36 +1,50 @@ - -#include -#include -#include -#include "SPTAG/AnnService/inc/Core/Common.h" -#include "SPTAG/AnnService/inc/Core/VectorIndex.h" - - -int -main(int argc, char *argv[]) { - using namespace SPTAG; - const int d = 128; - const int n = 100; - - auto p_data = new float[n * d]; - - auto index = VectorIndex::CreateInstance(IndexAlgoType::KDT, VectorValueType::Float); - - std::random_device rd; - std::mt19937 mt(rd()); - std::uniform_real_distribution dist(1.0, 2.0); - - for (auto i = 0; i < n; i++) { - for (auto j = 0; j < d; j++) { - p_data[i * d + j] = dist(mt) - 1; - } - } - std::cout << "generate random n * d finished."; - ByteArray data((uint8_t *) p_data, n * d * sizeof(float), true); - - auto vectorset = std::make_shared(data, VectorValueType::Float, d, n); - index->BuildIndex(vectorset, nullptr); - - std::cout << index->GetFeatureDim(); -} - +//// Licensed to the Apache Software Foundation (ASF) under one +//// or more contributor license agreements. See the NOTICE file +//// distributed with this work for additional information +//// regarding copyright ownership. The ASF licenses this file +//// to you under the Apache License, Version 2.0 (the +//// "License"); you may not use this file except in compliance +//// with the License. You may obtain a copy of the License at +//// +//// http://www.apache.org/licenses/LICENSE-2.0 +//// +//// Unless required by applicable law or agreed to in writing, +//// software distributed under the License is distributed on an +//// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//// KIND, either express or implied. See the License for the +//// specific language governing permissions and limitations +//// under the License. +// +//#include +//#include +//#include +//#include +//#include +// +// int +// main(int argc, char* argv[]) { +// using namespace SPTAG; +// const int d = 128; +// const int n = 100; +// +// auto p_data = new float[n * d]; +// +// auto index = VectorIndex::CreateInstance(IndexAlgoType::KDT, VectorValueType::Float); +// +// std::random_device rd; +// std::mt19937 mt(rd()); +// std::uniform_real_distribution dist(1.0, 2.0); +// +// for (auto i = 0; i < n; i++) { +// for (auto j = 0; j < d; j++) { +// p_data[i * d + j] = dist(mt) - 1; +// } +// } +// std::cout << "generate random n * d finished."; +// ByteArray data((uint8_t*)p_data, n * d * sizeof(float), true); +// +// auto vectorset = std::make_shared(data, VectorValueType::Float, d, n); +// index->BuildIndex(vectorset, nullptr); +// +// std::cout << index->GetFeatureDim(); +//} diff --git a/cpp/src/core/test/faiss_ori/gpuresource_test.cpp b/cpp/src/core/test/faiss_ori/gpuresource_test.cpp index 0dcc2766cc..90383b944c 100644 --- a/cpp/src/core/test/faiss_ori/gpuresource_test.cpp +++ b/cpp/src/core/test/faiss_ori/gpuresource_test.cpp @@ -17,46 +17,45 @@ #include -#include -#include -#include #include +#include #include +#include +#include #include -#include #include #include - -using namespace std::chrono_literals; +#include class TestGpuRes { public: TestGpuRes() { res_ = new faiss::gpu::StandardGpuResources; } + ~TestGpuRes() { delete res_; delete index_; } - std::shared_ptr Do() { - int d = 128; // dimension - int nb = 100000; // database size - int nq = 100; // nb of queries + + std::shared_ptr + Do() { + int d = 128; // dimension + int nb = 100000; // database size + int nq = 100; // nb of queries int nlist = 1638; - float *xb = new float[d * nb]; - float *xq = new float[d * nq]; + float* xb = new float[d * nb]; + float* xq = new float[d * nq]; for (int i = 0; i < nb; i++) { - for (int j = 0; j < d; j++) - xb[d * i + j] = drand48(); + for (int j = 0; j < d; j++) xb[d * i + j] = drand48(); xb[d * i] += i / 1000.; } for (int i = 0; i < nq; i++) { - for (int j = 0; j < d; j++) - xq[d * i + j] = drand48(); + for (int j = 0; j < d; j++) xq[d * i + j] = drand48(); xq[d * i] += i / 1000.; } @@ -68,9 +67,10 @@ class TestGpuRes { host_index.reset(faiss::gpu::index_gpu_to_cpu(index_)); return host_index; } + private: - faiss::gpu::GpuResources *res_ = nullptr; - faiss::Index *index_ = nullptr; + faiss::gpu::GpuResources* res_ = nullptr; + faiss::Index* index_ = nullptr; }; TEST(gpuresource, resource) { @@ -79,30 +79,28 @@ TEST(gpuresource, resource) { } TEST(test, resource_re) { - int d = 128; // dimension - int nb = 1000000; // database size - int nq = 100; // nb of queries + int d = 128; // dimension + int nb = 1000000; // database size + int nq = 100; // nb of queries int nlist = 16384; int k = 100; - float *xb = new float[d * nb]; - float *xq = new float[d * nq]; + float* xb = new float[d * nb]; + float* xq = new float[d * nq]; for (int i = 0; i < nb; i++) { - for (int j = 0; j < d; j++) - xb[d * i + j] = drand48(); + for (int j = 0; j < d; j++) xb[d * i + j] = drand48(); xb[d * i] += i / 1000.; } for (int i = 0; i < nq; i++) { - for (int j = 0; j < d; j++) - xq[d * i + j] = drand48(); + for (int j = 0; j < d; j++) xq[d * i + j] = drand48(); xq[d * i] += i / 1000.; } auto elems = nq * k; - auto res_ids = (int64_t *) malloc(sizeof(int64_t) * elems); - auto res_dis = (float *) malloc(sizeof(float) * elems); + auto res_ids = (int64_t*)malloc(sizeof(int64_t) * elems); + auto res_dis = (float*)malloc(sizeof(float) * elems); faiss::gpu::StandardGpuResources res; auto cpu_index = faiss::index_factory(d, "IVF16384, Flat"); @@ -117,7 +115,7 @@ TEST(test, resource_re) { auto load = [&] { std::cout << "start" << std::endl; faiss::gpu::StandardGpuResources res; - //res.noTempMemory(); + // res.noTempMemory(); for (int l = 0; l < 100; ++l) { auto x = faiss::gpu::index_cpu_to_gpu(&res, 1, new_index); delete x; @@ -126,42 +124,42 @@ TEST(test, resource_re) { }; auto search = [&] { - faiss::gpu::StandardGpuResources res; - auto device_index = faiss::gpu::index_cpu_to_gpu(&res, 1, new_index); - std::cout << "search start" << std::endl; - for (int l = 0; l < 10000; ++l) { - device_index->search(nq,xq,10, res_dis, res_ids); - } - std::cout << "search finish" << std::endl; - delete device_index; - delete cpu_index; + faiss::gpu::StandardGpuResources res; + auto device_index = faiss::gpu::index_cpu_to_gpu(&res, 1, new_index); + std::cout << "search start" << std::endl; + for (int l = 0; l < 10000; ++l) { + device_index->search(nq, xq, 10, res_dis, res_ids); + } + std::cout << "search finish" << std::endl; + delete device_index; + delete cpu_index; }; load(); search(); std::thread t1(search); - std::this_thread::sleep_for(1s); + std::this_thread::sleep_for(std::chrono::seconds(1)); std::thread t2(load); t1.join(); t2.join(); std::cout << "finish clone" << std::endl; - //std::this_thread::sleep_for(5s); + // std::this_thread::sleep_for(5s); // - //auto device_index_2 = faiss::gpu::index_cpu_to_gpu(&res, 1, cpu_index); - //device_index->train(nb, xb); - //device_index->add(nb, xb); + // auto device_index_2 = faiss::gpu::index_cpu_to_gpu(&res, 1, cpu_index); + // device_index->train(nb, xb); + // device_index->add(nb, xb); - //std::cout << "finish clone" << std::endl; - //std::this_thread::sleep_for(5s); + // std::cout << "finish clone" << std::endl; + // std::this_thread::sleep_for(5s); - //std::this_thread::sleep_for(2s); - //std::cout << "start clone" << std::endl; - //auto new_index = faiss::clone_index(device_index); - //std::cout << "start search" << std::endl; - //new_index->search(nq, xq, k, res_dis, res_ids); + // std::this_thread::sleep_for(2s); + // std::cout << "start clone" << std::endl; + // auto new_index = faiss::clone_index(device_index); + // std::cout << "start search" << std::endl; + // new_index->search(nq, xq, k, res_dis, res_ids); - //std::cout << "start clone" << std::endl; + // std::cout << "start clone" << std::endl; //{ // faiss::gpu::StandardGpuResources res; // auto cpu_index = faiss::index_factory(d, "IVF1638, Flat"); @@ -174,5 +172,5 @@ TEST(test, resource_re) { // std::cout << "finish clone" << std::endl; //} // - //std::cout << "finish clone" << std::endl; + // std::cout << "finish clone" << std::endl; } diff --git a/cpp/src/core/test/kdtree.cpp b/cpp/src/core/test/kdtree.cpp index b3de8f345c..351a177d4b 100644 --- a/cpp/src/core/test/kdtree.cpp +++ b/cpp/src/core/test/kdtree.cpp @@ -1,134 +1,149 @@ - -#include -#include -#include "knowhere/index/vector_index/cpu_kdt_rng.h" -#include "knowhere/index/vector_index/definitions.h" -#include "knowhere/adapter/sptag.h" -#include "knowhere/adapter/structure.h" - - -using namespace zilliz::knowhere; - -DatasetPtr -generate_dataset(int64_t n, int64_t d, int64_t base) { - auto elems = n * d; - auto p_data = (float *) malloc(elems * sizeof(float)); - auto p_id = (int64_t *) malloc(elems * sizeof(int64_t)); - assert(p_data != nullptr && p_id != nullptr); - - for (auto i = 0; i < n; ++i) { - for (auto j = 0; j < d; ++j) { - p_data[i * d + j] = float(base + i); - } - p_id[i] = i; - } - - std::vector shape{n, d}; - auto tensor = ConstructFloatTensorSmart((uint8_t *) p_data, elems * sizeof(float), shape); - std::vector tensors{tensor}; - std::vector tensor_fields{ConstructFloatField("data")}; - auto tensor_schema = std::make_shared(tensor_fields); - - auto id_array = ConstructInt64ArraySmart((uint8_t *) p_id, n * sizeof(int64_t)); - std::vector arrays{id_array}; - std::vector array_fields{ConstructInt64Field("id")}; - auto array_schema = std::make_shared(tensor_fields); - - auto dataset = std::make_shared(std::move(arrays), array_schema, - std::move(tensors), tensor_schema); - - return dataset; -} - -DatasetPtr -generate_queries(int64_t n, int64_t d, int64_t k, int64_t base) { - size_t size = sizeof(float) * n * d; - auto v = (float *) malloc(size); - // TODO(lxj): check malloc - for (auto i = 0; i < n; ++i) { - for (auto j = 0; j < d; ++j) { - v[i * d + j] = float(base + i); - } - } - - std::vector data; - auto buffer = MakeMutableBufferSmart((uint8_t *) v, size); - std::vector shape{n, d}; - auto float_type = std::make_shared(); - auto tensor = std::make_shared(float_type, buffer, shape); - data.push_back(tensor); - - Config meta; - meta[META_ROWS] = int64_t (n); - meta[META_DIM] = int64_t (d); - meta[META_K] = int64_t (k); - - auto type = std::make_shared(); - auto field = std::make_shared("data", type); - std::vector fields{field}; - auto schema = std::make_shared(fields); - - return std::make_shared(data, schema); -} - - -int -main(int argc, char *argv[]) { - auto kdt_index = std::make_shared(); - - const auto d = 10; - const auto k = 3; - const auto nquery = 10; - - // ID [0, 99] - auto train = generate_dataset(100, d, 0); - // ID [100] - auto base = generate_dataset(1, d, 0); - auto queries = generate_queries(nquery, d, k, 0); - - // Build Preprocessor - auto preprocessor = kdt_index->BuildPreprocessor(train, Config()); - - // Set Preprocessor - kdt_index->set_preprocessor(preprocessor); - - Config train_config; - train_config["TPTNumber"] = "64"; - // Train - kdt_index->Train(train, train_config); - - // Add - kdt_index->Add(base, Config()); - - auto binary = kdt_index->Serialize(); - auto new_index = std::make_shared(); - new_index->Load(binary); -// auto new_index = kdt_index; - - Config search_config; - search_config[META_K] = int64_t (k); - - // Search - auto result = new_index->Search(queries, search_config); - - // Print Result - { - auto ids = result->array()[0]; - auto dists = result->array()[1]; - - std::stringstream ss_id; - std::stringstream ss_dist; - for (auto i = 0; i < nquery; i++) { - for (auto j = 0; j < k; ++j) { - ss_id << *ids->data()->GetValues(1, i * k + j) << " "; - ss_dist << *dists->data()->GetValues(1, i * k + j) << " "; - } - ss_id << std::endl; - ss_dist << std::endl; - } - std::cout << "id\n" << ss_id.str() << std::endl; - std::cout << "dist\n" << ss_dist.str() << std::endl; - } -} - - +//// Licensed to the Apache Software Foundation (ASF) under one +//// or more contributor license agreements. See the NOTICE file +//// distributed with this work for additional information +//// regarding copyright ownership. The ASF licenses this file +//// to you under the Apache License, Version 2.0 (the +//// "License"); you may not use this file except in compliance +//// with the License. You may obtain a copy of the License at +//// +//// http://www.apache.org/licenses/LICENSE-2.0 +//// +//// Unless required by applicable law or agreed to in writing, +//// software distributed under the License is distributed on an +//// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//// KIND, either express or implied. See the License for the +//// specific language governing permissions and limitations +//// under the License. +// +//#include +//#include +//#include "knowhere/adapter/sptag.h" +//#include "knowhere/adapter/structure.h" +//#include "knowhere/index/vector_index/cpu_kdt_rng.h" +//#include "knowhere/index/vector_index/definitions.h" +// +// namespace { +// +// namespace kn = zilliz::knowhere; +// +//} // namespace +// +// kn::DatasetPtr +// generate_dataset(int64_t n, int64_t d, int64_t base) { +// auto elems = n * d; +// auto p_data = (float*)malloc(elems * sizeof(float)); +// auto p_id = (int64_t*)malloc(elems * sizeof(int64_t)); +// assert(p_data != nullptr && p_id != nullptr); +// +// for (auto i = 0; i < n; ++i) { +// for (auto j = 0; j < d; ++j) { +// p_data[i * d + j] = float(base + i); +// } +// p_id[i] = i; +// } +// +// std::vector shape{n, d}; +// auto tensor = ConstructFloatTensorSmart((uint8_t*)p_data, elems * sizeof(float), shape); +// std::vector tensors{tensor}; +// std::vector tensor_fields{ConstructFloatField("data")}; +// auto tensor_schema = std::make_shared(tensor_fields); +// +// auto id_array = ConstructInt64ArraySmart((uint8_t*)p_id, n * sizeof(int64_t)); +// std::vector arrays{id_array}; +// std::vector array_fields{ConstructInt64Field("id")}; +// auto array_schema = std::make_shared(tensor_fields); +// +// auto dataset = std::make_shared(std::move(arrays), array_schema, std::move(tensors), tensor_schema); +// +// return dataset; +//} +// +// kn::DatasetPtr +// generate_queries(int64_t n, int64_t d, int64_t k, int64_t base) { +// size_t size = sizeof(float) * n * d; +// auto v = (float*)malloc(size); +// // TODO(lxj): check malloc +// for (auto i = 0; i < n; ++i) { +// for (auto j = 0; j < d; ++j) { +// v[i * d + j] = float(base + i); +// } +// } +// +// std::vector data; +// auto buffer = MakeMutableBufferSmart((uint8_t*)v, size); +// std::vector shape{n, d}; +// auto float_type = std::make_shared(); +// auto tensor = std::make_shared(float_type, buffer, shape); +// data.push_back(tensor); +// +// Config meta; +// meta[META_ROWS] = int64_t(n); +// meta[META_DIM] = int64_t(d); +// meta[META_K] = int64_t(k); +// +// auto type = std::make_shared(); +// auto field = std::make_shared("data", type); +// std::vector fields{field}; +// auto schema = std::make_shared(fields); +// +// return std::make_shared(data, schema); +//} +// +// int +// main(int argc, char* argv[]) { +// auto kdt_index = std::make_shared(); +// +// const auto d = 10; +// const auto k = 3; +// const auto nquery = 10; +// +// // ID [0, 99] +// auto train = generate_dataset(100, d, 0); +// // ID [100] +// auto base = generate_dataset(1, d, 0); +// auto queries = generate_queries(nquery, d, k, 0); +// +// // Build Preprocessor +// auto preprocessor = kdt_index->BuildPreprocessor(train, Config()); +// +// // Set Preprocessor +// kdt_index->set_preprocessor(preprocessor); +// +// Config train_config; +// train_config["TPTNumber"] = "64"; +// // Train +// kdt_index->Train(train, train_config); +// +// // Add +// kdt_index->Add(base, Config()); +// +// auto binary = kdt_index->Serialize(); +// auto new_index = std::make_shared(); +// new_index->Load(binary); +// // auto new_index = kdt_index; +// +// Config search_config; +// search_config[META_K] = int64_t(k); +// +// // Search +// auto result = new_index->Search(queries, search_config); +// +// // Print Result +// { +// auto ids = result->array()[0]; +// auto dists = result->array()[1]; +// +// std::stringstream ss_id; +// std::stringstream ss_dist; +// for (auto i = 0; i < nquery; i++) { +// for (auto j = 0; j < k; ++j) { +// ss_id << *ids->data()->GetValues(1, i * k + j) << " "; +// ss_dist << *dists->data()->GetValues(1, i * k + j) << " "; +// } +// ss_id << std::endl; +// ss_dist << std::endl; +// } +// std::cout << "id\n" << ss_id.str() << std::endl; +// std::cout << "dist\n" << ss_dist.str() << std::endl; +// } +//} diff --git a/cpp/src/core/test/test_idmap.cpp b/cpp/src/core/test/test_idmap.cpp index 3d71bca931..8bf0d5c8d2 100644 --- a/cpp/src/core/test/test_idmap.cpp +++ b/cpp/src/core/test/test_idmap.cpp @@ -15,51 +15,51 @@ // specific language governing permissions and limitations // under the License. - #include - #include -#include "knowhere/index/vector_index/IndexIDMAP.h" #include "knowhere/adapter/Structure.h" -#include "knowhere/index/vector_index/helpers/Cloner.h" #include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/IndexIDMAP.h" +#include "knowhere/index/vector_index/helpers/Cloner.h" -#include "utils.h" +#include "test/utils.h" +namespace { -using namespace zilliz::knowhere; -using namespace zilliz::knowhere::cloner; +namespace kn = zilliz::knowhere; + +} // namespace static int device_id = 0; class IDMAPTest : public DataGen, public ::testing::Test { protected: - void SetUp() override { - FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024*1024*200, 1024*1024*300, 2); + void + SetUp() override { + kn::FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024 * 1024 * 200, 1024 * 1024 * 300, 2); Init_with_default(); - index_ = std::make_shared(); + index_ = std::make_shared(); } - void TearDown() override { - FaissGpuResourceMgr::GetInstance().Free(); + void + TearDown() override { + kn::FaissGpuResourceMgr::GetInstance().Free(); } protected: - IDMAPPtr index_ = nullptr; + kn::IDMAPPtr index_ = nullptr; }; -void AssertAnns(const DatasetPtr &result, - const int &nq, - const int &k) { +void +AssertAnns(const kn::DatasetPtr& result, const int& nq, const int& k) { auto ids = result->array()[0]; for (auto i = 0; i < nq; i++) { EXPECT_EQ(i, *(ids->data()->GetValues(1, i * k))); } } -void PrintResult(const DatasetPtr &result, - const int &nq, - const int &k) { +void +PrintResult(const kn::DatasetPtr& result, const int& nq, const int& k) { auto ids = result->array()[0]; auto dists = result->array()[1]; @@ -80,10 +80,10 @@ void PrintResult(const DatasetPtr &result, TEST_F(IDMAPTest, idmap_basic) { ASSERT_TRUE(!xb.empty()); - auto conf = std::make_shared(); + auto conf = std::make_shared(); conf->d = dim; conf->k = k; - conf->metric_type = METRICTYPE::L2; + conf->metric_type = kn::METRICTYPE::L2; index_->Train(conf); index_->Add(base_dataset, conf); @@ -97,7 +97,7 @@ TEST_F(IDMAPTest, idmap_basic) { index_->Seal(); auto binaryset = index_->Serialize(); - auto new_index = std::make_shared(); + auto new_index = std::make_shared(); new_index->Load(binaryset); auto re_result = index_->Search(query_dataset, conf); AssertAnns(re_result, nq, k); @@ -105,23 +105,23 @@ TEST_F(IDMAPTest, idmap_basic) { } TEST_F(IDMAPTest, idmap_serialize) { - auto serialize = [](const std::string &filename, BinaryPtr &bin, uint8_t *ret) { + auto serialize = [](const std::string& filename, kn::BinaryPtr& bin, uint8_t* ret) { FileIOWriter writer(filename); - writer(static_cast(bin->data.get()), bin->size); + writer(static_cast(bin->data.get()), bin->size); FileIOReader reader(filename); reader(ret, bin->size); }; - auto conf = std::make_shared(); + auto conf = std::make_shared(); conf->d = dim; conf->k = k; - conf->metric_type = METRICTYPE::L2; + conf->metric_type = kn::METRICTYPE::L2; { // serialize index index_->Train(conf); - index_->Add(base_dataset, Config()); + index_->Add(base_dataset, kn::Config()); auto re_result = index_->Search(query_dataset, conf); AssertAnns(re_result, nq, k); PrintResult(re_result, nq, k); @@ -151,10 +151,10 @@ TEST_F(IDMAPTest, idmap_serialize) { TEST_F(IDMAPTest, copy_test) { ASSERT_TRUE(!xb.empty()); - auto conf = std::make_shared(); + auto conf = std::make_shared(); conf->d = dim; conf->k = k; - conf->metric_type = METRICTYPE::L2; + conf->metric_type = kn::METRICTYPE::L2; index_->Train(conf); index_->Add(base_dataset, conf); @@ -164,7 +164,7 @@ TEST_F(IDMAPTest, copy_test) { ASSERT_TRUE(index_->GetRawIds() != nullptr); auto result = index_->Search(query_dataset, conf); AssertAnns(result, nq, k); - //PrintResult(result, nq, k); + // PrintResult(result, nq, k); { // clone @@ -175,12 +175,12 @@ TEST_F(IDMAPTest, copy_test) { { // cpu to gpu - auto clone_index = CopyCpuToGpu(index_, device_id, conf); + auto clone_index = kn::cloner::CopyCpuToGpu(index_, device_id, conf); auto clone_result = clone_index->Search(query_dataset, conf); AssertAnns(clone_result, nq, k); - ASSERT_THROW({ std::static_pointer_cast(clone_index)->GetRawVectors(); }, + ASSERT_THROW({ std::static_pointer_cast(clone_index)->GetRawVectors(); }, zilliz::knowhere::KnowhereException); - ASSERT_THROW({ std::static_pointer_cast(clone_index)->GetRawIds(); }, + ASSERT_THROW({ std::static_pointer_cast(clone_index)->GetRawIds(); }, zilliz::knowhere::KnowhereException); auto binary = clone_index->Serialize(); @@ -193,15 +193,15 @@ TEST_F(IDMAPTest, copy_test) { AssertAnns(clone_gpu_res, nq, k); // gpu to cpu - auto host_index = CopyGpuToCpu(clone_index, conf); + auto host_index = kn::cloner::CopyGpuToCpu(clone_index, conf); auto host_result = host_index->Search(query_dataset, conf); AssertAnns(host_result, nq, k); - ASSERT_TRUE(std::static_pointer_cast(host_index)->GetRawVectors() != nullptr); - ASSERT_TRUE(std::static_pointer_cast(host_index)->GetRawIds() != nullptr); + ASSERT_TRUE(std::static_pointer_cast(host_index)->GetRawVectors() != nullptr); + ASSERT_TRUE(std::static_pointer_cast(host_index)->GetRawIds() != nullptr); // gpu to gpu - auto device_index = CopyCpuToGpu(index_, device_id, conf); - auto new_device_index = std::static_pointer_cast(device_index)->CopyGpuToGpu(device_id, conf); + auto device_index = kn::cloner::CopyCpuToGpu(index_, device_id, conf); + auto new_device_index = std::static_pointer_cast(device_index)->CopyGpuToGpu(device_id, conf); auto device_result = new_device_index->Search(query_dataset, conf); AssertAnns(device_result, nq, k); } diff --git a/cpp/src/core/test/test_ivf.cpp b/cpp/src/core/test/test_ivf.cpp index 7196fe15e4..9835264bff 100644 --- a/cpp/src/core/test/test_ivf.cpp +++ b/cpp/src/core/test/test_ivf.cpp @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - #include #include @@ -25,21 +24,24 @@ #include #include +#include "knowhere/adapter/Structure.h" #include "knowhere/common/Exception.h" #include "knowhere/common/Timer.h" -#include "knowhere/adapter/Structure.h" -#include "knowhere/index/vector_index/helpers/Cloner.h" -#include "knowhere/index/vector_index/IndexIVF.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" -#include "knowhere/index/vector_index/IndexIVFPQ.h" #include "knowhere/index/vector_index/IndexGPUIVFPQ.h" -#include "knowhere/index/vector_index/IndexIVFSQ.h" #include "knowhere/index/vector_index/IndexGPUIVFSQ.h" +#include "knowhere/index/vector_index/IndexIVF.h" +#include "knowhere/index/vector_index/IndexIVFPQ.h" +#include "knowhere/index/vector_index/IndexIVFSQ.h" +#include "knowhere/index/vector_index/helpers/Cloner.h" -#include "utils.h" +#include "test/utils.h" -using namespace zilliz::knowhere; -using namespace zilliz::knowhere::cloner; +namespace { + +namespace kn = zilliz::knowhere; + +} // namespace using ::testing::TestWithParam; using ::testing::Values; @@ -47,23 +49,24 @@ using ::testing::Combine; constexpr int device_id = 0; constexpr int64_t DIM = 128; -constexpr int64_t NB = 1000000/100; +constexpr int64_t NB = 1000000 / 100; constexpr int64_t NQ = 10; constexpr int64_t K = 10; -IVFIndexPtr IndexFactory(const std::string &type) { +kn::IVFIndexPtr +IndexFactory(const std::string& type) { if (type == "IVF") { - return std::make_shared(); + return std::make_shared(); } else if (type == "IVFPQ") { - return std::make_shared(); + return std::make_shared(); } else if (type == "GPUIVF") { - return std::make_shared(device_id); + return std::make_shared(device_id); } else if (type == "GPUIVFPQ") { - return std::make_shared(device_id); + return std::make_shared(device_id); } else if (type == "IVFSQ") { - return std::make_shared(); + return std::make_shared(); } else if (type == "GPUIVFSQ") { - return std::make_shared(device_id); + return std::make_shared(device_id); } } @@ -76,24 +79,25 @@ enum class ParameterType { class ParamGenerator { public: - static ParamGenerator& GetInstance(){ + static ParamGenerator& + GetInstance() { static ParamGenerator instance; return instance; } - Config Gen(const ParameterType& type){ + kn::Config + Gen(const ParameterType& type) { if (type == ParameterType::ivf) { - auto tempconf = std::make_shared(); + auto tempconf = std::make_shared(); tempconf->d = DIM; tempconf->gpu_id = device_id; tempconf->nlist = 100; tempconf->nprobe = 16; tempconf->k = K; - tempconf->metric_type = METRICTYPE::L2; + tempconf->metric_type = kn::METRICTYPE::L2; return tempconf; - } - else if (type == ParameterType::ivfpq) { - auto tempconf = std::make_shared(); + } else if (type == ParameterType::ivfpq) { + auto tempconf = std::make_shared(); tempconf->d = DIM; tempconf->gpu_id = device_id; tempconf->nlist = 100; @@ -101,70 +105,64 @@ class ParamGenerator { tempconf->k = K; tempconf->m = 8; tempconf->nbits = 8; - tempconf->metric_type = METRICTYPE::L2; + tempconf->metric_type = kn::METRICTYPE::L2; return tempconf; - } - else if (type == ParameterType::ivfsq) { - auto tempconf = std::make_shared(); + } else if (type == ParameterType::ivfsq) { + auto tempconf = std::make_shared(); tempconf->d = DIM; tempconf->gpu_id = device_id; tempconf->nlist = 100; tempconf->nprobe = 16; tempconf->k = K; tempconf->nbits = 8; - tempconf->metric_type = METRICTYPE::L2; + tempconf->metric_type = kn::METRICTYPE::L2; return tempconf; } } }; -class IVFTest - : public DataGen, public TestWithParam<::std::tuple> { +class IVFTest : public DataGen, public TestWithParam<::std::tuple> { protected: - void SetUp() override { + void + SetUp() override { ParameterType parameter_type; std::tie(index_type, parameter_type) = GetParam(); - //Init_with_default(); + // Init_with_default(); Generate(DIM, NB, NQ); index_ = IndexFactory(index_type); conf = ParamGenerator::GetInstance().Gen(parameter_type); - FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024*1024*200, 1024*1024*600, 2); + kn::FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024 * 1024 * 200, 1024 * 1024 * 600, 2); } - void TearDown() override { - FaissGpuResourceMgr::GetInstance().Free(); + + void + TearDown() override { + kn::FaissGpuResourceMgr::GetInstance().Free(); } protected: std::string index_type; - Config conf; - IVFIndexPtr index_ = nullptr; + kn::Config conf; + kn::IVFIndexPtr index_ = nullptr; }; - - INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest, - Values( - std::make_tuple("IVF", ParameterType::ivf), - std::make_tuple("GPUIVF", ParameterType::ivf), -// std::make_tuple("IVFPQ", ParameterType::ivfpq), -// std::make_tuple("GPUIVFPQ", ParameterType::ivfpq), - std::make_tuple("IVFSQ", ParameterType::ivfsq), - std::make_tuple("GPUIVFSQ", ParameterType::ivfsq) - ) -); + Values(std::make_tuple("IVF", ParameterType::ivf), + std::make_tuple("GPUIVF", ParameterType::ivf), + // std::make_tuple("IVFPQ", ParameterType::ivfpq), + // std::make_tuple("GPUIVFPQ", ParameterType::ivfpq), + std::make_tuple("IVFSQ", ParameterType::ivfsq), + std::make_tuple("GPUIVFSQ", ParameterType::ivfsq))); -void AssertAnns(const DatasetPtr &result, - const int &nq, - const int &k) { +void +AssertAnns(const kn::DatasetPtr& result, const int& nq, const int& k) { auto ids = result->array()[0]; for (auto i = 0; i < nq; i++) { EXPECT_EQ(i, *(ids->data()->GetValues(1, i * k))); } } -void PrintResult(const DatasetPtr &result, - const int &nq, - const int &k) { +void +PrintResult(const kn::DatasetPtr& result, const int& nq, const int& k) { auto ids = result->array()[0]; auto dists = result->array()[1]; @@ -195,10 +193,10 @@ TEST_P(IVFTest, ivf_basic) { EXPECT_EQ(index_->Dimension(), dim); auto result = index_->Search(query_dataset, conf); AssertAnns(result, nq, conf->k); - //PrintResult(result, nq, k); + // PrintResult(result, nq, k); } -//TEST_P(IVFTest, gpu_to_cpu) { +// TEST_P(IVFTest, gpu_to_cpu) { // if (index_type.find("GPU") == std::string::npos) { return; } // // // else @@ -223,9 +221,9 @@ TEST_P(IVFTest, ivf_basic) { //} TEST_P(IVFTest, ivf_serialize) { - auto serialize = [](const std::string &filename, BinaryPtr &bin, uint8_t *ret) { + auto serialize = [](const std::string& filename, kn::BinaryPtr& bin, uint8_t* ret) { FileIOWriter writer(filename); - writer(static_cast(bin->data.get()), bin->size); + writer(static_cast(bin->data.get()), bin->size); FileIOReader reader(filename); reader(ret, bin->size); @@ -292,15 +290,14 @@ TEST_P(IVFTest, clone_test) { EXPECT_EQ(index_->Dimension(), dim); auto result = index_->Search(query_dataset, conf); AssertAnns(result, nq, conf->k); - //PrintResult(result, nq, k); + // PrintResult(result, nq, k); - auto AssertEqual = [&] (DatasetPtr p1, DatasetPtr p2) { + auto AssertEqual = [&](kn::DatasetPtr p1, kn::DatasetPtr p2) { auto ids_p1 = p1->array()[0]; auto ids_p2 = p2->array()[0]; for (int i = 0; i < nq * k; ++i) { - EXPECT_EQ(*(ids_p2->data()->GetValues(1, i)), - *(ids_p1->data()->GetValues(1, i))); + EXPECT_EQ(*(ids_p2->data()->GetValues(1, i)), *(ids_p1->data()->GetValues(1, i))); } }; @@ -310,17 +307,19 @@ TEST_P(IVFTest, clone_test) { auto finder = std::find(support_idx_vec.cbegin(), support_idx_vec.cend(), index_type); if (finder != support_idx_vec.cend()) { EXPECT_NO_THROW({ - auto clone_index = index_->Clone(); - auto clone_result = clone_index->Search(query_dataset, conf); - //AssertAnns(result, nq, conf->k); - AssertEqual(result, clone_result); - std::cout << "inplace clone [" << index_type << "] success" << std::endl; - }); + auto clone_index = index_->Clone(); + auto clone_result = clone_index->Search(query_dataset, conf); + // AssertAnns(result, nq, conf->k); + AssertEqual(result, clone_result); + std::cout << "inplace clone [" << index_type << "] success" << std::endl; + }); } else { - EXPECT_THROW({ - std::cout << "inplace clone [" << index_type << "] failed" << std::endl; - auto clone_index = index_->Clone(); - }, KnowhereException); + EXPECT_THROW( + { + std::cout << "inplace clone [" << index_type << "] failed" << std::endl; + auto clone_index = index_->Clone(); + }, + kn::KnowhereException); } } @@ -330,16 +329,18 @@ TEST_P(IVFTest, clone_test) { auto finder = std::find(support_idx_vec.cbegin(), support_idx_vec.cend(), index_type); if (finder != support_idx_vec.cend()) { EXPECT_NO_THROW({ - auto clone_index = CopyGpuToCpu(index_, Config()); - auto clone_result = clone_index->Search(query_dataset, conf); - AssertEqual(result, clone_result); - std::cout << "clone G <=> C [" << index_type << "] success" << std::endl; - }); + auto clone_index = kn::cloner::CopyGpuToCpu(index_, kn::Config()); + auto clone_result = clone_index->Search(query_dataset, conf); + AssertEqual(result, clone_result); + std::cout << "clone G <=> C [" << index_type << "] success" << std::endl; + }); } else { - EXPECT_THROW({ - std::cout << "clone G <=> C [" << index_type << "] failed" << std::endl; - auto clone_index = CopyGpuToCpu(index_, Config()); - }, KnowhereException); + EXPECT_THROW( + { + std::cout << "clone G <=> C [" << index_type << "] failed" << std::endl; + auto clone_index = kn::cloner::CopyGpuToCpu(index_, kn::Config()); + }, + kn::KnowhereException); } } @@ -349,22 +350,24 @@ TEST_P(IVFTest, clone_test) { auto finder = std::find(support_idx_vec.cbegin(), support_idx_vec.cend(), index_type); if (finder != support_idx_vec.cend()) { EXPECT_NO_THROW({ - auto clone_index = CopyCpuToGpu(index_, device_id, Config()); - auto clone_result = clone_index->Search(query_dataset, conf); - AssertEqual(result, clone_result); - std::cout << "clone C <=> G [" << index_type << "] success" << std::endl; - }); + auto clone_index = kn::cloner::CopyCpuToGpu(index_, device_id, kn::Config()); + auto clone_result = clone_index->Search(query_dataset, conf); + AssertEqual(result, clone_result); + std::cout << "clone C <=> G [" << index_type << "] success" << std::endl; + }); } else { - EXPECT_THROW({ - std::cout << "clone C <=> G [" << index_type << "] failed" << std::endl; - auto clone_index = CopyCpuToGpu(index_, device_id, Config()); - }, KnowhereException); + EXPECT_THROW( + { + std::cout << "clone C <=> G [" << index_type << "] failed" << std::endl; + auto clone_index = kn::cloner::CopyCpuToGpu(index_, device_id, kn::Config()); + }, + kn::KnowhereException); } } } TEST_P(IVFTest, seal_test) { - //FaissGpuResourceMgr::GetInstance().InitDevice(device_id); + // FaissGpuResourceMgr::GetInstance().InitDevice(device_id); std::vector support_idx_vec{"GPUIVF", "GPUIVFSQ"}; auto finder = std::find(support_idx_vec.cbegin(), support_idx_vec.cend(), index_type); @@ -385,44 +388,44 @@ TEST_P(IVFTest, seal_test) { auto result = index_->Search(query_dataset, conf); AssertAnns(result, nq, conf->k); - auto cpu_idx = CopyGpuToCpu(index_, Config()); + auto cpu_idx = kn::cloner::CopyGpuToCpu(index_, kn::Config()); - TimeRecorder tc("CopyToGpu"); - CopyCpuToGpu(cpu_idx, device_id, Config()); + kn::TimeRecorder tc("CopyToGpu"); + kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config()); auto without_seal = tc.RecordSection("Without seal"); cpu_idx->Seal(); tc.RecordSection("seal cost"); - CopyCpuToGpu(cpu_idx, device_id, Config()); + kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config()); auto with_seal = tc.RecordSection("With seal"); ASSERT_GE(without_seal, with_seal); } - -class GPURESTEST - : public DataGen, public ::testing::Test { +class GPURESTEST : public DataGen, public ::testing::Test { protected: - void SetUp() override { + void + SetUp() override { Generate(128, 1000000, 1000); - FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024*1024*200, 1024*1024*300, 2); + kn::FaissGpuResourceMgr::GetInstance().InitDevice(device_id, 1024 * 1024 * 200, 1024 * 1024 * 300, 2); k = 100; elems = nq * k; - ids = (int64_t *) malloc(sizeof(int64_t) * elems); - dis = (float *) malloc(sizeof(float) * elems); + ids = (int64_t*)malloc(sizeof(int64_t) * elems); + dis = (float*)malloc(sizeof(float) * elems); } - void TearDown() override { + void + TearDown() override { delete ids; delete dis; - FaissGpuResourceMgr::GetInstance().Free(); + kn::FaissGpuResourceMgr::GetInstance().Free(); } protected: std::string index_type; - IVFIndexPtr index_ = nullptr; + kn::IVFIndexPtr index_ = nullptr; - int64_t *ids = nullptr; - float *dis = nullptr; + int64_t* ids = nullptr; + float* dis = nullptr; int64_t elems = 0; }; @@ -433,16 +436,16 @@ TEST_F(GPURESTEST, gpu_ivf_resource_test) { assert(!xb.empty()); { - index_ = std::make_shared(-1); - ASSERT_EQ(std::dynamic_pointer_cast(index_)->GetGpuDevice(), -1); - std::dynamic_pointer_cast(index_)->SetGpuDevice(device_id); - ASSERT_EQ(std::dynamic_pointer_cast(index_)->GetGpuDevice(), device_id); + index_ = std::make_shared(-1); + ASSERT_EQ(std::dynamic_pointer_cast(index_)->GetGpuDevice(), -1); + std::dynamic_pointer_cast(index_)->SetGpuDevice(device_id); + ASSERT_EQ(std::dynamic_pointer_cast(index_)->GetGpuDevice(), device_id); - auto conf = std::make_shared(); + auto conf = std::make_shared(); conf->nlist = 1638; conf->d = dim; conf->gpu_id = device_id; - conf->metric_type = METRICTYPE::L2; + conf->metric_type = kn::METRICTYPE::L2; conf->k = k; conf->nprobe = 1; @@ -454,7 +457,7 @@ TEST_F(GPURESTEST, gpu_ivf_resource_test) { EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dimension(), dim); - TimeRecorder tc("knowere GPUIVF"); + kn::TimeRecorder tc("knowere GPUIVF"); for (int i = 0; i < search_count; ++i) { index_->Search(query_dataset, conf); if (i > search_count - 6 || i < 5) @@ -462,7 +465,7 @@ TEST_F(GPURESTEST, gpu_ivf_resource_test) { } tc.ElapseFromBegin("search all"); } - FaissGpuResourceMgr::GetInstance().Dump(); + kn::FaissGpuResourceMgr::GetInstance().Dump(); { // IVF-Search @@ -473,7 +476,7 @@ TEST_F(GPURESTEST, gpu_ivf_resource_test) { device_index.train(nb, xb.data()); device_index.add(nb, xb.data()); - TimeRecorder tc("ori IVF"); + kn::TimeRecorder tc("ori IVF"); for (int i = 0; i < search_count; ++i) { device_index.search(nq, xq.data(), k, dis, ids); if (i > search_count - 6 || i < 5) @@ -481,7 +484,6 @@ TEST_F(GPURESTEST, gpu_ivf_resource_test) { } tc.ElapseFromBegin("search all"); } - } TEST_F(GPURESTEST, gpuivfsq) { @@ -490,11 +492,11 @@ TEST_F(GPURESTEST, gpuivfsq) { index_type = "GPUIVFSQ"; index_ = IndexFactory(index_type); - auto conf = std::make_shared(); + auto conf = std::make_shared(); conf->nlist = 1638; conf->d = dim; conf->gpu_id = device_id; - conf->metric_type = METRICTYPE::L2; + conf->metric_type = kn::METRICTYPE::L2; conf->k = k; conf->nbits = 8; conf->nprobe = 1; @@ -507,11 +509,11 @@ TEST_F(GPURESTEST, gpuivfsq) { auto result = index_->Search(query_dataset, conf); AssertAnns(result, nq, k); - auto cpu_idx = CopyGpuToCpu(index_, Config()); + auto cpu_idx = kn::cloner::CopyGpuToCpu(index_, kn::Config()); cpu_idx->Seal(); - TimeRecorder tc("knowhere GPUSQ8"); - auto search_idx = CopyCpuToGpu(cpu_idx, device_id, Config()); + kn::TimeRecorder tc("knowhere GPUSQ8"); + auto search_idx = kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config()); tc.RecordSection("Copy to gpu"); for (int i = 0; i < search_count; ++i) { search_idx->Search(query_dataset, conf); @@ -523,8 +525,8 @@ TEST_F(GPURESTEST, gpuivfsq) { { // Ori gpuivfsq Test - const char *index_description = "IVF1638,SQ8"; - faiss::Index *ori_index = faiss::index_factory(dim, index_description, faiss::METRIC_L2); + const char* index_description = "IVF1638,SQ8"; + faiss::Index* ori_index = faiss::index_factory(dim, index_description, faiss::METRIC_L2); faiss::gpu::StandardGpuResources res; auto device_index = faiss::gpu::index_cpu_to_gpu(&res, device_id, ori_index); @@ -532,7 +534,7 @@ TEST_F(GPURESTEST, gpuivfsq) { device_index->add(nb, xb.data()); auto cpu_index = faiss::gpu::index_gpu_to_cpu(device_index); - auto idx = dynamic_cast(cpu_index); + auto idx = dynamic_cast(cpu_index); if (idx != nullptr) { idx->to_readonly(); } @@ -542,8 +544,8 @@ TEST_F(GPURESTEST, gpuivfsq) { faiss::gpu::GpuClonerOptions option; option.allInGpu = true; - TimeRecorder tc("ori GPUSQ8"); - faiss::Index *search_idx = faiss::gpu::index_cpu_to_gpu(&res, device_id, cpu_index, &option); + kn::TimeRecorder tc("ori GPUSQ8"); + faiss::Index* search_idx = faiss::gpu::index_cpu_to_gpu(&res, device_id, cpu_index, &option); tc.RecordSection("Copy to gpu"); for (int i = 0; i < search_count; ++i) { search_idx->search(nq, xq.data(), k, dis, ids); @@ -554,7 +556,6 @@ TEST_F(GPURESTEST, gpuivfsq) { delete cpu_index; delete search_idx; } - } TEST_F(GPURESTEST, copyandsearch) { @@ -564,11 +565,11 @@ TEST_F(GPURESTEST, copyandsearch) { index_type = "GPUIVFSQ"; index_ = IndexFactory(index_type); - auto conf = std::make_shared(); + auto conf = std::make_shared(); conf->nlist = 1638; conf->d = dim; conf->gpu_id = device_id; - conf->metric_type = METRICTYPE::L2; + conf->metric_type = kn::METRICTYPE::L2; conf->k = k; conf->nbits = 8; conf->nprobe = 1; @@ -581,32 +582,32 @@ TEST_F(GPURESTEST, copyandsearch) { auto result = index_->Search(query_dataset, conf); AssertAnns(result, nq, k); - auto cpu_idx = CopyGpuToCpu(index_, Config()); + auto cpu_idx = kn::cloner::CopyGpuToCpu(index_, kn::Config()); cpu_idx->Seal(); - auto search_idx = CopyCpuToGpu(cpu_idx, device_id, Config()); + auto search_idx = kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config()); auto search_func = [&] { - //TimeRecorder tc("search&load"); + // TimeRecorder tc("search&load"); for (int i = 0; i < search_count; ++i) { search_idx->Search(query_dataset, conf); - //if (i > search_count - 6 || i == 0) + // if (i > search_count - 6 || i == 0) // tc.RecordSection("search once"); } - //tc.ElapseFromBegin("search finish"); + // tc.ElapseFromBegin("search finish"); }; auto load_func = [&] { - //TimeRecorder tc("search&load"); + // TimeRecorder tc("search&load"); for (int i = 0; i < load_count; ++i) { - CopyCpuToGpu(cpu_idx, device_id, Config()); - //if (i > load_count -5 || i < 5) - //tc.RecordSection("Copy to gpu"); + kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config()); + // if (i > load_count -5 || i < 5) + // tc.RecordSection("Copy to gpu"); } - //tc.ElapseFromBegin("load finish"); + // tc.ElapseFromBegin("load finish"); }; - TimeRecorder tc("basic"); - CopyCpuToGpu(cpu_idx, device_id, Config()); + kn::TimeRecorder tc("basic"); + kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config()); tc.RecordSection("Copy to gpu once"); search_idx->Search(query_dataset, conf); tc.RecordSection("search once"); @@ -626,11 +627,11 @@ TEST_F(GPURESTEST, TrainAndSearch) { index_type = "GPUIVFSQ"; index_ = IndexFactory(index_type); - auto conf = std::make_shared(); + auto conf = std::make_shared(); conf->nlist = 1638; conf->d = dim; conf->gpu_id = device_id; - conf->metric_type = METRICTYPE::L2; + conf->metric_type = kn::METRICTYPE::L2; conf->k = k; conf->nbits = 8; conf->nprobe = 1; @@ -641,9 +642,9 @@ TEST_F(GPURESTEST, TrainAndSearch) { auto new_index = IndexFactory(index_type); new_index->set_index_model(model); new_index->Add(base_dataset, conf); - auto cpu_idx = CopyGpuToCpu(new_index, Config()); + auto cpu_idx = kn::cloner::CopyGpuToCpu(new_index, kn::Config()); cpu_idx->Seal(); - auto search_idx = CopyCpuToGpu(cpu_idx, device_id, Config()); + auto search_idx = kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config()); constexpr int train_count = 1; constexpr int search_count = 5000; @@ -655,18 +656,18 @@ TEST_F(GPURESTEST, TrainAndSearch) { test_idx->Add(base_dataset, conf); } }; - auto search_stage = [&](VectorIndexPtr& search_idx) { + auto search_stage = [&](kn::VectorIndexPtr& search_idx) { for (int i = 0; i < search_count; ++i) { auto result = search_idx->Search(query_dataset, conf); AssertAnns(result, nq, k); } }; - //TimeRecorder tc("record"); - //train_stage(); - //tc.RecordSection("train cost"); - //search_stage(search_idx); - //tc.RecordSection("search cost"); + // TimeRecorder tc("record"); + // train_stage(); + // tc.RecordSection("train cost"); + // search_stage(search_idx); + // tc.RecordSection("search cost"); { // search and build parallel @@ -684,7 +685,7 @@ TEST_F(GPURESTEST, TrainAndSearch) { } { // search parallel - auto search_idx_2 = CopyCpuToGpu(cpu_idx, device_id, Config()); + auto search_idx_2 = kn::cloner::CopyCpuToGpu(cpu_idx, device_id, kn::Config()); std::thread search_1(search_stage, std::ref(search_idx)); std::thread search_2(search_stage, std::ref(search_idx_2)); search_1.join(); @@ -692,6 +693,4 @@ TEST_F(GPURESTEST, TrainAndSearch) { } } - - // TODO(lxj): Add exception test diff --git a/cpp/src/core/test/test_json.cpp b/cpp/src/core/test/test_json.cpp index b3c4fd4993..acdddad30a 100644 --- a/cpp/src/core/test/test_json.cpp +++ b/cpp/src/core/test/test_json.cpp @@ -15,13 +15,17 @@ // specific language governing permissions and limitations // under the License. - #include "knowhere/common/config.h" -using namespace zilliz::knowhere; +namespace { -int main(){ - Config cfg; +namespace kn = zilliz::knowhere; + +} // namespace + +int +main() { + kn::Config cfg; cfg["size"] = size_t(199); auto size = cfg.get_with_default("size", 123); diff --git a/cpp/src/core/test/test_kdt.cpp b/cpp/src/core/test/test_kdt.cpp index 3780f655cc..45809bf396 100644 --- a/cpp/src/core/test/test_kdt.cpp +++ b/cpp/src/core/test/test_kdt.cpp @@ -15,35 +15,36 @@ // specific language governing permissions and limitations // under the License. - #include #include #include +#include "knowhere/adapter/SptagAdapter.h" +#include "knowhere/adapter/Structure.h" #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/IndexKDT.h" #include "knowhere/index/vector_index/helpers/Definitions.h" -#include "knowhere/adapter/SptagAdapter.h" -#include "knowhere/adapter/Structure.h" -#include "utils.h" +#include "test/utils.h" +namespace { -using namespace zilliz::knowhere; +namespace kn = zilliz::knowhere; + +} // namespace using ::testing::TestWithParam; using ::testing::Values; using ::testing::Combine; - -class KDTTest - : public DataGen, public ::testing::Test { +class KDTTest : public DataGen, public ::testing::Test { protected: - void SetUp() override { - index_ = std::make_shared(); + void + SetUp() override { + index_ = std::make_shared(); - auto tempconf = std::make_shared(); + auto tempconf = std::make_shared(); tempconf->tptnubmber = 1; tempconf->k = 10; conf = tempconf; @@ -52,22 +53,20 @@ class KDTTest } protected: - Config conf; - std::shared_ptr index_ = nullptr; + kn::Config conf; + std::shared_ptr index_ = nullptr; }; -void AssertAnns(const DatasetPtr &result, - const int &nq, - const int &k) { +void +AssertAnns(const kn::DatasetPtr& result, const int& nq, const int& k) { auto ids = result->array()[0]; for (auto i = 0; i < nq; i++) { EXPECT_EQ(i, *(ids->data()->GetValues(1, i * k))); } } -void PrintResult(const DatasetPtr &result, - const int &nq, - const int &k) { +void +PrintResult(const kn::DatasetPtr& result, const int& nq, const int& k) { auto ids = result->array()[0]; auto dists = result->array()[1]; @@ -124,25 +123,25 @@ TEST_F(KDTTest, kdt_serialize) { index_->set_preprocessor(preprocessor); auto model = index_->Train(base_dataset, conf); - //index_->Add(base_dataset, conf); + // index_->Add(base_dataset, conf); auto binaryset = index_->Serialize(); - auto new_index = std::make_shared(); + auto new_index = std::make_shared(); new_index->Load(binaryset); auto result = new_index->Search(query_dataset, conf); AssertAnns(result, nq, k); PrintResult(result, nq, k); ASSERT_EQ(new_index->Count(), nb); ASSERT_EQ(new_index->Dimension(), dim); - ASSERT_THROW({new_index->Clone();}, zilliz::knowhere::KnowhereException); - ASSERT_NO_THROW({new_index->Seal();}); + ASSERT_THROW({ new_index->Clone(); }, zilliz::knowhere::KnowhereException); + ASSERT_NO_THROW({ new_index->Seal(); }); { int fileno = 0; - const std::string &base_name = "/tmp/kdt_serialize_test_bin_"; + const std::string& base_name = "/tmp/kdt_serialize_test_bin_"; std::vector filename_list; - std::vector> meta_list; - for (auto &iter: binaryset.binary_map_) { - const std::string &filename = base_name + std::to_string(fileno); + std::vector> meta_list; + for (auto& iter : binaryset.binary_map_) { + const std::string& filename = base_name + std::to_string(fileno); FileIOWriter writer(filename); writer(iter.second->data.get(), iter.second->size); @@ -151,7 +150,7 @@ TEST_F(KDTTest, kdt_serialize) { ++fileno; } - BinarySet load_data_list; + kn::BinarySet load_data_list; for (int i = 0; i < filename_list.size() && i < meta_list.size(); ++i) { auto bin_size = meta_list[i].second; FileIOReader reader(filename_list[i]); @@ -163,7 +162,7 @@ TEST_F(KDTTest, kdt_serialize) { load_data_list.Append(meta_list[i].first, data, bin_size); } - auto new_index = std::make_shared(); + auto new_index = std::make_shared(); new_index->Load(load_data_list); auto result = new_index->Search(query_dataset, conf); AssertAnns(result, nq, k); diff --git a/cpp/src/core/test/test_nsg.cpp b/cpp/src/core/test/test_nsg.cpp index f156925027..a4b6006e71 100644 --- a/cpp/src/core/test/test_nsg.cpp +++ b/cpp/src/core/test/test_nsg.cpp @@ -15,46 +15,46 @@ // specific language governing permissions and limitations // under the License. - #include #include -#include -#include "index.h" +#include "knowhere/index/index.h" +#include "test/utils.h" //#include -using namespace zilliz::knowhere; - -void load_data(std::string &filename, float *&data, unsigned &num, - unsigned &dim) { // load data with sift10K pattern +void +load_data(std::string& filename, float*& data, unsigned& num, + unsigned& dim) { // load data with sift10K pattern std::ifstream in(filename, std::ios::binary); if (!in.is_open()) { std::cout << "open file error" << std::endl; exit(-1); } - in.read((char *) &dim, 4); + in.read((char*)&dim, 4); in.seekg(0, std::ios::end); std::ios::pos_type ss = in.tellg(); - size_t fsize = (size_t) ss; - num = (unsigned) (fsize / (dim + 1) / 4); - data = new float[(size_t) num * (size_t) dim]; + size_t fsize = (size_t)ss; + num = (unsigned)(fsize / (dim + 1) / 4); + data = new float[(size_t)num * (size_t)dim]; in.seekg(0, std::ios::beg); for (size_t i = 0; i < num; i++) { in.seekg(4, std::ios::cur); - in.read((char *) (data + i * dim), dim * 4); + in.read((char*)(data + i * dim), dim * 4); } in.close(); } -void test_distance() { +void +test_distance() { std::vector xb{1, 2, 3, 4}; std::vector xq{2, 2, 3, 4}; float r = calculate(xb.data(), xq.data(), 4); std::cout << r << std::endl; } -int main() { +int +main() { test_distance(); BuildParams params; @@ -62,16 +62,16 @@ int main() { params.candidate_pool_size = 100; params.out_degree = 50; - float *data = nullptr; - long *ids = nullptr; + float* data = nullptr; + int64_t* ids = nullptr; unsigned ntotal, dim; std::string filename = "/home/zilliz/opt/workspace/wook/efanna_graph/tests/siftsmall/siftsmall_base.fvecs"; - //std::string filename = "/home/zilliz/opt/workspace/wook/efanna_graph/tests/sift/sift_base.fvecs"; + // std::string filename = "/home/zilliz/opt/workspace/wook/efanna_graph/tests/sift/sift_base.fvecs"; load_data(filename, data, ntotal, dim); assert(data); - //float x = calculate(data + dim * 0, data + dim * 62, dim); - //std::cout << x << std::endl; + // float x = calculate(data + dim * 0, data + dim * 62, dim); + // std::cout << x << std::endl; NsgIndex index(dim, ntotal); @@ -81,24 +81,23 @@ int main() { std::chrono::duration diff = e - s; std::cout << "indexing time: " << diff.count() << "\n"; - int k = 10; int nq = 1000; SearchParams s_params; s_params.search_length = 50; - auto dist = new float[nq*k]; - auto ids_b = new long[nq*k]; + auto dist = new float[nq * k]; + auto ids_b = new int64_t[nq * k]; s = std::chrono::high_resolution_clock::now(); - //ProfilerStart("xx.prof"); + // ProfilerStart("xx.prof"); index.Search(data, nq, dim, k, dist, ids_b, s_params); - //ProfilerStop(); + // ProfilerStop(); e = std::chrono::high_resolution_clock::now(); diff = e - s; std::cout << "search time: " << diff.count() << "\n"; for (int i = 0; i < k; ++i) { std::cout << "id " << ids_b[i] << std::endl; - //std::cout << "dist " << dist[i] << std::endl; + // std::cout << "dist " << dist[i] << std::endl; } delete[] dist; @@ -106,5 +105,3 @@ int main() { return 0; } - - diff --git a/cpp/src/core/test/test_nsg/test_nsg.cpp b/cpp/src/core/test/test_nsg/test_nsg.cpp index 8f0187e0a3..6f7ca1eba9 100644 --- a/cpp/src/core/test/test_nsg/test_nsg.cpp +++ b/cpp/src/core/test/test_nsg/test_nsg.cpp @@ -15,35 +15,39 @@ // specific language governing permissions and limitations // under the License. - #include #include #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/FaissBaseIndex.h" #include "knowhere/index/vector_index/IndexNSG.h" -#include "knowhere/index/vector_index/nsg/NSGIO.h" #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" +#include "knowhere/index/vector_index/nsg/NSGIO.h" -#include "../utils.h" +#include "test/utils.h" +namespace { + +namespace kn = zilliz::knowhere; + +} // namespace -using namespace zilliz::knowhere; using ::testing::TestWithParam; using ::testing::Values; using ::testing::Combine; constexpr int64_t DEVICE_ID = 1; - class NSGInterfaceTest : public DataGen, public ::testing::Test { +class NSGInterfaceTest : public DataGen, public ::testing::Test { protected: - void SetUp() override { - //Init_with_default(); - FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID, 1024*1024*200, 1024*1024*600, 2); + void + SetUp() override { + // Init_with_default(); + kn::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID, 1024 * 1024 * 200, 1024 * 1024 * 600, 2); Generate(256, 1000000, 1); - index_ = std::make_shared(); + index_ = std::make_shared(); - auto tmp_conf = std::make_shared(); + auto tmp_conf = std::make_shared(); tmp_conf->gpu_id = DEVICE_ID; tmp_conf->knng = 100; tmp_conf->nprobe = 32; @@ -51,28 +55,28 @@ constexpr int64_t DEVICE_ID = 1; tmp_conf->search_length = 60; tmp_conf->out_degree = 70; tmp_conf->candidate_pool_size = 500; - tmp_conf->metric_type = METRICTYPE::L2; + tmp_conf->metric_type = kn::METRICTYPE::L2; train_conf = tmp_conf; - auto tmp2_conf = std::make_shared(); + auto tmp2_conf = std::make_shared(); tmp2_conf->k = k; tmp2_conf->search_length = 30; search_conf = tmp2_conf; } - void TearDown() override { - FaissGpuResourceMgr::GetInstance().Free(); + void + TearDown() override { + kn::FaissGpuResourceMgr::GetInstance().Free(); } protected: - std::shared_ptr index_; - Config train_conf; - Config search_conf; + std::shared_ptr index_; + kn::Config train_conf; + kn::Config search_conf; }; -void AssertAnns(const DatasetPtr &result, - const int &nq, - const int &k) { +void +AssertAnns(const kn::DatasetPtr& result, const int& nq, const int& k) { auto ids = result->array()[0]; for (auto i = 0; i < nq; i++) { EXPECT_EQ(i, *(ids->data()->GetValues(1, i * k))); @@ -87,33 +91,32 @@ TEST_F(NSGInterfaceTest, basic_test) { AssertAnns(result, nq, k); auto binaryset = index_->Serialize(); - auto new_index = std::make_shared(); + auto new_index = std::make_shared(); new_index->Load(binaryset); auto new_result = new_index->Search(query_dataset, search_conf); AssertAnns(result, nq, k); ASSERT_EQ(index_->Count(), nb); ASSERT_EQ(index_->Dimension(), dim); - ASSERT_THROW({index_->Clone();}, zilliz::knowhere::KnowhereException); + ASSERT_THROW({ index_->Clone(); }, zilliz::knowhere::KnowhereException); ASSERT_NO_THROW({ - index_->Add(base_dataset, Config()); + index_->Add(base_dataset, kn::Config()); index_->Seal(); }); { - //std::cout << "k = 1" << std::endl; - //new_index->Search(GenQuery(1), Config::object{{"k", 1}}); - //new_index->Search(GenQuery(10), Config::object{{"k", 1}}); - //new_index->Search(GenQuery(100), Config::object{{"k", 1}}); - //new_index->Search(GenQuery(1000), Config::object{{"k", 1}}); - //new_index->Search(GenQuery(10000), Config::object{{"k", 1}}); + // std::cout << "k = 1" << std::endl; + // new_index->Search(GenQuery(1), Config::object{{"k", 1}}); + // new_index->Search(GenQuery(10), Config::object{{"k", 1}}); + // new_index->Search(GenQuery(100), Config::object{{"k", 1}}); + // new_index->Search(GenQuery(1000), Config::object{{"k", 1}}); + // new_index->Search(GenQuery(10000), Config::object{{"k", 1}}); - //std::cout << "k = 5" << std::endl; - //new_index->Search(GenQuery(1), Config::object{{"k", 5}}); - //new_index->Search(GenQuery(20), Config::object{{"k", 5}}); - //new_index->Search(GenQuery(100), Config::object{{"k", 5}}); - //new_index->Search(GenQuery(300), Config::object{{"k", 5}}); - //new_index->Search(GenQuery(500), Config::object{{"k", 5}}); + // std::cout << "k = 5" << std::endl; + // new_index->Search(GenQuery(1), Config::object{{"k", 5}}); + // new_index->Search(GenQuery(20), Config::object{{"k", 5}}); + // new_index->Search(GenQuery(100), Config::object{{"k", 5}}); + // new_index->Search(GenQuery(300), Config::object{{"k", 5}}); + // new_index->Search(GenQuery(500), Config::object{{"k", 5}}); } } - diff --git a/cpp/src/core/test/utils.cpp b/cpp/src/core/test/utils.cpp index df9410bd6c..a921b69802 100644 --- a/cpp/src/core/test/utils.cpp +++ b/cpp/src/core/test/utils.cpp @@ -15,24 +15,35 @@ // specific language governing permissions and limitations // under the License. +#include "test/utils.h" -#include "utils.h" +#include +#include +#include INITIALIZE_EASYLOGGINGPP -void InitLog() { +namespace { + +namespace kn = zilliz::knowhere; + +} // namespace + +void +InitLog() { el::Configurations defaultConf; defaultConf.setToDefault(); - defaultConf.set(el::Level::Debug, - el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)"); + defaultConf.set(el::Level::Debug, el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)"); el::Loggers::reconfigureLogger("default", defaultConf); } -void DataGen::Init_with_default() { +void +DataGen::Init_with_default() { Generate(dim, nb, nq); } -void DataGen::Generate(const int &dim, const int &nb, const int &nq) { +void +DataGen::Generate(const int& dim, const int& nb, const int& nq) { this->nb = nb; this->nq = nq; this->dim = dim; @@ -43,9 +54,10 @@ void DataGen::Generate(const int &dim, const int &nb, const int &nq) { base_dataset = generate_dataset(nb, dim, xb.data(), ids.data()); query_dataset = generate_query_dataset(nq, dim, xq.data()); - } -zilliz::knowhere::DatasetPtr DataGen::GenQuery(const int &nq) { + +zilliz::knowhere::DatasetPtr +DataGen::GenQuery(const int& nq) { xq.resize(nq * dim); for (int i = 0; i < nq * dim; ++i) { xq[i] = xb[i]; @@ -53,37 +65,28 @@ zilliz::knowhere::DatasetPtr DataGen::GenQuery(const int &nq) { return generate_query_dataset(nq, dim, xq.data()); } -void GenAll(const int64_t dim, - const int64_t &nb, - std::vector &xb, - std::vector &ids, - const int64_t &nq, - std::vector &xq) { +void +GenAll(const int64_t dim, const int64_t& nb, std::vector& xb, std::vector& ids, const int64_t& nq, + std::vector& xq) { xb.resize(nb * dim); xq.resize(nq * dim); ids.resize(nb); GenAll(dim, nb, xb.data(), ids.data(), nq, xq.data()); } -void GenAll(const int64_t &dim, - const int64_t &nb, - float *xb, - int64_t *ids, - const int64_t &nq, - float *xq) { +void +GenAll(const int64_t& dim, const int64_t& nb, float* xb, int64_t* ids, const int64_t& nq, float* xq) { GenBase(dim, nb, xb, ids); for (int64_t i = 0; i < nq * dim; ++i) { xq[i] = xb[i]; } } -void GenBase(const int64_t &dim, - const int64_t &nb, - float *xb, - int64_t *ids) { +void +GenBase(const int64_t& dim, const int64_t& nb, float* xb, int64_t* ids) { for (auto i = 0; i < nb; ++i) { for (auto j = 0; j < dim; ++j) { - //p_data[i * d + j] = float(base + i); + // p_data[i * d + j] = float(base + i); xb[i * dim + j] = drand48(); } xb[dim * i] += i / 1000.; @@ -91,7 +94,7 @@ void GenBase(const int64_t &dim, } } -FileIOReader::FileIOReader(const std::string &fname) { +FileIOReader::FileIOReader(const std::string& fname) { name = fname; fs = std::fstream(name, std::ios::in | std::ios::binary); } @@ -100,12 +103,13 @@ FileIOReader::~FileIOReader() { fs.close(); } -size_t FileIOReader::operator()(void *ptr, size_t size) { - fs.read(reinterpret_cast(ptr), size); +size_t +FileIOReader::operator()(void* ptr, size_t size) { + fs.read(reinterpret_cast(ptr), size); return size; } -FileIOWriter::FileIOWriter(const std::string &fname) { +FileIOWriter::FileIOWriter(const std::string& fname) { name = fname; fs = std::fstream(name, std::ios::out | std::ios::binary); } @@ -114,39 +118,37 @@ FileIOWriter::~FileIOWriter() { fs.close(); } -size_t FileIOWriter::operator()(void *ptr, size_t size) { - fs.write(reinterpret_cast(ptr), size); +size_t +FileIOWriter::operator()(void* ptr, size_t size) { + fs.write(reinterpret_cast(ptr), size); return size; } -using namespace zilliz::knowhere; - -DatasetPtr -generate_dataset(int64_t nb, int64_t dim, float *xb, long *ids) { +kn::DatasetPtr +generate_dataset(int64_t nb, int64_t dim, float* xb, int64_t* ids) { std::vector shape{nb, dim}; - auto tensor = ConstructFloatTensor((uint8_t *) xb, nb * dim * sizeof(float), shape); - std::vector tensors{tensor}; - std::vector tensor_fields{ConstructFloatField("data")}; - auto tensor_schema = std::make_shared(tensor_fields); + auto tensor = kn::ConstructFloatTensor((uint8_t*)xb, nb * dim * sizeof(float), shape); + std::vector tensors{tensor}; + std::vector tensor_fields{kn::ConstructFloatField("data")}; + auto tensor_schema = std::make_shared(tensor_fields); - auto id_array = ConstructInt64Array((uint8_t *) ids, nb * sizeof(int64_t)); - std::vector arrays{id_array}; - std::vector array_fields{ConstructInt64Field("id")}; - auto array_schema = std::make_shared(tensor_fields); + auto id_array = kn::ConstructInt64Array((uint8_t*)ids, nb * sizeof(int64_t)); + std::vector arrays{id_array}; + std::vector array_fields{kn::ConstructInt64Field("id")}; + auto array_schema = std::make_shared(tensor_fields); - auto dataset = std::make_shared(std::move(arrays), array_schema, - std::move(tensors), tensor_schema); + auto dataset = std::make_shared(std::move(arrays), array_schema, std::move(tensors), tensor_schema); return dataset; } -DatasetPtr -generate_query_dataset(int64_t nb, int64_t dim, float *xb) { +kn::DatasetPtr +generate_query_dataset(int64_t nb, int64_t dim, float* xb) { std::vector shape{nb, dim}; - auto tensor = ConstructFloatTensor((uint8_t *) xb, nb * dim * sizeof(float), shape); - std::vector tensors{tensor}; - std::vector tensor_fields{ConstructFloatField("data")}; - auto tensor_schema = std::make_shared(tensor_fields); + auto tensor = kn::ConstructFloatTensor((uint8_t*)xb, nb * dim * sizeof(float), shape); + std::vector tensors{tensor}; + std::vector tensor_fields{kn::ConstructFloatField("data")}; + auto tensor_schema = std::make_shared(tensor_fields); - auto dataset = std::make_shared(std::move(tensors), tensor_schema); + auto dataset = std::make_shared(std::move(tensors), tensor_schema); return dataset; } diff --git a/cpp/src/core/test/utils.h b/cpp/src/core/test/utils.h index 41dbbc1f30..07a104ad4a 100644 --- a/cpp/src/core/test/utils.h +++ b/cpp/src/core/test/utils.h @@ -15,24 +15,27 @@ // specific language governing permissions and limitations // under the License. - #pragma once -#include -#include #include +#include #include +#include +#include #include "knowhere/adapter/Structure.h" #include "knowhere/common/Log.h" -class DataGen { +class DataGen { protected: - void Init_with_default(); + void + Init_with_default(); - void Generate(const int &dim, const int &nb, const int &nq); + void + Generate(const int& dim, const int& nb, const int& nq); - zilliz::knowhere::DatasetPtr GenQuery(const int&nq); + zilliz::knowhere::DatasetPtr + GenQuery(const int& nq); protected: int nb = 10000; @@ -46,49 +49,41 @@ class DataGen { zilliz::knowhere::DatasetPtr query_dataset = nullptr; }; +extern void +GenAll(const int64_t dim, const int64_t& nb, std::vector& xb, std::vector& ids, const int64_t& nq, + std::vector& xq); -extern void GenAll(const int64_t dim, - const int64_t &nb, - std::vector &xb, - std::vector &ids, - const int64_t &nq, - std::vector &xq); +extern void +GenAll(const int64_t& dim, const int64_t& nb, float* xb, int64_t* ids, const int64_t& nq, float* xq); -extern void GenAll(const int64_t &dim, - const int64_t &nb, - float *xb, - int64_t *ids, - const int64_t &nq, - float *xq); +extern void +GenBase(const int64_t& dim, const int64_t& nb, float* xb, int64_t* ids); -extern void GenBase(const int64_t &dim, - const int64_t &nb, - float *xb, - int64_t *ids); - -extern void InitLog(); +extern void +InitLog(); zilliz::knowhere::DatasetPtr -generate_dataset(int64_t nb, int64_t dim, float *xb, long *ids); +generate_dataset(int64_t nb, int64_t dim, float* xb, int64_t* ids); zilliz::knowhere::DatasetPtr -generate_query_dataset(int64_t nb, int64_t dim, float *xb); +generate_query_dataset(int64_t nb, int64_t dim, float* xb); struct FileIOWriter { std::fstream fs; std::string name; - FileIOWriter(const std::string &fname); + explicit FileIOWriter(const std::string& fname); ~FileIOWriter(); - size_t operator()(void *ptr, size_t size); + size_t + operator()(void* ptr, size_t size); }; struct FileIOReader { std::fstream fs; std::string name; - FileIOReader(const std::string &fname); + explicit FileIOReader(const std::string& fname); ~FileIOReader(); - size_t operator()(void *ptr, size_t size); + size_t + operator()(void* ptr, size_t size); }; - From c2ab2bc0925805e0b6fa03ee952b42806803cb77 Mon Sep 17 00:00:00 2001 From: "xiaojun.lin" Date: Sun, 29 Sep 2019 10:13:57 +0800 Subject: [PATCH 146/217] MS-565 rename test->unittest and fix parameter autotune bug Former-commit-id: 65392a505107e20491035d417940a66e67f82ef9 --- cpp/src/core/CMakeLists.txt | 2 +- cpp/src/core/{test => unittest}/CMakeLists.txt | 2 +- cpp/src/core/{test => unittest}/SPTAG.cpp | 0 .../{test => unittest}/faiss_ori/CMakeLists.txt | 0 .../faiss_ori/gpuresource_test.cpp | 0 cpp/src/core/{test => unittest}/kdtree.cpp | 0 cpp/src/core/{test => unittest}/sift.50NN.graph | Bin .../core/{test => unittest}/siftsmall_base.fvecs | Bin cpp/src/core/{test => unittest}/test_idmap.cpp | 2 +- cpp/src/core/{test => unittest}/test_ivf.cpp | 2 +- cpp/src/core/{test => unittest}/test_json.cpp | 0 cpp/src/core/{test => unittest}/test_kdt.cpp | 2 +- cpp/src/core/{test => unittest}/test_nsg.cpp | 0 .../core/{test => unittest}/test_nsg/CMakeLists.txt | 0 .../core/{test => unittest}/test_nsg/test_nsg.cpp | 2 +- cpp/src/core/{test => unittest}/utils.cpp | 2 +- cpp/src/core/{test => unittest}/utils.h | 0 cpp/src/wrapper/ConfAdapter.cpp | 2 +- 18 files changed, 8 insertions(+), 8 deletions(-) rename cpp/src/core/{test => unittest}/CMakeLists.txt (98%) rename cpp/src/core/{test => unittest}/SPTAG.cpp (100%) rename cpp/src/core/{test => unittest}/faiss_ori/CMakeLists.txt (100%) rename cpp/src/core/{test => unittest}/faiss_ori/gpuresource_test.cpp (100%) rename cpp/src/core/{test => unittest}/kdtree.cpp (100%) rename cpp/src/core/{test => unittest}/sift.50NN.graph (100%) rename cpp/src/core/{test => unittest}/siftsmall_base.fvecs (100%) rename cpp/src/core/{test => unittest}/test_idmap.cpp (99%) rename cpp/src/core/{test => unittest}/test_ivf.cpp (99%) rename cpp/src/core/{test => unittest}/test_json.cpp (100%) rename cpp/src/core/{test => unittest}/test_kdt.cpp (99%) rename cpp/src/core/{test => unittest}/test_nsg.cpp (100%) rename cpp/src/core/{test => unittest}/test_nsg/CMakeLists.txt (100%) rename cpp/src/core/{test => unittest}/test_nsg/test_nsg.cpp (99%) rename cpp/src/core/{test => unittest}/utils.cpp (99%) rename cpp/src/core/{test => unittest}/utils.h (100%) diff --git a/cpp/src/core/CMakeLists.txt b/cpp/src/core/CMakeLists.txt index f1a13b4520..9125f6ea2c 100644 --- a/cpp/src/core/CMakeLists.txt +++ b/cpp/src/core/CMakeLists.txt @@ -95,7 +95,7 @@ endif() set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) if(BUILD_UNIT_TEST STREQUAL "ON") - add_subdirectory(test) + add_subdirectory(unittest) endif() config_summary() diff --git a/cpp/src/core/test/CMakeLists.txt b/cpp/src/core/unittest/CMakeLists.txt similarity index 98% rename from cpp/src/core/test/CMakeLists.txt rename to cpp/src/core/unittest/CMakeLists.txt index ecb52695bc..6b5775c2cd 100644 --- a/cpp/src/core/test/CMakeLists.txt +++ b/cpp/src/core/unittest/CMakeLists.txt @@ -30,7 +30,7 @@ set(util_srcs ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/ArrowAdapter.cpp ${CORE_SOURCE_DIR}/knowhere/knowhere/common/Exception.cpp ${CORE_SOURCE_DIR}/knowhere/knowhere/common/Timer.cpp - ${CORE_SOURCE_DIR}/test/utils.cpp + ${CORE_SOURCE_DIR}/unittest/utils.cpp ) # diff --git a/cpp/src/core/test/SPTAG.cpp b/cpp/src/core/unittest/SPTAG.cpp similarity index 100% rename from cpp/src/core/test/SPTAG.cpp rename to cpp/src/core/unittest/SPTAG.cpp diff --git a/cpp/src/core/test/faiss_ori/CMakeLists.txt b/cpp/src/core/unittest/faiss_ori/CMakeLists.txt similarity index 100% rename from cpp/src/core/test/faiss_ori/CMakeLists.txt rename to cpp/src/core/unittest/faiss_ori/CMakeLists.txt diff --git a/cpp/src/core/test/faiss_ori/gpuresource_test.cpp b/cpp/src/core/unittest/faiss_ori/gpuresource_test.cpp similarity index 100% rename from cpp/src/core/test/faiss_ori/gpuresource_test.cpp rename to cpp/src/core/unittest/faiss_ori/gpuresource_test.cpp diff --git a/cpp/src/core/test/kdtree.cpp b/cpp/src/core/unittest/kdtree.cpp similarity index 100% rename from cpp/src/core/test/kdtree.cpp rename to cpp/src/core/unittest/kdtree.cpp diff --git a/cpp/src/core/test/sift.50NN.graph b/cpp/src/core/unittest/sift.50NN.graph similarity index 100% rename from cpp/src/core/test/sift.50NN.graph rename to cpp/src/core/unittest/sift.50NN.graph diff --git a/cpp/src/core/test/siftsmall_base.fvecs b/cpp/src/core/unittest/siftsmall_base.fvecs similarity index 100% rename from cpp/src/core/test/siftsmall_base.fvecs rename to cpp/src/core/unittest/siftsmall_base.fvecs diff --git a/cpp/src/core/test/test_idmap.cpp b/cpp/src/core/unittest/test_idmap.cpp similarity index 99% rename from cpp/src/core/test/test_idmap.cpp rename to cpp/src/core/unittest/test_idmap.cpp index 8bf0d5c8d2..9ee4ea732d 100644 --- a/cpp/src/core/test/test_idmap.cpp +++ b/cpp/src/core/unittest/test_idmap.cpp @@ -23,7 +23,7 @@ #include "knowhere/index/vector_index/IndexIDMAP.h" #include "knowhere/index/vector_index/helpers/Cloner.h" -#include "test/utils.h" +#include "unittest/utils.h" namespace { diff --git a/cpp/src/core/test/test_ivf.cpp b/cpp/src/core/unittest/test_ivf.cpp similarity index 99% rename from cpp/src/core/test/test_ivf.cpp rename to cpp/src/core/unittest/test_ivf.cpp index 9835264bff..2aa6937dbe 100644 --- a/cpp/src/core/test/test_ivf.cpp +++ b/cpp/src/core/unittest/test_ivf.cpp @@ -35,7 +35,7 @@ #include "knowhere/index/vector_index/IndexIVFSQ.h" #include "knowhere/index/vector_index/helpers/Cloner.h" -#include "test/utils.h" +#include "unittest/utils.h" namespace { diff --git a/cpp/src/core/test/test_json.cpp b/cpp/src/core/unittest/test_json.cpp similarity index 100% rename from cpp/src/core/test/test_json.cpp rename to cpp/src/core/unittest/test_json.cpp diff --git a/cpp/src/core/test/test_kdt.cpp b/cpp/src/core/unittest/test_kdt.cpp similarity index 99% rename from cpp/src/core/test/test_kdt.cpp rename to cpp/src/core/unittest/test_kdt.cpp index 45809bf396..85c8650d1b 100644 --- a/cpp/src/core/test/test_kdt.cpp +++ b/cpp/src/core/unittest/test_kdt.cpp @@ -26,7 +26,7 @@ #include "knowhere/index/vector_index/IndexKDT.h" #include "knowhere/index/vector_index/helpers/Definitions.h" -#include "test/utils.h" +#include "unittest/utils.h" namespace { diff --git a/cpp/src/core/test/test_nsg.cpp b/cpp/src/core/unittest/test_nsg.cpp similarity index 100% rename from cpp/src/core/test/test_nsg.cpp rename to cpp/src/core/unittest/test_nsg.cpp diff --git a/cpp/src/core/test/test_nsg/CMakeLists.txt b/cpp/src/core/unittest/test_nsg/CMakeLists.txt similarity index 100% rename from cpp/src/core/test/test_nsg/CMakeLists.txt rename to cpp/src/core/unittest/test_nsg/CMakeLists.txt diff --git a/cpp/src/core/test/test_nsg/test_nsg.cpp b/cpp/src/core/unittest/test_nsg/test_nsg.cpp similarity index 99% rename from cpp/src/core/test/test_nsg/test_nsg.cpp rename to cpp/src/core/unittest/test_nsg/test_nsg.cpp index 6f7ca1eba9..b93a2ddc35 100644 --- a/cpp/src/core/test/test_nsg/test_nsg.cpp +++ b/cpp/src/core/unittest/test_nsg/test_nsg.cpp @@ -24,7 +24,7 @@ #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #include "knowhere/index/vector_index/nsg/NSGIO.h" -#include "test/utils.h" +#include "unittest/utils.h" namespace { diff --git a/cpp/src/core/test/utils.cpp b/cpp/src/core/unittest/utils.cpp similarity index 99% rename from cpp/src/core/test/utils.cpp rename to cpp/src/core/unittest/utils.cpp index a921b69802..e81ad8ea02 100644 --- a/cpp/src/core/test/utils.cpp +++ b/cpp/src/core/unittest/utils.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "test/utils.h" +#include "unittest/utils.h" #include #include diff --git a/cpp/src/core/test/utils.h b/cpp/src/core/unittest/utils.h similarity index 100% rename from cpp/src/core/test/utils.h rename to cpp/src/core/unittest/utils.h diff --git a/cpp/src/wrapper/ConfAdapter.cpp b/cpp/src/wrapper/ConfAdapter.cpp index df1ac6bb70..09d219b999 100644 --- a/cpp/src/wrapper/ConfAdapter.cpp +++ b/cpp/src/wrapper/ConfAdapter.cpp @@ -81,7 +81,7 @@ IVFConfAdapter::MatchNlist(const int64_t& size, const int64_t& nlist) { // calculate a proper nlist if nlist not specified or size less than TYPICAL_COUNT return int(size / TYPICAL_COUNT * 16384); } - return 0; + return nlist; } knowhere::Config From 715d823d0b44294039fe988197e6da0f442546f7 Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 29 Sep 2019 11:52:55 +0800 Subject: [PATCH 147/217] MS-600 Reconstruct unittest code Former-commit-id: e48bed725840b4044dc4cb647905972c193ebacd --- cpp/CHANGELOG.md | 1 + cpp/src/CMakeLists.txt | 55 +++++++++---------- cpp/src/sdk/CMakeLists.txt | 4 +- .../examples/grpcsimple/src/ClientTest.cpp | 1 - cpp/unittest/CMakeLists.txt | 33 ++++------- cpp/unittest/db/CMakeLists.txt | 16 ++---- cpp/unittest/db/{db_tests.cpp => test_db.cpp} | 0 .../{mysql_db_test.cpp => test_db_mysql.cpp} | 0 .../db/{engine_test.cpp => test_engine.cpp} | 0 .../db/{mem_test.cpp => test_mem.cpp} | 0 .../db/{meta_tests.cpp => test_meta.cpp} | 0 ...ysql_meta_test.cpp => test_meta_mysql.cpp} | 0 .../db/{misc_test.cpp => test_misc.cpp} | 0 .../db/{search_test.cpp => test_search.cpp} | 0 cpp/unittest/metrics/CMakeLists.txt | 13 ++--- ...etricbase_test.cpp => test_metricbase.cpp} | 0 .../{metrics_test.cpp => test_metrics.cpp} | 0 ...rometheus_test.cpp => test_prometheus.cpp} | 0 cpp/unittest/scheduler/CMakeLists.txt | 15 ++--- ...{algorithm_test.cpp => test_algorithm.cpp} | 0 .../{event_test.cpp => test_event.cpp} | 0 .../{node_test.cpp => test_node.cpp} | 0 .../{normal_test.cpp => test_normal.cpp} | 0 .../{resource_test.cpp => test_resource.cpp} | 0 ...ory_test.cpp => test_resource_factory.cpp} | 0 ...rce_mgr_test.cpp => test_resource_mgr.cpp} | 0 ...{schedinst_test.cpp => test_schedinst.cpp} | 0 ...{scheduler_test.cpp => test_scheduler.cpp} | 0 ...{tasktable_test.cpp => test_tasktable.cpp} | 0 cpp/unittest/server/CMakeLists.txt | 8 +-- .../server/{cache_test.cpp => test_cache.cpp} | 0 .../{config_test.cpp => test_config.cpp} | 0 .../server/{rpc_test.cpp => test_rpc.cpp} | 0 cpp/unittest/wrapper/CMakeLists.txt | 19 +++---- .../{wrapper_test.cpp => test_wrapper.cpp} | 2 +- 35 files changed, 69 insertions(+), 98 deletions(-) rename cpp/unittest/db/{db_tests.cpp => test_db.cpp} (100%) rename cpp/unittest/db/{mysql_db_test.cpp => test_db_mysql.cpp} (100%) rename cpp/unittest/db/{engine_test.cpp => test_engine.cpp} (100%) rename cpp/unittest/db/{mem_test.cpp => test_mem.cpp} (100%) rename cpp/unittest/db/{meta_tests.cpp => test_meta.cpp} (100%) rename cpp/unittest/db/{mysql_meta_test.cpp => test_meta_mysql.cpp} (100%) rename cpp/unittest/db/{misc_test.cpp => test_misc.cpp} (100%) rename cpp/unittest/db/{search_test.cpp => test_search.cpp} (100%) rename cpp/unittest/metrics/{metricbase_test.cpp => test_metricbase.cpp} (100%) rename cpp/unittest/metrics/{metrics_test.cpp => test_metrics.cpp} (100%) rename cpp/unittest/metrics/{prometheus_test.cpp => test_prometheus.cpp} (100%) rename cpp/unittest/scheduler/{algorithm_test.cpp => test_algorithm.cpp} (100%) rename cpp/unittest/scheduler/{event_test.cpp => test_event.cpp} (100%) rename cpp/unittest/scheduler/{node_test.cpp => test_node.cpp} (100%) rename cpp/unittest/scheduler/{normal_test.cpp => test_normal.cpp} (100%) rename cpp/unittest/scheduler/{resource_test.cpp => test_resource.cpp} (100%) rename cpp/unittest/scheduler/{resource_factory_test.cpp => test_resource_factory.cpp} (100%) rename cpp/unittest/scheduler/{resource_mgr_test.cpp => test_resource_mgr.cpp} (100%) rename cpp/unittest/scheduler/{schedinst_test.cpp => test_schedinst.cpp} (100%) rename cpp/unittest/scheduler/{scheduler_test.cpp => test_scheduler.cpp} (100%) rename cpp/unittest/scheduler/{tasktable_test.cpp => test_tasktable.cpp} (100%) rename cpp/unittest/server/{cache_test.cpp => test_cache.cpp} (100%) rename cpp/unittest/server/{config_test.cpp => test_config.cpp} (100%) rename cpp/unittest/server/{rpc_test.cpp => test_rpc.cpp} (100%) rename cpp/unittest/wrapper/{wrapper_test.cpp => test_wrapper.cpp} (99%) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 33d34fcd34..a41bc23484 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -33,6 +33,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-575 - Add Clang-format & Clang-tidy & Cpplint - MS-586 - Remove BUILD_FAISS_WITH_MKL option - MS-590 - Refine cmake code to support cpplint +- MS-600 - Reconstruct unittest code # Milvus 0.4.0 (2019-09-12) diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 6ee3966c4f..9810c6b7d8 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -20,29 +20,24 @@ include_directories(${MILVUS_SOURCE_DIR}) include_directories(${MILVUS_ENGINE_SRC}) -add_subdirectory(core) +include_directories(${CUDA_TOOLKIT_ROOT_DIR}/include) +include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status) +include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus) +#this statement must put here, since the CORE_INCLUDE_DIRS is defined in code/CMakeList.txt +add_subdirectory(core) set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) foreach (dir ${CORE_INCLUDE_DIRS}) include_directories(${dir}) endforeach () aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files) - aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) - +aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler db_scheduler_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context db_scheduler_context_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task db_scheduler_task_files) -set(db_scheduler_files - ${db_scheduler_files} - ${db_scheduler_context_files} - ${db_scheduler_task_files} - ) set(grpc_service_files ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.grpc.pb.cc @@ -51,8 +46,6 @@ set(grpc_service_files ${MILVUS_ENGINE_SRC}/grpc/gen-status/status.pb.cc ) -aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files) - aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_main_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_files) @@ -70,9 +63,7 @@ set(scheduler_files aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files) aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_files) - aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files) - aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_files) set(engine_files @@ -82,16 +73,11 @@ set(engine_files ${db_engine_files} ${db_insert_files} ${db_meta_files} - ${db_scheduler_files} ${metrics_files} ${utils_files} ${wrapper_files} ) -include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") -include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status) -include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus) - set(client_grpc_lib grpcpp_channelz grpc++ @@ -112,6 +98,12 @@ set(boost_lib boost_serialization_static ) +set(cuda_lib + ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so + cudart + cublas + ) + set(third_party_libs sqlite ${client_grpc_lib} @@ -123,17 +115,15 @@ set(third_party_libs snappy zlib zstd - cudart - cublas + ${cuda_lib} mysqlpp - ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so - cudart ) + if (MILVUS_ENABLE_PROFILING STREQUAL "ON") set(third_party_libs ${third_party_libs} - gperftools - libunwind - ) + gperftools + libunwind + ) endif () link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") @@ -141,7 +131,6 @@ set(engine_libs pthread libgomp.a libgfortran.a - ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so ) if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") @@ -152,7 +141,11 @@ if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") endif () cuda_add_library(milvus_engine STATIC ${engine_files}) -target_link_libraries(milvus_engine ${engine_libs} knowhere ${third_party_libs}) +target_link_libraries(milvus_engine + knowhere + ${engine_libs} + ${third_party_libs} + ) add_library(metrics STATIC ${metrics_files}) @@ -180,7 +173,9 @@ add_executable(milvus_server ${utils_files} ) -target_link_libraries(milvus_server ${server_libs}) +target_link_libraries(milvus_server + ${server_libs} + ) install(TARGETS milvus_server DESTINATION bin) diff --git a/cpp/src/sdk/CMakeLists.txt b/cpp/src/sdk/CMakeLists.txt index e7bf989e96..a2991a49b4 100644 --- a/cpp/src/sdk/CMakeLists.txt +++ b/cpp/src/sdk/CMakeLists.txt @@ -17,11 +17,9 @@ # under the License. #------------------------------------------------------------------------------- - -aux_source_directory(interface interface_files) - include_directories(include) +aux_source_directory(interface interface_files) aux_source_directory(grpc grpc_client_files) add_library(milvus_sdk STATIC diff --git a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp index 45ea707975..ce511714b2 100644 --- a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp @@ -17,7 +17,6 @@ #include "sdk/examples/grpcsimple/src/ClientTest.h" #include "MilvusApi.h" -#include "cache/CpuCacheMgr.h" #include #include diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 0f454ccc43..6c9aeadcd1 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -17,25 +17,25 @@ # under the License. #------------------------------------------------------------------------------- +include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") + +foreach(dir ${CORE_INCLUDE_DIRS}) + include_directories(${dir}) +endforeach() + include_directories(${MILVUS_SOURCE_DIR}) include_directories(${MILVUS_ENGINE_SRC}) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + +link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files) - aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) - +aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler db_scheduler_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context db_scheduler_context_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task db_scheduler_task_files) -set(db_scheduler_files - ${db_scheduler_files} - ${db_scheduler_context_files} - ${db_scheduler_task_files} - ) set(grpc_service_files ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.grpc.pb.cc @@ -44,8 +44,6 @@ set(grpc_service_files ${MILVUS_ENGINE_SRC}/grpc/gen-status/status.pb.cc ) -aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files) - aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_main_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_files) @@ -63,12 +61,10 @@ set(scheduler_files aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files) aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_files) - aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files) - aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_files) -set(unittest_files +set(entry_file ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp) set(helper_files @@ -87,7 +83,6 @@ set(common_files ${db_engine_files} ${db_insert_files} ${db_meta_files} - ${db_scheduler_files} ${metrics_files} ${scheduler_files} ${wrapper_files} @@ -118,12 +113,6 @@ set(unittest_libs cublas ) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -message(STATUS "CMAKE_CURRENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}") -foreach(dir ${CORE_INCLUDE_DIRS}) - include_directories(${dir}) -endforeach() - add_subdirectory(db) add_subdirectory(wrapper) add_subdirectory(metrics) diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 91a51e484b..2cbf55a208 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -20,18 +20,14 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files) -include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") -link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") - - -set(db_test_files +cuda_add_executable(test_db ${common_files} ${test_files} - ) + ) -cuda_add_executable(db_test ${db_test_files}) +target_link_libraries(test_db + knowhere + ${unittest_libs}) -target_link_libraries(db_test knowhere ${unittest_libs}) - -install(TARGETS db_test DESTINATION unittest) +install(TARGETS test_db DESTINATION unittest) diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/test_db.cpp similarity index 100% rename from cpp/unittest/db/db_tests.cpp rename to cpp/unittest/db/test_db.cpp diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/test_db_mysql.cpp similarity index 100% rename from cpp/unittest/db/mysql_db_test.cpp rename to cpp/unittest/db/test_db_mysql.cpp diff --git a/cpp/unittest/db/engine_test.cpp b/cpp/unittest/db/test_engine.cpp similarity index 100% rename from cpp/unittest/db/engine_test.cpp rename to cpp/unittest/db/test_engine.cpp diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/test_mem.cpp similarity index 100% rename from cpp/unittest/db/mem_test.cpp rename to cpp/unittest/db/test_mem.cpp diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/test_meta.cpp similarity index 100% rename from cpp/unittest/db/meta_tests.cpp rename to cpp/unittest/db/test_meta.cpp diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/test_meta_mysql.cpp similarity index 100% rename from cpp/unittest/db/mysql_meta_test.cpp rename to cpp/unittest/db/test_meta_mysql.cpp diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/test_misc.cpp similarity index 100% rename from cpp/unittest/db/misc_test.cpp rename to cpp/unittest/db/test_misc.cpp diff --git a/cpp/unittest/db/search_test.cpp b/cpp/unittest/db/test_search.cpp similarity index 100% rename from cpp/unittest/db/search_test.cpp rename to cpp/unittest/db/test_search.cpp diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 04eab826bc..eba8146baa 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -19,16 +19,13 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files) -include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") -link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") - -set(metrics_test_files +add_executable(test_metrics ${common_files} ${test_files} ) -add_executable(metrics_test ${metrics_test_files}) +target_link_libraries(test_metrics + knowhere + ${unittest_libs}) -target_link_libraries(metrics_test knowhere ${unittest_libs}) - -install(TARGETS metrics_test DESTINATION unittest) \ No newline at end of file +install(TARGETS test_metrics DESTINATION unittest) \ No newline at end of file diff --git a/cpp/unittest/metrics/metricbase_test.cpp b/cpp/unittest/metrics/test_metricbase.cpp similarity index 100% rename from cpp/unittest/metrics/metricbase_test.cpp rename to cpp/unittest/metrics/test_metricbase.cpp diff --git a/cpp/unittest/metrics/metrics_test.cpp b/cpp/unittest/metrics/test_metrics.cpp similarity index 100% rename from cpp/unittest/metrics/metrics_test.cpp rename to cpp/unittest/metrics/test_metrics.cpp diff --git a/cpp/unittest/metrics/prometheus_test.cpp b/cpp/unittest/metrics/test_prometheus.cpp similarity index 100% rename from cpp/unittest/metrics/prometheus_test.cpp rename to cpp/unittest/metrics/test_prometheus.cpp diff --git a/cpp/unittest/scheduler/CMakeLists.txt b/cpp/unittest/scheduler/CMakeLists.txt index 27ee11775c..087f93f017 100644 --- a/cpp/unittest/scheduler/CMakeLists.txt +++ b/cpp/unittest/scheduler/CMakeLists.txt @@ -19,18 +19,15 @@ aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files) -include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") -link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") - -set(scheduler_test_files +cuda_add_executable(test_scheduler ${common_files} - ${unittest_files} + ${entry_file} ${test_files} ) -cuda_add_executable(scheduler_test ${scheduler_test_files}) +target_link_libraries(test_scheduler + knowhere + ${unittest_libs}) -target_link_libraries(scheduler_test knowhere ${unittest_libs}) - -install(TARGETS scheduler_test DESTINATION unittest) +install(TARGETS test_scheduler DESTINATION unittest) diff --git a/cpp/unittest/scheduler/algorithm_test.cpp b/cpp/unittest/scheduler/test_algorithm.cpp similarity index 100% rename from cpp/unittest/scheduler/algorithm_test.cpp rename to cpp/unittest/scheduler/test_algorithm.cpp diff --git a/cpp/unittest/scheduler/event_test.cpp b/cpp/unittest/scheduler/test_event.cpp similarity index 100% rename from cpp/unittest/scheduler/event_test.cpp rename to cpp/unittest/scheduler/test_event.cpp diff --git a/cpp/unittest/scheduler/node_test.cpp b/cpp/unittest/scheduler/test_node.cpp similarity index 100% rename from cpp/unittest/scheduler/node_test.cpp rename to cpp/unittest/scheduler/test_node.cpp diff --git a/cpp/unittest/scheduler/normal_test.cpp b/cpp/unittest/scheduler/test_normal.cpp similarity index 100% rename from cpp/unittest/scheduler/normal_test.cpp rename to cpp/unittest/scheduler/test_normal.cpp diff --git a/cpp/unittest/scheduler/resource_test.cpp b/cpp/unittest/scheduler/test_resource.cpp similarity index 100% rename from cpp/unittest/scheduler/resource_test.cpp rename to cpp/unittest/scheduler/test_resource.cpp diff --git a/cpp/unittest/scheduler/resource_factory_test.cpp b/cpp/unittest/scheduler/test_resource_factory.cpp similarity index 100% rename from cpp/unittest/scheduler/resource_factory_test.cpp rename to cpp/unittest/scheduler/test_resource_factory.cpp diff --git a/cpp/unittest/scheduler/resource_mgr_test.cpp b/cpp/unittest/scheduler/test_resource_mgr.cpp similarity index 100% rename from cpp/unittest/scheduler/resource_mgr_test.cpp rename to cpp/unittest/scheduler/test_resource_mgr.cpp diff --git a/cpp/unittest/scheduler/schedinst_test.cpp b/cpp/unittest/scheduler/test_schedinst.cpp similarity index 100% rename from cpp/unittest/scheduler/schedinst_test.cpp rename to cpp/unittest/scheduler/test_schedinst.cpp diff --git a/cpp/unittest/scheduler/scheduler_test.cpp b/cpp/unittest/scheduler/test_scheduler.cpp similarity index 100% rename from cpp/unittest/scheduler/scheduler_test.cpp rename to cpp/unittest/scheduler/test_scheduler.cpp diff --git a/cpp/unittest/scheduler/tasktable_test.cpp b/cpp/unittest/scheduler/test_tasktable.cpp similarity index 100% rename from cpp/unittest/scheduler/tasktable_test.cpp rename to cpp/unittest/scheduler/test_tasktable.cpp diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index d5bbd6cc47..4420e2a1a7 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -43,11 +43,11 @@ set(server_test_files ${grpc_server_files} ${grpc_service_files} ${util_files} - ${unittest_files} + ${entry_file} ${test_files} ) -cuda_add_executable(server_test ${server_test_files}) +cuda_add_executable(test_server ${server_test_files}) set(client_grpc_lib grpcpp_channelz @@ -56,7 +56,7 @@ set(client_grpc_lib grpc_protobuf grpc_protoc) -target_link_libraries(server_test +target_link_libraries(test_server knowhere stdc++ snappy @@ -66,7 +66,7 @@ target_link_libraries(server_test ${unittest_libs} ) -install(TARGETS server_test DESTINATION unittest) +install(TARGETS test_server DESTINATION unittest) configure_file(appendix/server_config.yaml "${CMAKE_CURRENT_BINARY_DIR}/milvus/conf/server_config.yaml" diff --git a/cpp/unittest/server/cache_test.cpp b/cpp/unittest/server/test_cache.cpp similarity index 100% rename from cpp/unittest/server/cache_test.cpp rename to cpp/unittest/server/test_cache.cpp diff --git a/cpp/unittest/server/config_test.cpp b/cpp/unittest/server/test_config.cpp similarity index 100% rename from cpp/unittest/server/config_test.cpp rename to cpp/unittest/server/test_config.cpp diff --git a/cpp/unittest/server/rpc_test.cpp b/cpp/unittest/server/test_rpc.cpp similarity index 100% rename from cpp/unittest/server/rpc_test.cpp rename to cpp/unittest/server/test_rpc.cpp diff --git a/cpp/unittest/wrapper/CMakeLists.txt b/cpp/unittest/wrapper/CMakeLists.txt index 633cbaf053..8eae47b3d4 100644 --- a/cpp/unittest/wrapper/CMakeLists.txt +++ b/cpp/unittest/wrapper/CMakeLists.txt @@ -17,8 +17,7 @@ # under the License. #------------------------------------------------------------------------------- -include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") -link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files) set(wrapper_files ${MILVUS_ENGINE_SRC}/wrapper/DataTransfer.cpp @@ -31,13 +30,13 @@ set(util_files ${MILVUS_ENGINE_SRC}/utils/Status.cpp ) -set(knowhere_libs +add_executable(test_wrapper + ${test_files} + ${wrapper_files} + ${util_files}) + +target_link_libraries(test_wrapper knowhere - cudart - cublas - ) + ${unittest_libs}) -add_executable(wrapper_test wrapper_test.cpp ${wrapper_files} ${util_files}) -target_link_libraries(wrapper_test ${knowhere_libs} ${unittest_libs}) - -install(TARGETS wrapper_test DESTINATION unittest) \ No newline at end of file +install(TARGETS test_wrapper DESTINATION unittest) \ No newline at end of file diff --git a/cpp/unittest/wrapper/wrapper_test.cpp b/cpp/unittest/wrapper/test_wrapper.cpp similarity index 99% rename from cpp/unittest/wrapper/wrapper_test.cpp rename to cpp/unittest/wrapper/test_wrapper.cpp index 432739e482..b97b96d205 100644 --- a/cpp/unittest/wrapper/wrapper_test.cpp +++ b/cpp/unittest/wrapper/test_wrapper.cpp @@ -16,7 +16,7 @@ // under the License. #include "utils/easylogging++.h" -#include "src/wrapper/VecIndex.h" +#include "wrapper/VecIndex.h" #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" #include "knowhere/index/vector_index/helpers/IndexParameter.h" #include "wrapper/utils.h" From 7633425ff5bb412281860b1500997ec8a4fc65df Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sun, 29 Sep 2019 14:27:32 +0800 Subject: [PATCH 148/217] add resources limits Former-commit-id: 625e61cec4d2a64e52faa5b64f355e3671e7ec27 --- ci/main_jenkinsfile | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index 40224fe894..d65ad512c2 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -33,12 +33,30 @@ pipeline { cloud 'build-kubernetes' label 'build' defaultContainer 'jnlp' - containerTemplate { - name 'milvus-build-env' - image 'registry.zilliz.com/milvus/milvus-build-env:v0.12' - ttyEnabled true - command 'cat' - } + yaml """ +apiVersion: v1 +kind: Pod +metadata: + name: milvus-build-env + labels: + app: milvus + componet: build-env +spec: + containers: + - name: milvus-build-env + image: registry.zilliz.com/milvus/milvus-build-env:v0.12 + command: + - cat + tty: true + resources: + limits: + memory: "28Gi" + cpu: "10.0" + nvidia.com/gpu: 1 + requests: + memory: "14Gi" + cpu: "5.0" +""" } } stages { From eb23e112c19b84a3389f750b0290e1e8c65077f9 Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 29 Sep 2019 14:33:01 +0800 Subject: [PATCH 149/217] fix code coverage error Former-commit-id: 9242dca4a2cd1eef432b4d4d5eb6da10efff9935 --- cpp/coverage.sh | 6 +++--- cpp/src/db/meta/MySQLConnectionPool.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/coverage.sh b/cpp/coverage.sh index 84459fcc84..c0defe6f5a 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -70,11 +70,11 @@ fi for test in `ls ${DIR_UNITTEST}`; do echo $test case ${test} in - db_test) - # set run args for db_test + test_db) + # set run args for test_db args="mysql://${MYSQL_USER_NAME}:${MYSQL_PASSWORD}@${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB_NAME}" ;; - *_test) + *test_*) args="" ;; esac diff --git a/cpp/src/db/meta/MySQLConnectionPool.h b/cpp/src/db/meta/MySQLConnectionPool.h index 1b5345c762..9c6b751d1e 100644 --- a/cpp/src/db/meta/MySQLConnectionPool.h +++ b/cpp/src/db/meta/MySQLConnectionPool.h @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "mysql++/mysql++.h" +#include #include #include From fd25f05979c3462ed921cdeb5fce1f1d3c7b0c88 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sun, 29 Sep 2019 15:04:27 +0800 Subject: [PATCH 150/217] add resources limits Former-commit-id: 5f249b0c9ce1a2c5667512e2420941f115a797aa --- ci/main_jenkinsfile_no_ut | 30 ++++++++++++++++++++++++------ ci/nightly_main_jenkinsfile | 30 ++++++++++++++++++++++++------ 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut index 9948322c3f..72f6e04948 100644 --- a/ci/main_jenkinsfile_no_ut +++ b/ci/main_jenkinsfile_no_ut @@ -33,12 +33,30 @@ pipeline { cloud 'build-kubernetes' label 'build' defaultContainer 'jnlp' - containerTemplate { - name 'milvus-build-env' - image 'registry.zilliz.com/milvus/milvus-build-env:v0.12' - ttyEnabled true - command 'cat' - } + yaml """ +apiVersion: v1 +kind: Pod +metadata: + name: milvus-build-env + labels: + app: milvus + componet: build-env +spec: + containers: + - name: milvus-build-env + image: registry.zilliz.com/milvus/milvus-build-env:v0.12 + command: + - cat + tty: true + resources: + limits: + memory: "28Gi" + cpu: "10.0" + nvidia.com/gpu: 1 + requests: + memory: "14Gi" + cpu: "5.0" +""" } } stages { diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index 28352e0c83..51c733f0ea 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -33,12 +33,30 @@ pipeline { cloud 'build-kubernetes' label 'build' defaultContainer 'jnlp' - containerTemplate { - name 'milvus-build-env' - image 'registry.zilliz.com/milvus/milvus-build-env:v0.12' - ttyEnabled true - command 'cat' - } + yaml """ +apiVersion: v1 +kind: Pod +metadata: + name: milvus-build-env + labels: + app: milvus + componet: build-env +spec: + containers: + - name: milvus-build-env + image: registry.zilliz.com/milvus/milvus-build-env:v0.12 + command: + - cat + tty: true + resources: + limits: + memory: "28Gi" + cpu: "10.0" + nvidia.com/gpu: 1 + requests: + memory: "14Gi" + cpu: "5.0" +""" } } stages { From 26a6bb6b87c0bd9656b03cc359c2e81551432030 Mon Sep 17 00:00:00 2001 From: wxyu Date: Sun, 29 Sep 2019 15:05:00 +0800 Subject: [PATCH 151/217] MS-599 search wrong result when table created with metric_type: IP Former-commit-id: 7dd107fa0126919ba7948cc0bacb6b1426f2d7c8 --- cpp/CHANGELOG.md | 1 + cpp/build-support/code_style_clion.xml | 3 +-- cpp/src/scheduler/task/SearchTask.cpp | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 33d34fcd34..d3e2318cd1 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -9,6 +9,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-572 - Milvus crash when get SIGINT - MS-577 - Unittest Query randomly hung - MS-587 - Count get wrong result after adding vectors and index built immediately +- MS-599 - search wrong result when table created with metric_type: IP ## Improvement - MS-552 - Add and change the easylogging library diff --git a/cpp/build-support/code_style_clion.xml b/cpp/build-support/code_style_clion.xml index 4c7dc4718c..f2edeec3b6 100644 --- a/cpp/build-support/code_style_clion.xml +++ b/cpp/build-support/code_style_clion.xml @@ -1,4 +1,4 @@ - +